diff --git a/docker/Dockerfile b/docker/Dockerfile index a773e521..c6cc2ec2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -52,19 +52,13 @@ ARG CLAUDE_CODE_VERSION=2.1.177 RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && rm -rf /var/lib/apt/lists/* \ - && npm config set prefix /usr/local \ && npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} -# Every built-in agent backend routes its SDK-owned subprocess through the -# same setpriv policy. Claude's SDK accepts only one executable path, so it -# uses the small backend-specific wrapper below; Codex and Antigravity invoke -# the generic launcher directly. +# The trusted harness uses this launcher for one stateful worker containing +# the complete registry-selected Agent lifecycle. Every worker descendant +# inherits the same UID/GID, empty capability sets, and no-new-privileges bit. COPY docker/coder_eval_drop_privilege.sh /usr/local/bin/coder_eval_drop_privilege.sh -COPY docker/coder_eval_claude_agent.sh /usr/local/bin/coder_eval_claude_agent.sh -RUN chmod 0555 \ - /usr/local/bin/coder_eval_drop_privilege.sh \ - /usr/local/bin/coder_eval_claude_agent.sh \ - && test "$(command -v claude)" = "/usr/local/bin/claude" +RUN chmod 0555 /usr/local/bin/coder_eval_drop_privilege.sh # uv: matches host sandbox.py's `uv venv` + `uv pip install` fast path RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh @@ -102,8 +96,10 @@ ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS}" && \ uv export --frozen --extra codex --extra antigravity ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin -# Sanity check: the in-container entrypoint subcommand must be wired up. -RUN coder-eval _run-task-internal --help > /dev/null +# Sanity check the public container entrypoint and the worker module that the +# isolation proxy launches directly. +RUN coder-eval _run-task-internal --help > /dev/null \ + && python -I -m coder_eval.isolation.agent_worker < /dev/null # Stamp the image with the installed coder_eval version so the host can do # a pre-flight version assertion against `docker image inspect` BEFORE diff --git a/docker/coder_eval_claude_agent.sh b/docker/coder_eval_claude_agent.sh deleted file mode 100644 index 1b6be19d..00000000 --- a/docker/coder_eval_claude_agent.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -exec /usr/local/bin/coder_eval_drop_privilege.sh /usr/local/bin/claude "$@" diff --git a/docs/DOCKER_ISOLATION.md b/docs/DOCKER_ISOLATION.md index 7f4c8064..a7e1af0b 100644 --- a/docs/DOCKER_ISOLATION.md +++ b/docs/DOCKER_ISOLATION.md @@ -36,11 +36,11 @@ Both build `coder-eval-agent:` and tag it `:latest`. ## Agent/grader identity boundary -`sandbox.docker.agent_isolation` defaults to `true`. The container harness and grader remain root, while every evaluated Claude, Codex, or Antigravity subprocess runs as `agent:agent` (`2000:2000`). +`sandbox.docker.agent_isolation` defaults to `true`. The container harness and grader remain root, while the complete lifecycle of the registry-selected agent runs in a stateful worker as `agent:agent` (`2000:2000`). This applies equally to the built-in agents and to third-party `AgentRegistry` plugins installed in the run image; there is no built-in-kind allowlist. The agent launcher clears inheritable, ambient, and bounding capabilities and sets `no_new_privs`. Generated work is placed in `/work/agent`. Hidden task data, results, raw task/plugin/reference/template sources, and grader inputs live below root-only `/opt/coder-eval/grader`. Raw source bind mounts remain read-only and are never chmod/chowned; only disposable staging copies and the generated workspace are changed. -Older/custom images must declare `org.coder-eval.agent-isolation=uid-gid-v1`. A protected run rejects an image without that label before making an LLM call. Images derived with `FROM coder-eval-agent:` inherit it. Runtime-kit injection into an unrelated base does not yet provide the required Linux users and `setpriv` launchers, so it is not compatible with protected mode. +Older/custom images must declare `org.coder-eval.agent-isolation=uid-gid-v1`. A protected run rejects an image without that label before making an LLM call. Images derived with `FROM coder-eval-agent:` inherit it. Runtime-kit injection into an unrelated base does not yet provide the required Linux user and `setpriv` launcher, so it is not compatible with protected mode. ## Running a task in Docker @@ -321,7 +321,7 @@ The host's run dir is bind-mounted read-write at `/opt/coder-eval/grader/output` The host's `DockerRunner` rewrites host paths to protected container paths, renders `docker run`, and tails container stdout into `docker.log`. Inputs land at `/opt/coder-eval/grader/input`, output at `/opt/coder-eval/grader/output`, the raw task directory at `/opt/coder-eval/grader/task_dir`, and the agent workspace at `/work/agent`. -Inside the container, the root entrypoint verifies the protected parent. The standard Orchestrator prepares the workspace as root, grants only that generated tree to UID 2000, and launches the selected agent through the shared privilege-drop policy. The host reads the final result from the protected output mount and feeds the existing aggregation pipeline. +Inside the container, the root entrypoint verifies the protected parent. The standard Orchestrator prepares the workspace as root, grants only that generated tree to UID 2000, and launches one stateful agent worker through the shared privilege-drop policy. The worker loads the same registry, constructs the selected agent, and owns `start` / `communicate` / `stop` plus all descendants. Stream events and typed turn results cross back to the root orchestrator over a framed protocol; no agent implementation is instantiated in the privileged process. The host reads the final result from the protected output mount and feeds the existing aggregation pipeline. Protected runs use Docker's init reaper and default to a 512-process limit when `limits.max_pids` is not specified. An explicit `max_pids` value takes precedence. Before trusted post-run/finalization begins, the harness stops the SDK, repeatedly kills every remaining UID-2000 process, and fails closed if that UID cannot be emptied. diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 49ef119c..4e1475be 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -22,9 +22,6 @@ import contextlib import logging import os -import shlex -import shutil -import tempfile import time from collections.abc import AsyncIterator, Callable from contextlib import AsyncExitStack @@ -42,10 +39,7 @@ TurnTimeoutError, truncate_crash_message, ) -from coder_eval.isolation.agent_identity import agent_isolation_enabled from coder_eval.models import ( - AGENT_HOME, - CONTAINER_DROP_SHIM, AgentKind, AntigravityAgentConfig, ApiRoute, @@ -72,11 +66,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.utils import ( - AGENT_ENV_SCRUB_PREFIXES, - AGENT_ENV_SCRUB_VARS, - expand_env_vars, -) +from coder_eval.utils import expand_env_vars logger = logging.getLogger(__name__) @@ -225,7 +215,6 @@ def __init__( # Absolute dirs to prepend to PATH so sandbox mock CLIs shadow real ones # for the harness's run_command tool — applied at spawn (see start()). self._env_path_prepend: list[str] = [] - self._drop_shim_dir: Path | None = None # _state / _iteration / _iteration_was_incremented / pending_turn lifecycle # bookkeeping lives on the Agent base class (shared defaults + helpers). self._log = PrefixedAdapter(logger, {"prefix": instance_name}) @@ -234,22 +223,6 @@ def _effective_model(self) -> str: """Resolve the model: task ``agent.model`` > ``ANTIGRAVITY_MODEL`` > default.""" return self.config.model or settings.antigravity_model or _DEFAULT_MODEL - def _stage_localharness_drop_shim(self) -> Path: - """Shadow localharness with a wrapper around the shared setpriv policy.""" - - real = shutil.which("localharness") - if real is None: - raise RuntimeError("agent isolation is enabled but localharness is not available on PATH") - shim_dir = Path(tempfile.mkdtemp(prefix="antigravity-drop-")) - wrapper = shim_dir / "localharness" - wrapper.write_text( - f'#!/usr/bin/env bash\nexec {CONTAINER_DROP_SHIM} {shlex.quote(real)} "$@"\n', - encoding="utf-8", - ) - wrapper.chmod(0o555) - self._drop_shim_dir = shim_dir - return shim_dir - def _resolve_skills_paths(self, plugin_tools_dir: str | None) -> list[str]: """Resolve skill search-path roots for the harness's native ``skills_paths``. @@ -343,8 +316,6 @@ async def start( """ self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) - if agent_isolation_enabled(): - self._env_path_prepend.insert(0, str(self._stage_localharness_drop_shim())) self._state = AgentState.WORKING try: @@ -420,33 +391,20 @@ async def _harness_spawn_guard(self) -> AsyncIterator[None]: PATH window, or its harness would inherit another task's mock dirs. """ async with _harness_spawn_lock(): - scrubbed = { - name: os.environ.pop(name) - for name in list(os.environ) - if name in AGENT_ENV_SCRUB_VARS or name.startswith(AGENT_ENV_SCRUB_PREFIXES) - } + if not self._env_path_prepend: + yield + return path_key = next((k for k in os.environ if k.upper() == "PATH"), "PATH") - original_path = os.environ.get(path_key) - original_home = os.environ.get("HOME") - if self._env_path_prepend: - os.environ[path_key] = os.pathsep.join([*self._env_path_prepend, original_path or ""]) - self._log.debug("PATH prepend for harness spawn: %s", os.pathsep.join(self._env_path_prepend)) - if agent_isolation_enabled(): - os.environ["HOME"] = AGENT_HOME + original = os.environ.get(path_key) + os.environ[path_key] = os.pathsep.join([*self._env_path_prepend, original or ""]) + self._log.debug("PATH prepend for harness spawn: %s", os.pathsep.join(self._env_path_prepend)) try: yield finally: - os.environ.update(scrubbed) - if self._env_path_prepend: - if original_path is None: - os.environ.pop(path_key, None) - else: - os.environ[path_key] = original_path - if agent_isolation_enabled(): - if original_home is None: - os.environ.pop("HOME", None) - else: - os.environ["HOME"] = original_home + if original is None: + os.environ.pop(path_key, None) + else: + os.environ[path_key] = original async def communicate( self, @@ -621,10 +579,6 @@ async def _teardown(self) -> None: if stack is not None: with contextlib.suppress(Exception): await stack.aclose() - shim_dir, self._drop_shim_dir = self._drop_shim_dir, None - if shim_dir is not None: - with contextlib.suppress(Exception): - await asyncio.to_thread(shutil.rmtree, shim_dir, ignore_errors=True) class _AntigravityTurnState: diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index de678f72..71cb2267 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -37,10 +37,7 @@ format_timeout_reason, ) from coder_eval.formatting import format_messages, format_payload -from coder_eval.isolation.agent_identity import agent_isolation_enabled from coder_eval.models import ( - AGENT_HOME, - CONTAINER_CLAUDE_SHIM, AgentKind, ApiRoute, BedrockRoute, @@ -73,7 +70,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.utils import dump_dataclass, process_plugins, scrub_agent_env_overrides +from coder_eval.utils import dump_dataclass, process_plugins logger = logging.getLogger(__name__) @@ -768,11 +765,9 @@ def _build_sdk_env( Returns: Tuple of (env_vars_dict, model_override_or_None). """ - base_env: dict[str, str] = scrub_agent_env_overrides() + base_env: dict[str, str] = {} if path := os.environ.get("PATH"): base_env["PATH"] = path - if agent_isolation_enabled(): - base_env["HOME"] = AGENT_HOME if path_prepend: prefix = os.pathsep.join(path_prepend) @@ -1204,10 +1199,6 @@ def _build_claude_query( if isinstance(self.config.claude_settings, dict) else self.config.claude_settings, mcp_servers=self._extra_mcp_servers, - # The SDK accepts a single CLI executable path. The baked wrapper - # invokes the real Claude binary through the same setpriv policy as - # the other backends (UID/GID drop, no capabilities, no_new_privs). - cli_path=CONTAINER_CLAUDE_SHIM if agent_isolation_enabled() else None, **self.config.sdk_options, ) diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 68a6262e..c6e4b2d3 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -25,10 +25,7 @@ TurnTimeoutError, truncate_crash_message, ) -from coder_eval.isolation.agent_identity import agent_isolation_enabled, grant_agent_workspace from coder_eval.models import ( - AGENT_HOME, - CONTAINER_DROP_SHIM, AgentKind, ApiRoute, AssistantMessage, @@ -55,7 +52,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.utils import expand_env_vars, scrub_agent_env_overrides +from coder_eval.utils import expand_env_vars logger = logging.getLogger(__name__) @@ -663,7 +660,6 @@ def __init__( self.working_directory: Path | None = None self._env_path_prepend: list[str] = [] self._login_shell_home: Path | None = None - self._runtime_codex_home: Path | None = None # _state / _iteration / _iteration_was_incremented / pending_turn lifecycle # bookkeeping lives on the Agent base class (shared defaults + helpers). self._log = PrefixedAdapter(logger, {"prefix": instance_name}) @@ -693,13 +689,6 @@ async def start( self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) self._setup_login_shell_home() - if agent_isolation_enabled(): - if self._login_shell_home is None: - self._login_shell_home = Path(tempfile.mkdtemp(prefix="coder-eval-codex-home-")) - self._runtime_codex_home = Path(AGENT_HOME) / ".codex" - await asyncio.to_thread(self._runtime_codex_home.mkdir, parents=True, exist_ok=True) - grant_agent_workspace(self._login_shell_home) - grant_agent_workspace(self._runtime_codex_home) self._state = AgentState.WORKING try: @@ -707,11 +696,7 @@ async def start( # Build CodexConfig with environment variables for custom API configuration env_override = self._build_codex_env() - launch_args_override = self._drop_privilege_launch_args() - if env_override is not None or launch_args_override is not None: - config = CodexConfig(env=env_override, launch_args_override=launch_args_override) - else: - config = None + config = CodexConfig(env=env_override) if env_override else None # Initialize the Codex client (context manager compatible). Close any # prior client first: start() is driven through execute_with_retry, so @@ -1092,16 +1077,6 @@ def _effective_model(self) -> str | None: """ return self.config.model or settings.codex_model - @staticmethod - def _drop_privilege_launch_args() -> tuple[str, ...] | None: - """Replace the SDK app-server argv with the shared UID-drop launcher.""" - - if not agent_isolation_enabled(): - return None - from codex_cli_bin import bundled_codex_path - - return (CONTAINER_DROP_SHIM, str(bundled_codex_path()), "app-server", "--listen", "stdio://") - def _build_codex_env(self) -> dict[str, str] | None: """Build the environment passed to the Codex app-server. @@ -1116,7 +1091,7 @@ def _build_codex_env(self) -> dict[str, str] | None: (and normalizes the PATH key case-insensitively), so a full PATH value here safely replaces the inherited one. """ - env: dict[str, str] = scrub_agent_env_overrides() + env: dict[str, str] = {} api_key = os.getenv("CODEX_API_KEY") if api_key: env["CODEX_API_KEY"] = api_key @@ -1141,12 +1116,6 @@ def _build_codex_env(self) -> dict[str, str] | None: codex_home = self._codex_home() codex_home.mkdir(parents=True, exist_ok=True) env["CODEX_HOME"] = str(codex_home) - elif agent_isolation_enabled(): - env["HOME"] = AGENT_HOME - env["ZDOTDIR"] = AGENT_HOME - codex_home = self._codex_home() - codex_home.mkdir(parents=True, exist_ok=True) - env["CODEX_HOME"] = str(codex_home) return env if env else None @staticmethod @@ -1188,14 +1157,10 @@ def _setup_login_shell_home(self) -> None: self._cleanup_login_shell_home() if not (self._env_path_prepend and self._login_shell_profiles_supported()): return - # The harness remains root in protected Docker runs. Its HOME and - # ZDOTDIR are private grader state and must never be restored by an - # agent login shell. Use the dedicated agent home as both the runtime - # home and the only profile source in that mode. - original_home = AGENT_HOME if agent_isolation_enabled() else os.environ.get("HOME", "") + original_home = os.environ.get("HOME", "") # Where the user's REAL zsh dotfiles live: their own ZDOTDIR when set, # else their home (zsh's fallback). - original_zdotdir = AGENT_HOME if agent_isolation_enabled() else os.environ.get("ZDOTDIR", "") or original_home + original_zdotdir = os.environ.get("ZDOTDIR", "") or original_home # The profile only ever executes under a POSIX shell, so the PATH # separator is ':' regardless of the host building it. quoted_prepend = shlex.quote(":".join(self._env_path_prepend)) @@ -1801,10 +1766,9 @@ async def _recover_subagent_tool_calls( # Best-effort: a recovery hiccup must never fail the turn. self._log.debug("CodexAgent: sub-agent recovery failed for %s: %s", thread_id, exc) - def _codex_home(self) -> Path: + @staticmethod + def _codex_home() -> Path: """Codex data directory (rollouts live under ``/sessions``).""" - if self._runtime_codex_home is not None: - return self._runtime_codex_home return Path(os.environ.get("CODEX_HOME") or (Path.home() / ".codex")) async def _await_rollout_file(self, home: Path, thread_id: str, *, attempts: int = 20) -> Path | None: diff --git a/src/coder_eval/isolation/agent_worker.py b/src/coder_eval/isolation/agent_worker.py new file mode 100644 index 00000000..ec1aa5ea --- /dev/null +++ b/src/coder_eval/isolation/agent_worker.py @@ -0,0 +1,663 @@ +"""Unprivileged process boundary for evaluated agents. + +The in-container orchestrator keeps ownership of trusted sandbox preparation and +grading. When UID/GID isolation is enabled it talks to exactly one stateful +worker process which constructs and drives the registry-selected ``Agent`` as +the dedicated ``agent`` user. This makes isolation independent of SDK-specific +subprocess hooks and therefore applies to third-party AgentRegistry plugins too. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import json +import logging +import os +import secrets +import signal +import sys +import tempfile +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any, NoReturn + +from coder_eval.agent import Agent +from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError +from coder_eval.models import ( + AGENT_HOME, + CONTAINER_DROP_SHIM, + AgentState, + ApiRoute, + BaseAgentConfig, + BedrockRoute, + DirectRoute, + LiteLLMRoute, + TurnRecord, +) +from coder_eval.streaming.callbacks import StreamCallback, safe_emit +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import StreamEvent +from coder_eval.streaming.wire import deserialize_event, serialize_event +from coder_eval.utils import SKIP, serialize_value + + +logger = logging.getLogger(__name__) + +_RPC_PREFIX = "\x1ecoder-eval-agent-rpc\x1e:" +_MAX_LINE_BYTES = 64 * 1024 * 1024 +_STOP_TIMEOUT_SECONDS = 5.0 +_SAFE_CODER_EVAL_ENV = frozenset({"CODER_EVAL_IN_CONTAINER"}) +_SCRUB_ENV_VARS = frozenset({"AWS_BEARER_TOKEN_BEDROCK", "SKILLS_REPO_PATH", "TASK_DIR"}) +_LINUX_CAPABILITY_FIELDS = ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb") + + +def build_agent_worker_environment() -> dict[str, str]: + """Return the least-privilege environment inherited by the agent worker.""" + + env = dict(os.environ) + for name in list(env): + if name in _SCRUB_ENV_VARS or (name.startswith("CODER_EVAL_") and name not in _SAFE_CODER_EVAL_ENV): + env.pop(name, None) + env.update( + { + "HOME": AGENT_HOME, + "LOGNAME": "agent", + "USER": "agent", + "ZDOTDIR": AGENT_HOME, + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", + } + ) + return env + + +def _linux_security_status() -> dict[str, Any]: + """Read the kernel-enforced privilege state used by the startup handshake.""" + + try: + fields = { + name: value + for line in Path("/proc/self/status").read_text(encoding="utf-8").splitlines() + if ":" in line + for name, value in [line.split(":", 1)] + } + uids = [int(value) for value in fields["Uid"].split()] + gids = [int(value) for value in fields["Gid"].split()] + groups = [int(value) for value in fields["Groups"].split()] + capabilities = {name: int(fields[name].strip(), 16) for name in _LINUX_CAPABILITY_FIELDS} + no_new_privs = int(fields["NoNewPrivs"].strip()) + except (KeyError, OSError, ValueError): + return {"uids": None, "gids": None, "groups": None, "capabilities": None, "no_new_privs": None} + return { + "uids": uids, + "gids": gids, + "groups": groups, + "capabilities": capabilities, + "no_new_privs": no_new_privs, + } + + +def _route_to_payload(route: ApiRoute | None) -> dict[str, Any] | None: + if route is None: + return None + return {"type": type(route).__name__, "data": dataclasses.asdict(route)} + + +def _route_from_payload(payload: dict[str, Any] | None) -> ApiRoute | None: + if payload is None: + return None + route_types: dict[str, type[DirectRoute] | type[BedrockRoute] | type[LiteLLMRoute]] = { + "DirectRoute": DirectRoute, + "BedrockRoute": BedrockRoute, + "LiteLLMRoute": LiteLLMRoute, + } + route_type = route_types.get(str(payload.get("type"))) + if route_type is None: + raise ValueError(f"unknown agent worker route type: {payload.get('type')!r}") + data = payload.get("data") + if not isinstance(data, dict): + raise TypeError("agent worker route data must be an object") + return route_type(**data) + + +def _json_safe(value: Any) -> Any: + encoded = serialize_value(value) + return None if encoded is SKIP else encoded + + +def _snapshot(agent: Agent[Any] | None) -> dict[str, Any]: + if agent is None: + return {"state": AgentState.FINISHED.value, "pending_turn": None, "sdk_options": None, "environment": {}} + pending = agent.pending_turn + return { + "state": agent.get_state().value, + "pending_turn": pending.model_dump(mode="json") if pending is not None else None, + "sdk_options": _json_safe(agent.get_sdk_options()), + "environment": _json_safe(agent.get_environment_info()), + } + + +def _error_snapshot(agent: Agent[Any] | None) -> dict[str, Any]: + """Preserve failure metadata even when an optional agent getter is broken.""" + + if agent is None: + return _snapshot(None) + try: + return _snapshot(agent) + except Exception: + logger.warning("Agent metadata getters failed while reporting a worker error", exc_info=True) + pending = agent.pending_turn + state = agent.get_state() + return { + "state": state.value, + "pending_turn": pending.model_dump(mode="json") if pending is not None else None, + "sdk_options": None, + "environment": {}, + } + + +def _error_payload(exc: Exception) -> dict[str, Any]: + details: dict[str, Any] = {} + if isinstance(exc, TurnTimeoutError): + details = { + "timeout_seconds": exc.timeout_seconds, + "task_id": exc.task_id, + "iteration": exc.iteration, + } + return {"type": type(exc).__name__, "message": str(exc), "details": details} + + +def _new_stop_path() -> Path: + """Return an absent flag inside a directory writable only by the root proxy.""" + + directory = Path(tempfile.mkdtemp(prefix="coder-eval-agent-stop-", dir="/tmp")) + directory.chmod(0o711) + return directory / "stop" + + +def _remove_stop_path(path: Path) -> None: + with contextlib.suppress(OSError): + path.unlink() + with contextlib.suppress(OSError): + path.parent.rmdir() + + +class _WorkerWriter: + """Nonce-framed writer shared by RPC responses and stream callbacks.""" + + def __init__(self, nonce: str) -> None: + self._prefix = f"{_RPC_PREFIX}{nonce}:" + self._lock = threading.Lock() + + def write(self, payload: dict[str, Any]) -> None: + line = self._prefix + json.dumps(payload, separators=(",", ":"), ensure_ascii=False) + with self._lock, contextlib.suppress(BrokenPipeError, OSError): + sys.stdout.write(line + "\n") + sys.stdout.flush() + + def on_event(self, event: StreamEvent) -> None: + self.write({"kind": "event", "event": serialize_event(event)}) + + +class _WorkerServer: + def __init__(self) -> None: + self.agent: Agent[Any] | None = None + + async def handle(self, method: str, params: dict[str, Any]) -> tuple[Any, bool]: + if method == "ping": + get_euid = getattr(os, "geteuid", None) + get_egid = getattr(os, "getegid", None) + uid = get_euid() if callable(get_euid) else None + gid = get_egid() if callable(get_egid) else None + return {"uid": uid, "gid": gid, **_linux_security_status()}, False + + if method == "start": + if self.agent is not None: + with contextlib.suppress(Exception): + await self.agent.stop() + from coder_eval.agents import AgentRegistry, create_agent + from coder_eval.plugins import ensure_plugins_loaded + + ensure_plugins_loaded() + kind = str(params["agent_kind"]) + registration = AgentRegistry.get(kind) + if registration is None: + raise AgentRegistry.unregistered_kind_error(kind) + config = registration.config_class.model_validate(params["config"]) + route = _route_from_payload(params.get("route")) + kwargs = params.get("constructor_kwargs") or {} + if not isinstance(kwargs, dict): + raise TypeError("agent worker constructor_kwargs must be an object") + self.agent = create_agent(kind, config, route=route, **kwargs) + await self.agent.start( + str(params["working_directory"]), + env_path_prepend=list(params.get("env_path_prepend") or []), + plugin_tools_dir=params.get("plugin_tools_dir"), + ) + return _snapshot(self.agent), False + + if self.agent is None: + raise RuntimeError(f"agent worker received {method!r} before start") + + if method == "communicate": + stop_path_raw = params.get("stop_path") + stop_path = Path(stop_path_raw) if isinstance(stop_path_raw, str) else None + writer = params.pop("_writer") + record = await self.agent.communicate( + str(params["user_input"]), + stream_callback=writer, + timeout=params.get("timeout"), + max_turns=params.get("max_turns"), + should_stop=(lambda: stop_path.exists()) if stop_path is not None else None, + ) + return {"record": record.model_dump(mode="json"), "snapshot": _snapshot(self.agent)}, False + + if method == "discard_pending_turn": + await self.agent.discard_pending_turn() + return _snapshot(self.agent), False + + if method == "stop": + await self.agent.stop() + result = _snapshot(self.agent) + self.agent = None + return result, True + + raise ValueError(f"unknown agent worker method: {method!r}") + + async def close(self) -> None: + if self.agent is not None: + with contextlib.suppress(Exception): + await self.agent.stop() + self.agent = None + + +async def _serve_worker() -> None: + """Serve nonce-authenticated JSON requests from stdin until stop or EOF.""" + + server = _WorkerServer() + nonce: str | None = None + writer: _WorkerWriter | None = None + try: + while raw := await asyncio.to_thread(sys.stdin.buffer.readline): + request = json.loads(raw) + request_nonce = request.get("nonce") + if nonce is None: + if not isinstance(request_nonce, str) or len(request_nonce) < 32: + raise ValueError("agent worker handshake is missing a strong nonce") + nonce = request_nonce + writer = _WorkerWriter(nonce) + elif request_nonce != nonce: + continue + + assert writer is not None + request_id = request.get("id") + params = request.get("params") or {} + if not isinstance(params, dict): + writer.write( + { + "kind": "response", + "id": request_id, + "ok": False, + "error": {"type": "TypeError", "message": "request params must be an object", "details": {}}, + "snapshot": _snapshot(server.agent), + } + ) + continue + if request.get("method") == "communicate": + params["_writer"] = writer + should_exit = False + try: + result, should_exit = await server.handle(str(request.get("method")), params) + writer.write({"kind": "response", "id": request_id, "ok": True, "result": result}) + except Exception as exc: + writer.write( + { + "kind": "response", + "id": request_id, + "ok": False, + "error": _error_payload(exc), + "snapshot": _error_snapshot(server.agent), + } + ) + if should_exit: + return + finally: + await server.close() + + +def agent_worker_internal_command() -> None: + """Run the private agent-worker protocol on stdin/stdout.""" + + asyncio.run(_serve_worker()) + + +class IsolatedAgentProxy(Agent[BaseAgentConfig]): + """Root-side ``Agent`` implementation backed by one dropped-UID worker.""" + + def __init__( + self, + agent_kind: str, + config: BaseAgentConfig, + *, + route: ApiRoute | None, + constructor_kwargs: dict[str, Any] | None = None, + ) -> None: + self.agent_kind = agent_kind + self.config = config + self.route = route + self.constructor_kwargs = constructor_kwargs or {} + self._process: asyncio.subprocess.Process | None = None + self._stdout_task: asyncio.Task[None] | None = None + self._stderr_task: asyncio.Task[None] | None = None + self._nonce = secrets.token_hex(32) + self._next_request_id = 0 + self._pending: dict[int, asyncio.Future[dict[str, Any]]] = {} + self._sdk_options: dict[str, Any] | None = None + self._environment_info: dict[str, Any] = {} + self._active_collector: EventCollector | None = None + self._active_callback: StreamCallback | None = None + self._active_should_stop: Callable[[], bool] | None = None + self._active_stop_path: Path | None = None + self._active_event_seen = False + + @property + def _response_prefix(self) -> str: + return f"{_RPC_PREFIX}{self._nonce}:" + + async def _spawn(self, working_directory: str) -> None: + from coder_eval.isolation.agent_identity import require_isolation_runtime + + require_isolation_runtime() + self._process = await asyncio.create_subprocess_exec( + CONTAINER_DROP_SHIM, + sys.executable, + "-I", + "-m", + "coder_eval.isolation.agent_worker", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=working_directory, + env=build_agent_worker_environment(), + start_new_session=True, + limit=_MAX_LINE_BYTES, + ) + self._stdout_task = asyncio.create_task(self._read_stdout()) + self._stderr_task = asyncio.create_task(self._read_stderr()) + hello = await self._request("ping", {}) + from coder_eval.models import AGENT_GID, AGENT_UID + + capabilities = hello.get("capabilities") if isinstance(hello, dict) else None + privilege_drop_ok = ( + isinstance(hello, dict) + and hello.get("uid") == AGENT_UID + and hello.get("gid") == AGENT_GID + and hello.get("uids") == [AGENT_UID] * 4 + and hello.get("gids") == [AGENT_GID] * 4 + and hello.get("groups") == [] + and hello.get("no_new_privs") == 1 + and isinstance(capabilities, dict) + and set(capabilities) == set(_LINUX_CAPABILITY_FIELDS) + and all(value == 0 for value in capabilities.values()) + ) + if not privilege_drop_ok: + await self.kill() + raise RuntimeError(f"agent worker did not enter the configured unprivileged security domain: {hello!r}") + + async def _read_stdout(self) -> None: + process = self._process + assert process is not None and process.stdout is not None + try: + while raw := await process.stdout.readline(): + line = raw.decode("utf-8", errors="replace").rstrip("\r\n") + if not line.startswith(self._response_prefix): + logger.info("[agent-worker] %s", line) + continue + try: + payload = json.loads(line[len(self._response_prefix) :]) + except json.JSONDecodeError as exc: + raise ValueError("malformed agent-worker protocol line") from exc + if payload.get("kind") == "event": + event = deserialize_event(str(payload.get("event", ""))) + if event is None: + raise ValueError("invalid event in agent-worker protocol") + self._active_event_seen = True + if self._active_collector is not None: + self._active_collector.on_event(event) + safe_emit(self._active_callback, event) + self._publish_stop_flag_if_needed() + continue + if payload.get("kind") != "response": + raise ValueError("unknown agent-worker protocol message") + request_id = payload.get("id") + future = self._pending.get(request_id) if isinstance(request_id, int) else None + if future is None: + raise ValueError(f"agent-worker response has no pending request: {request_id!r}") + if not future.done(): + future.set_result(payload) + except (ValueError, asyncio.IncompleteReadError) as exc: + logger.error("Agent-worker output protocol failed: %s", exc) + self.kill_sync() + finally: + error = AgentCrashError("isolated agent worker exited before completing the request") + for future in list(self._pending.values()): + if not future.done(): + future.set_exception(error) + + async def _read_stderr(self) -> None: + process = self._process + assert process is not None and process.stderr is not None + while True: + try: + raw = await process.stderr.readline() + except ValueError: + logger.warning("Dropped an agent-worker stderr line over %d bytes", _MAX_LINE_BYTES) + continue + if not raw: + return + logger.info("[agent-worker] %s", raw.decode("utf-8", errors="replace").rstrip()) + + def _publish_stop_flag_if_needed(self) -> None: + if self._active_should_stop is None or self._active_stop_path is None: + return + try: + if self._active_should_stop(): + self._active_stop_path.touch(exist_ok=True) + except Exception: + logger.warning("Agent early-stop callback failed (ignored)", exc_info=True) + + async def _request(self, method: str, params: dict[str, Any]) -> Any: + process = self._process + if process is None or process.stdin is None or process.returncode is not None: + raise AgentCrashError("isolated agent worker is not running") + self._next_request_id += 1 + request_id = self._next_request_id + future = asyncio.get_running_loop().create_future() + self._pending[request_id] = future + request = {"nonce": self._nonce, "id": request_id, "method": method, "params": params} + try: + process.stdin.write((json.dumps(request, separators=(",", ":")) + "\n").encode()) + await process.stdin.drain() + response = await future + finally: + self._pending.pop(request_id, None) + if not response.get("ok"): + self._apply_snapshot(response.get("snapshot")) + self._raise_remote_error(response.get("error")) + return response.get("result") + + def _apply_snapshot(self, snapshot: Any) -> None: + if not isinstance(snapshot, dict): + return + state = snapshot.get("state") + if isinstance(state, str): + with contextlib.suppress(ValueError): + self._state = AgentState(state) + pending = snapshot.get("pending_turn") + self.pending_turn = TurnRecord.model_validate(pending) if isinstance(pending, dict) else None + sdk_options = snapshot.get("sdk_options") + self._sdk_options = sdk_options if isinstance(sdk_options, dict) else None + environment = snapshot.get("environment") + self._environment_info = environment if isinstance(environment, dict) else {} + + @staticmethod + def _raise_remote_error(error: Any) -> NoReturn: + if not isinstance(error, dict): + raise AgentCrashError("isolated agent worker returned an invalid error") + error_type = str(error.get("type")) + message = str(error.get("message", "isolated agent worker failed")) + details_raw = error.get("details") + details = details_raw if isinstance(details_raw, dict) else {} + if error_type == "TurnTimeoutError": + raise TurnTimeoutError( + float(details.get("timeout_seconds", 0)), + task_id=details.get("task_id"), + iteration=details.get("iteration"), + ) + exception_types: dict[str, type[Exception]] = { + "AgentConfigError": AgentConfigError, + "AgentCrashError": AgentCrashError, + "FileNotFoundError": FileNotFoundError, + "ImportError": ImportError, + "RuntimeError": RuntimeError, + "TypeError": TypeError, + "ValueError": ValueError, + } + exception_type = exception_types.get(error_type, AgentCrashError) + raise exception_type(message) + + async def start( + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + ) -> None: + if self._process is None or self._process.returncode is not None: + await self._spawn(working_directory) + result = await self._request( + "start", + { + "agent_kind": self.agent_kind, + "config": self.config.model_dump(mode="json"), + "route": _route_to_payload(self.route), + "constructor_kwargs": self.constructor_kwargs, + "working_directory": working_directory, + "env_path_prepend": env_path_prepend or [], + "plugin_tools_dir": plugin_tools_dir, + }, + ) + self._apply_snapshot(result) + + async def communicate( + self, + user_input: str, + *, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + max_turns: int | None = None, + should_stop: Callable[[], bool] | None = None, + ) -> TurnRecord: + collector = EventCollector() + stop_path: Path | None = None + if should_stop is not None: + stop_path = await asyncio.to_thread(_new_stop_path) + self._active_collector = collector + self._active_callback = stream_callback + self._active_should_stop = should_stop + self._active_stop_path = stop_path + self._active_event_seen = False + try: + result = await self._request( + "communicate", + { + "user_input": user_input, + "timeout": timeout, + "max_turns": max_turns, + "stop_path": str(stop_path) if stop_path else None, + }, + ) + if not isinstance(result, dict): + raise AgentCrashError("isolated agent worker returned an invalid turn result") + self._apply_snapshot(result.get("snapshot")) + return TurnRecord.model_validate(result.get("record")) + except BaseException: + if self.pending_turn is None and self._active_event_seen: + partial = collector.build_turn_record() + self.pending_turn = partial.model_copy( + update={"crashed": True, "crash_reason": "agent worker terminated"} + ) + raise + finally: + self._active_collector = None + self._active_callback = None + self._active_should_stop = None + self._active_stop_path = None + if stop_path is not None: + await asyncio.to_thread(_remove_stop_path, stop_path) + + async def discard_pending_turn(self) -> None: + if self._process is None or self._process.returncode is not None: + await super().discard_pending_turn() + return + result = await self._request("discard_pending_turn", {}) + self._apply_snapshot(result) + + async def stop(self) -> None: + process = self._process + if process is None: + self._mark_stopped() + return + if process.returncode is None: + try: + result = await asyncio.wait_for(self._request("stop", {}), timeout=_STOP_TIMEOUT_SECONDS) + self._apply_snapshot(result) + except Exception: + logger.warning("Agent worker did not stop cleanly; terminating its process group", exc_info=True) + await self.kill() + await self._finish_process_tasks() + self._process = None + self._mark_stopped() + + def kill_sync(self) -> None: + process = self._process + if process is None or process.returncode is not None: + return + try: + kill_process_group = getattr(os, "killpg", None) + if not callable(kill_process_group): + raise AttributeError("os.killpg is unavailable") + sigkill = getattr(signal, "SIGKILL", signal.SIGTERM) + kill_process_group(process.pid, sigkill) + except (AttributeError, OSError, ProcessLookupError): + with contextlib.suppress(ProcessLookupError): + process.kill() + + async def kill(self) -> None: + process = self._process + self.kill_sync() + if process is not None: + with contextlib.suppress(Exception): + await process.wait() + await self._finish_process_tasks() + + async def _finish_process_tasks(self) -> None: + for task in (self._stdout_task, self._stderr_task): + if task is not None: + with contextlib.suppress(Exception, asyncio.CancelledError): + await task + self._stdout_task = None + self._stderr_task = None + + def get_sdk_options(self) -> dict[str, Any] | None: + return self._sdk_options + + def get_environment_info(self) -> dict[str, Any]: + return dict(self._environment_info) + + +if __name__ == "__main__": + agent_worker_internal_command() diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 791e28c7..dd2a0cbe 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -700,18 +700,6 @@ def _validate_agent_isolation_compatibility(self) -> None: if not self._docker_config.agent_isolation: return - agent_type = str(self.rt.task.agent.type) if self.rt.task.agent and self.rt.task.agent.type else "" - supported_agents = { - AgentKind.CLAUDE_CODE.value, - AgentKind.CODEX.value, - AgentKind.ANTIGRAVITY.value, - AgentKind.NONE.value, - } - if agent_type not in supported_agents: - raise DockerRunError( - f"docker.agent_isolation has no verified UID-drop launch seam for agent type {agent_type!r}" - ) - # These criterion implementations can execute another agent or arbitrary # task-authored commands in the privileged harness. If that execution # imports candidate-controlled code, it can act as a confused deputy and @@ -1357,10 +1345,9 @@ def _build_argv( # noqa: PLR0912, PLR0915 - one ordered rendering pipeline mirr # so the in-container Orchestrator writes task.json/task.log/etc. # directly to the host filesystem via bind-mount. argv += ["-v", f"{output_dir}:{CONTAINER_OUTPUT_DIR}"] - # Mount the original task dir at the SAME host path so the - # in-container Orchestrator can set TASK_DIR (used by run_command - # criteria via `$TASK_DIR/foo.json`) to a path that resolves - # identically inside and outside the container. + # Mount the original task dir at the protected container task path so + # the in-container orchestrator can resolve staged task inputs without + # exposing the host path to the agent identity. host_task_dir: Path | None = None if self.rt.task_file: host_task_dir = self.rt.task_file.parent.resolve() diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 730f7ef2..a6507ac5 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -24,7 +24,6 @@ AGENT_UID, AGENT_USERNAME, CONTAINER_AGENT_WORK_DIR, - CONTAINER_CLAUDE_SHIM, CONTAINER_DROP_SHIM, CONTAINER_GRADER_DIR, CONTAINER_INPUT_DIR, @@ -278,7 +277,6 @@ "AGENT_UID", "AGENT_USERNAME", "CONTAINER_AGENT_WORK_DIR", - "CONTAINER_CLAUDE_SHIM", "CONTAINER_DROP_SHIM", "CONTAINER_GRADER_DIR", "CONTAINER_INPUT_DIR", diff --git a/src/coder_eval/models/container_paths.py b/src/coder_eval/models/container_paths.py index 35877a98..f22db416 100644 --- a/src/coder_eval/models/container_paths.py +++ b/src/coder_eval/models/container_paths.py @@ -31,7 +31,6 @@ AGENT_HOME = "/home/agent" CONTAINER_DROP_SHIM = "/usr/local/bin/coder_eval_drop_privilege.sh" -CONTAINER_CLAUDE_SHIM = "/usr/local/bin/coder_eval_claude_agent.sh" # Paths a task's WORKDIR must never collide with: the container root and every # framework-owned public or private mount. Consumed by SandboxConfig's diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index d23e547d..a177d51b 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -198,9 +198,8 @@ class DockerDriverConfig(BaseModel): agent_isolation: bool = Field( default=True, description=( - "Run the evaluated agent under the image's dedicated unprivileged UID/GID and expose local plugins " - "only through manifest-verified bundles. Enabled by default. Set false only for temporary migration " - "of a trusted task; false is not a secure evaluation boundary." + "Run the evaluated agent under the image's dedicated unprivileged UID/GID. Enabled by default. " + "Set false only for temporary migration of a trusted task; false is not a secure evaluation boundary." ), ) working_dir: str | None = Field( diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 36eee20d..27d69636 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -1311,6 +1311,17 @@ async def _create_agent(self) -> Agent[Any]: "x-ce-task-id": self._log_task_id, "x-ce-attempt": self._cost_attempt_nonce, } + from coder_eval.isolation.agent_identity import agent_isolation_enabled + + if agent_isolation_enabled(): + from coder_eval.isolation.agent_worker import IsolatedAgentProxy + + return IsolatedAgentProxy( + str(self.task.agent.type), + self.task.agent, + route=self.route, + constructor_kwargs=kwargs, + ) return create_agent(self.task.agent.type, self.task.agent, route=self.route, **kwargs) async def _communicate_with_retry( diff --git a/src/coder_eval/utils.py b/src/coder_eval/utils.py index 95f2ef93..1624dae6 100644 --- a/src/coder_eval/utils.py +++ b/src/coder_eval/utils.py @@ -86,36 +86,6 @@ def process_plugins( return processed -AGENT_ENV_SCRUB_VARS: tuple[str, ...] = ( - "SKILLS_REPO_PATH", - "TASK_DIR", - # The evaluator's Bedrock credential. No agent needs to INHERIT it: the Claude - # backend sets it explicitly from a resolved BedrockRoute (and blanks it on the - # LiteLLM route), Codex authenticates via CODEX_API_KEY, and Antigravity does not - # use Bedrock. Left inherited it reaches the dropped agent process, where it is - # readable through that process's own environment -- the UID barrier stops - # filesystem access to grading material but cannot hide an agent's own env. - # Scrubbing it also stops an inherited token from silently steering a DirectRoute - # run onto Bedrock (the CLI auto-selects on `process.env.AWS_BEARER_TOKEN_BEDROCK`). - "AWS_BEARER_TOKEN_BEDROCK", -) -AGENT_ENV_SCRUB_PREFIXES: tuple[str, ...] = ("CODER_EVAL_",) - - -def scrub_agent_env_overrides() -> dict[str, str]: - """Mask harness-only variables in SDK subprocess environments. - - Claude and Codex merge their explicit environment over ``os.environ``; - empty-string overrides are therefore the only concurrency-safe removal - mechanism. Antigravity has no environment seam and removes the same names - during its serialized spawn window. - """ - - return { - name: "" for name in os.environ if name in AGENT_ENV_SCRUB_VARS or name.startswith(AGENT_ENV_SCRUB_PREFIXES) - } - - SKIP = object() # Sentinel marking values that serialize_value should drop from the result. diff --git a/tests/test_agent_worker.py b/tests/test_agent_worker.py new file mode 100644 index 00000000..da27b2d4 --- /dev/null +++ b/tests/test_agent_worker.py @@ -0,0 +1,116 @@ +"""Generic registry-agent worker protocol tests.""" + +from __future__ import annotations + +import stat +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any, Literal + +import pytest + +from coder_eval.agent import Agent +from coder_eval.agents.registry import AgentRegistry +from coder_eval.isolation.agent_worker import _new_stop_path, _remove_stop_path, _WorkerServer +from coder_eval.models import AgentState, BaseAgentConfig, DirectRoute, TurnRecord +from coder_eval.streaming.callbacks import StreamCallback + + +class _PluginConfig(BaseAgentConfig): + type: Literal["worker-test-plugin"] = "worker-test-plugin" # type: ignore[assignment] + + +class _PluginAgent(Agent[_PluginConfig]): + def __init__(self, config: _PluginConfig, route: DirectRoute | None = None, **kwargs: Any) -> None: + self.config = config + self.route = route + self.kwargs = kwargs + self._state = AgentState.WORKING + self.started_as: tuple[int | None, int | None] | None = None + + async def start( + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + ) -> None: + del working_directory, env_path_prepend, plugin_tools_dir + import os + + self.started_as = ( + os.geteuid() if hasattr(os, "geteuid") else None, + os.getegid() if hasattr(os, "getegid") else None, + ) + + async def communicate( + self, + user_input: str, + *, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + max_turns: int | None = None, + should_stop: Callable[[], bool] | None = None, + ) -> TurnRecord: + del stream_callback, timeout, max_turns, should_stop + return TurnRecord(iteration=1, user_input=user_input, agent_output="plugin-ok") + + async def stop(self) -> None: + self._mark_stopped() + + +class _EventWriter: + def on_event(self, _event: object) -> None: + return None + + +@pytest.mark.skipif(sys.platform != "linux", reason="the isolated worker runs only in Linux containers") +def test_stop_flag_lives_in_a_directory_the_agent_cannot_modify() -> None: + stop_path = _new_stop_path() + try: + assert not stop_path.exists() + assert stat.S_IMODE(stop_path.parent.stat().st_mode) == 0o711 + finally: + _remove_stop_path(stop_path) + + +@pytest.fixture +def registered_worker_plugin(): + saved = dict(AgentRegistry._registry) + AgentRegistry.register("worker-test-plugin", _PluginConfig)(_PluginAgent) + try: + yield + finally: + AgentRegistry._registry.clear() + AgentRegistry._registry.update(saved) + + +async def test_worker_constructs_any_registry_agent_without_an_allowlist( + registered_worker_plugin: None, tmp_path: Path +) -> None: + server = _WorkerServer() + start_result, should_exit = await server.handle( + "start", + { + "agent_kind": "worker-test-plugin", + "config": {"type": "worker-test-plugin"}, + "route": {"type": "DirectRoute", "data": {"judge_transport": None}}, + "constructor_kwargs": {"marker": "forwarded"}, + "working_directory": str(tmp_path), + }, + ) + + assert should_exit is False + assert start_result["state"] == AgentState.WORKING.value + assert isinstance(server.agent, _PluginAgent) + assert server.agent.kwargs == {"marker": "forwarded"} + + turn_result, should_exit = await server.handle( + "communicate", + {"user_input": "hello", "_writer": _EventWriter()}, + ) + + assert should_exit is False + assert TurnRecord.model_validate(turn_result["record"]).agent_output == "plugin-ok" + await server.close() diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 2123fd13..95ee117f 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -18,7 +18,6 @@ CodexAgent, ) from coder_eval.models import AgentConfig, AgentKind, parse_agent_config -from coder_eval.utils import AGENT_ENV_SCRUB_VARS class TestCodexAgentInitialization: @@ -129,21 +128,6 @@ def test_sandbox_is_full_access(self, monkeypatch, mode, in_container, os_name): class TestCodexEnvironmentConfiguration: """Test _build_codex_env: only CODEX_API_KEY travels via env.""" - @pytest.fixture(autouse=True) - def _no_ambient_scrub_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Keep ambient evaluator credentials out of the exact-equality assertions. - - ``_build_codex_env`` starts from ``scrub_agent_env_overrides()``, so any - scrubbed name that the host happens to export (a developer's or CI's - ``AWS_BEARER_TOKEN_BEDROCK``, say) shows up as an extra masking entry and - breaks assertions that are only about what codex itself contributes. The - masking behavior has its own coverage in - ``tests/test_docker_identity_isolation.py``. - """ - - for name in AGENT_ENV_SCRUB_VARS: - monkeypatch.delenv(name, raising=False) - def test_build_codex_env_returns_none_without_key(self, monkeypatch): """No CODEX_API_KEY -> None (base URL alone is not enough).""" monkeypatch.delenv("CODEX_API_KEY", raising=False) diff --git a/tests/test_docker_build_failure.py b/tests/test_docker_build_failure.py index 31c7d024..32f28e0d 100644 --- a/tests/test_docker_build_failure.py +++ b/tests/test_docker_build_failure.py @@ -33,11 +33,6 @@ def _make_runner(run_dir: Path) -> DockerRunner: task_id="suri", description="t", initial_prompt="do", - # A concrete agent type is required now that docker.agent_isolation - # defaults to true: the isolation gate rejects a type with no verified - # UID-drop launch seam before the build runs, and this test covers - # build-failure observability rather than that gate. - agent={"type": "claude-code"}, sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig(image="x:1", dockerfile_path="/df")), success_criteria=[FileExistsCriterion(description="c", path="out.txt")], ) diff --git a/tests/test_docker_identity_isolation.py b/tests/test_docker_identity_isolation.py index 60a94a6a..0f5e58e0 100644 --- a/tests/test_docker_identity_isolation.py +++ b/tests/test_docker_identity_isolation.py @@ -9,7 +9,7 @@ import pytest -from coder_eval.agents.codex_agent import CodexAgent +from coder_eval.isolation.agent_worker import build_agent_worker_environment from coder_eval.isolation.docker_runner import DockerRunError, DockerRunner, _preflight_agent_isolation_image from coder_eval.models import ( AGENT_GID, @@ -21,9 +21,7 @@ RunCommandCriterion, SandboxConfig, TaskDefinition, - parse_agent_config, ) -from coder_eval.utils import scrub_agent_env_overrides REPO_ROOT = Path(__file__).resolve().parents[1] @@ -53,25 +51,26 @@ def test_agent_launcher_targets_only_agent_identity() -> None: assert "--regid=agent" in script -def test_agent_environment_scrubs_only_present_harness_paths(monkeypatch: pytest.MonkeyPatch) -> None: +def test_agent_worker_environment_scrubs_harness_paths(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("SKILLS_REPO_PATH", "/private/skills") monkeypatch.setenv("TASK_DIR", "/private/task") monkeypatch.setenv("CODER_EVAL_AGENT_ISOLATION", "1") + monkeypatch.setenv("CODER_EVAL_IN_CONTAINER", "1") monkeypatch.setenv("ANTHROPIC_API_KEY", "needed-by-agent") - # Keep the exact-equality assertion below valid on a host that exports it. - monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) - overrides = scrub_agent_env_overrides() + worker_env = build_agent_worker_environment() - assert overrides == { - "SKILLS_REPO_PATH": "", - "TASK_DIR": "", - "CODER_EVAL_AGENT_ISOLATION": "", - } - assert "ANTHROPIC_API_KEY" not in overrides + assert "SKILLS_REPO_PATH" not in worker_env + assert "TASK_DIR" not in worker_env + assert "CODER_EVAL_AGENT_ISOLATION" not in worker_env + assert worker_env["CODER_EVAL_IN_CONTAINER"] == "1" + assert worker_env["ANTHROPIC_API_KEY"] == "needed-by-agent" + assert worker_env["HOME"] == AGENT_HOME + assert worker_env["PYTHONNOUSERSITE"] == "1" + assert worker_env["PYTHONSAFEPATH"] == "1" -def test_agent_environment_scrubs_inherited_bedrock_credential(monkeypatch: pytest.MonkeyPatch) -> None: +def test_agent_worker_environment_scrubs_inherited_bedrock_credential(monkeypatch: pytest.MonkeyPatch) -> None: """The evaluated agent must not inherit the evaluator's Bedrock token. The UID barrier blocks filesystem access to grading material but cannot hide a @@ -83,30 +82,14 @@ def test_agent_environment_scrubs_inherited_bedrock_credential(monkeypatch: pyte monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "evaluator-only-secret") monkeypatch.setenv("AWS_REGION", "us-east-1") - overrides = scrub_agent_env_overrides() + worker_env = build_agent_worker_environment() - assert overrides["AWS_BEARER_TOKEN_BEDROCK"] == "" + assert "AWS_BEARER_TOKEN_BEDROCK" not in worker_env # The region is not a credential and stays inherited. - assert "AWS_REGION" not in overrides - - -def test_isolated_codex_profiles_never_restore_root_harness_home(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("HOME", "/root") - monkeypatch.setenv("ZDOTDIR", "/root/private-zdot") - monkeypatch.setenv("CODER_EVAL_AGENT_ISOLATION", "1") - monkeypatch.setattr(CodexAgent, "_login_shell_profiles_supported", staticmethod(lambda: True)) - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX)) - agent._env_path_prepend = ["/work/agent/cli_mocks"] - - agent._setup_login_shell_home() - try: - assert agent._login_shell_home is not None - for profile in (".bash_profile", ".profile", ".zshenv", ".zprofile", ".zshrc"): - content = (agent._login_shell_home / profile).read_text(encoding="utf-8") - assert f"export HOME={AGENT_HOME}" in content - assert "/root" not in content - finally: - agent._cleanup_login_shell_home() + assert worker_env["AWS_REGION"] == "us-east-1" + assert worker_env["USER"] == "agent" + assert worker_env["LOGNAME"] == "agent" + assert worker_env["ZDOTDIR"] == AGENT_HOME def test_agent_teardown_rescans_until_uid_has_no_processes(monkeypatch: pytest.MonkeyPatch) -> None: @@ -159,3 +142,13 @@ def test_isolation_rejects_dynamic_privileged_criterion(tmp_path: Path) -> None: with pytest.raises(RuntimeError, match="dynamic criteria"): runner._validate_agent_isolation_compatibility() + + +def test_isolation_does_not_hardcode_agent_kinds(tmp_path: Path) -> None: + task = MagicMock() + task.agent.type = "third-party-agent" + task.success_criteria = [] + task.sandbox.docker = DockerDriverConfig(agent_isolation=True) + rt = MagicMock(task=task, task_file=tmp_path / "task.yaml", run_dir=tmp_path / "run") + + DockerRunner(rt)._validate_agent_isolation_compatibility() diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 7eee4c19..3a7fb1a9 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -363,6 +363,23 @@ async def test_orchestrator_create_agent(tmp_path): assert agent.config.type == "claude-code" +@pytest.mark.asyncio +async def test_orchestrator_creates_registry_agnostic_worker_proxy_when_isolated(tmp_path, monkeypatch): + """Isolation wraps the registry selection instead of branching per agent kind.""" + + from coder_eval.isolation.agent_worker import IsolatedAgentProxy + + task, _ = load_task(Path("tasks/hello_date.yaml")) + orchestrator = Orchestrator(task=task, run_dir=tmp_path / "run", variant_id="test-variant") + orchestrator.route = DirectRoute() + monkeypatch.setenv("CODER_EVAL_AGENT_ISOLATION", "1") + + agent = await orchestrator._create_agent() + + assert isinstance(agent, IsolatedAgentProxy) + assert agent.agent_kind == "claude-code" + + # ============================================================================ # Batch Orchestration Tests (Phase 1) # ============================================================================