From 36bc5c6d2fe4f4904438fa1b6034ecbfec023a96 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Mon, 3 Aug 2026 11:55:11 +0800 Subject: [PATCH 01/25] refactor(state): initial state refactors Signed-off-by: Jiaxing Qiu --- robots/libero/toolkit.py | 61 ++++--- robots/libero/tools.py | 248 +++++++++++++--------------- rpent/tools/common.py | 88 ++++++++++ rpent/tools/state.py | 348 +++++++++++++++++++++++++++++++++++++++ rpent/tools/toolkit.py | 8 +- 5 files changed, 592 insertions(+), 161 deletions(-) create mode 100644 rpent/tools/state.py diff --git a/robots/libero/toolkit.py b/robots/libero/toolkit.py index 475c1e74..71345caa 100644 --- a/robots/libero/toolkit.py +++ b/robots/libero/toolkit.py @@ -5,13 +5,13 @@ """ from __future__ import annotations -import shutil import time from functools import partial from typing import Any from robots.libero import tools as libero_tools from rpent.dashboard.events import DashboardEventSink, ToolResultEvent +from rpent.tools.state import EnvState from rpent.tools.toolkit import ToolCancelled, Toolkit from rpent.utils.logging import get_logger, get_output_dir @@ -19,6 +19,22 @@ class LiberoToolkit(Toolkit): """Toolkit for the LIBERO environment.""" + _WIPE_STREAMS = ( + "image", + "image_cam", + "depth", + "world", + "image_wrist", + "depth_wrist", + "world_wrist", + "wrist_meta", + "image_cam_hi", + "world_hi", + "image_wrist_hi", + "world_wrist_hi", + "segments", + "action_videos", + ) # Tool schemas keyed by name (built once from the canonical ordered list # in libero_tools.TOOLS_SPEC) so each tool registers with its own spec. _SPECS = {spec["name"]: spec for spec in libero_tools.TOOLS_SPEC} @@ -30,8 +46,8 @@ def __init__( dashboard_events: DashboardEventSink, video_path: str | None = None, ) -> None: - super().__init__(dashboard_events=dashboard_events) - self._next_step: int = 0 + state = EnvState(get_output_dir()) + super().__init__(dashboard_events=dashboard_events, state=state) self._video_path: str | None = video_path self.init_primitives_clean(primitives_kwargs=primitives_kwargs) self._register_libero_tools() @@ -44,10 +60,14 @@ def _register_libero_tools(self) -> None: # Inspection tools do not advance environment state. Most are stateless # module functions; segment is bound to the primitives-owned SAM3 client. inspection_handlers = { - "view_driver_state": libero_tools.view_driver_state, - "view_camera_meta": libero_tools.view_camera_meta, - "back_project": libero_tools.back_project, - "segment": self._primitives.segment, + "view_driver_state": partial( + libero_tools.view_driver_state, state=self._state + ), + "view_camera_meta": partial( + libero_tools.view_camera_meta, state=self._state + ), + "back_project": partial(libero_tools.back_project, state=self._state), + "segment": partial(self._primitives.segment, state=self._state), } for name, handler in inspection_handlers.items(): self.add_tool(name, specs[name], handler) @@ -79,11 +99,11 @@ def _step(self, name: str, **kwargs) -> dict: else: result_dict = {"value": result} - self._next_step += 1 - step_idx = self._next_step - output_dir = get_output_dir() + step_idx = self._state.next_step_idx if self._dashboard_events.enabled: - video_dir = libero_tools.artifact_path(output_dir, "action_videos") + video_dir = libero_tools.artifact_path( + self._state.output_dir, "action_videos" + ) video_path = video_dir / f"step_{step_idx:02d}_{name}.mp4" try: self._primitives.save_frame_slice(start_frame, str(video_path), fps=20) @@ -93,11 +113,11 @@ def _step(self, name: str, **kwargs) -> dict: ) libero_tools.dump_state( self._primitives, - str(output_dir), + self._state, step_idx=step_idx, log={"command": command, "result": result_dict, "elapsed_s": elapsed}, ) - out = libero_tools.view_driver_state(step_idx) + out = libero_tools.view_driver_state(step_idx, state=self._state) out["agent_elapsed_s"] = elapsed if result_dict.get("interrupted"): out.update(result_dict) @@ -109,14 +129,9 @@ def init_primitives_clean( primitives_kwargs: dict[str, Any], ) -> None: """Wipe stale run artifacts, build the LiberoPrimitives, dump step 0.""" - out_dir = get_output_dir() - out_dir.mkdir(parents=True, exist_ok=True) - for sub in libero_tools.ARTIFACT_DIRECTORIES: - target = out_dir / sub - if target.exists(): - shutil.rmtree(target) + self._state.reset(wipe_streams=self._WIPE_STREAMS) + out_dir = self._state.output_dir for target in ( - libero_tools.artifact_path(out_dir, "states"), libero_tools.artifact_path(out_dir, "metadata", camera="agentview", resolution="low"), libero_tools.artifact_path(out_dir, "episode_video"), ): @@ -129,11 +144,11 @@ def init_primitives_clean( ) primitives.reset() primitives.start_recording() - libero_tools.dump_state(primitives, str(out_dir), step_idx=0, log=None) + libero_tools.dump_state(primitives, self._state, step_idx=0, log=None) self._dashboard_events.emit( ToolResultEvent( name="view_driver_state", - result=libero_tools.view_driver_state(0), + result=libero_tools.view_driver_state(0, state=self._state), ) ) @@ -155,4 +170,4 @@ def close(self) -> None: def write_recipe(self, recipe_tag: str) -> str: """Write the LIBERO recipe JSONL from the dumped state trace.""" - return libero_tools.write_recipe_from_states(str(get_output_dir()), recipe_tag) + return libero_tools.write_recipe_from_states(self._state, recipe_tag) diff --git a/robots/libero/tools.py b/robots/libero/tools.py index 44145227..6b95f6cf 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -11,7 +11,8 @@ import numpy as np from robots.libero.env_client import LiberoEnvClient -from rpent.utils.logging import get_logger, get_output_dir +from rpent.tools.state import EnvState, StepRecord +from rpent.utils.logging import get_logger from rpent.utils.sam3_client import Sam3Client from rpent.utils.vla_client import VLAClient @@ -662,6 +663,8 @@ def segment( step: int | None = None, point: list[int] | None = None, min_score: float = 0.2, + *, + state: EnvState, ) -> dict: """Call SAM3 on an existing image artifact without advancing the env. @@ -669,7 +672,7 @@ def segment( artifacts. Errors are structured so the agent can continue with image inspection and ``back_project``. """ - nn = _latest_step() if step is None else int(step) + nn = state.latest_step if step is None else int(step) if nn is None: return {"error": "no state entries; cannot select segment image"} @@ -681,7 +684,7 @@ def segment( return {"error": "segment needs exactly one of prompt or point"} try: image_path, world_path, artifact_pairs = _select_segment_artifacts( - nn, camera + state, nn, camera ) except ValueError as e: return {"error": str(e)} @@ -720,7 +723,7 @@ def segment( "fallback": "Use manual visual localization and back_project.", } - out_dir = get_output_dir() + out_dir = state.output_dir segment_path, overlay_candidate_path, segment_index = ( _next_segment_artifact_paths(out_dir, nn) ) @@ -785,53 +788,25 @@ def segment( return result -# --------------------------------------------------------------------------- -# State artifacts -# --------------------------------------------------------------------------- - - -def _append_state(output_dir: str, blob: dict) -> None: - """Append *blob* to ``/states.json`` atomically.""" - path = artifact_path(output_dir, "states") - tmp_path = path.parent / f"{path.name}.tmp" - if path.exists(): - with open(path) as f: - states = json.load(f) - else: - states = [] - states.append(blob) - with open(tmp_path, "w") as f: - json.dump(states, f, indent=2) - os.replace(tmp_path, path) - - -def write_recipe_from_states(output_dir: str, recipe_tag: str) -> str: +def write_recipe_from_states(state: EnvState, recipe_tag: str) -> str: """Find a command sequence that gets ``libero_terminated=True``. Export non-error LIBERO primitive commands from ``states.json`` and successful segment calls from ``segments/segment_*.json``. """ - states_path = artifact_path(output_dir, "states") - if states_path.exists(): - with open(states_path) as f: - states = json.load(f) - else: - states = [] - command_events = [] - for step_idx, entry in enumerate(states): - if not entry: - continue - command = entry.get("command") + for record in state.records(): + command = record.command if command is None: continue if command.get("action") not in PRIMITIVE_TOOL_NAMES: continue - result = entry.get("result") + result = record.result if isinstance(result, dict) and result.get("error"): continue - command_events.append(((step_idx, -1), command)) + command_events.append(((record.step_idx, -1), command)) + output_dir = state.output_dir for artifact in artifact_path(output_dir, "segments").glob("segment_*.json"): with artifact.open() as f: segment = json.load(f) @@ -853,7 +828,7 @@ def write_recipe_from_states(output_dir: str, recipe_tag: str) -> str: event_order = (source_step, int(segment["segment_index"])) command_events.append((event_order, command)) - recipe_path = os.path.join(output_dir, f"recipe_{recipe_tag}.jsonl") + recipe_path = os.path.join(state.output_dir, f"recipe_{recipe_tag}.jsonl") tmp_path = recipe_path + ".tmp" command_events.sort(key=lambda event: event[0]) with open(tmp_path, "w") as f: @@ -889,8 +864,8 @@ def _world_from_depth(depth_metric: np.ndarray, camera_meta: dict) -> np.ndarray return (camera_points @ extrinsic.T)[..., :3] -def dump_state(primitives: LiberoPrimitives, output_dir: str, step_idx: int, - log: dict | None = None) -> dict: +def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, + log: dict | None = None) -> StepRecord: """Dump state snapshot, images, and depth for step *step_idx*. Writes: @@ -911,6 +886,7 @@ def dump_state(primitives: LiberoPrimitives, output_dir: str, step_idx: int, ``command``, ``result``, and ``elapsed_s`` fields are merged into the step blob so a single entry captures everything. """ + output_dir = env_state.output_dir for directory in ARTIFACT_DIRECTORIES: (Path(output_dir) / directory).mkdir(parents=True, exist_ok=True) @@ -918,6 +894,7 @@ def dump_state(primitives: LiberoPrimitives, output_dir: str, step_idx: int, wrist_world_map = None agent_world_map_hi = None wrist_world_map_hi = None + wrist_meta = None # Reuse one raw observation snapshot for state and per-step artifacts. raw = primitives.env.raw_obs() # Expose robot proprioception and object names, but never privileged @@ -1056,6 +1033,7 @@ def dump_state(primitives: LiberoPrimitives, output_dir: str, step_idx: int, "w", ) as f: json.dump(wmeta_out, f, indent=2) + wrist_meta = wmeta except Exception as e: logger.warning("wrist depth/world dump failed: %s", e) @@ -1129,25 +1107,39 @@ def dump_state(primitives: LiberoPrimitives, output_dir: str, step_idx: int, except FileNotFoundError: pass - blob = { - "step_idx": step_idx, - "libero_terminated": primitives.env.episode_terminated, - "episode_truncated": primitives.env.episode_truncated, - "task_language": primitives.env.get_task_language(), - "state": state, - "world_map": agent_world_map, - "wrist_world_map": wrist_world_map, - "world_map_hi": agent_world_map_hi, - "wrist_world_map_hi": wrist_world_map_hi, - } - # Merge the execution log (command + result + elapsed_s) into the - # state blob so a single entry captures everything for the step. - if log is not None: - blob["command"] = log.get("command") - blob["result"] = log.get("result") - blob["elapsed_s"] = log.get("elapsed_s") - _append_state(output_dir, blob) - return blob + frame_streams = [ + stream + for stream in ("image", "image_cam", "image_wrist") + if env_state.artifact_path(step_idx, stream).exists() + ] + depth_streams = [ + stream + for stream in ("depth", "depth_wrist") + if env_state.artifact_path(step_idx, stream).exists() + ] + camera_meta = {"agentview": agentview_meta} + if wrist_meta is not None: + camera_meta["wrist"] = wrist_meta + record = StepRecord( + step_idx=step_idx, + state=state, + frames=frame_streams, + depth=depth_streams, + camera_meta=camera_meta, + command=log.get("command") if log else None, + result=log.get("result") if log else None, + elapsed_s=log.get("elapsed_s") if log else None, + extras={ + "libero_terminated": primitives.env.episode_terminated, + "episode_truncated": primitives.env.episode_truncated, + "task_language": primitives.env.get_task_language(), + "world_map": agent_world_map, + "wrist_world_map": wrist_world_map, + "world_map_hi": agent_world_map_hi, + "wrist_world_map_hi": wrist_world_map_hi, + }, + ) + return env_state.append(record) # --------------------------------------------------------------------------- @@ -1517,38 +1509,9 @@ def dump_state(primitives: LiberoPrimitives, output_dir: str, step_idx: int, ] -# --------------------------------------------------------------------------- -# State trace readers -# --------------------------------------------------------------------------- - - -def _load_states() -> list: - """Return the parsed state trace from the local output dir.""" - path = artifact_path(get_output_dir(), "states") - if not path.exists(): - return [] - with open(path) as f: - return json.load(f) - - -def _latest_step() -> int | None: - states = _load_states() - if not states: - return None - return states[-1]["step_idx"] - - -def _load_step(nn: int) -> dict: - """Look up the state blob for step ``nn`` from states.json.""" - for entry in _load_states(): - if entry.get("step_idx") == nn: - return entry - raise FileNotFoundError(f"step {nn} not present in states.json") - - -def _load_image_path(nn: int, kind: str) -> str | None: +def _load_image_path(state: EnvState, nn: int, kind: str) -> str | None: """Return the path to a dumped state image. None if not present.""" - out_dir = get_output_dir() + out_dir = state.output_dir if kind == "agent": path = artifact_path(out_dir, "policy_image", step=nn, camera="agentview", resolution="low") elif kind == "camera": @@ -1562,8 +1525,12 @@ def _load_image_path(nn: int, kind: str) -> str | None: return str(path) -def _load_camera_meta(camera: str = "agentview", nn: int | None = None) -> dict: - out_dir = get_output_dir() +def _load_camera_meta( + state: EnvState, + camera: str = "agentview", + nn: int | None = None, +) -> dict: + out_dir = state.output_dir if camera == "agentview": path = artifact_path(out_dir, "metadata", camera="agentview", resolution="low") elif camera == "wrist" and nn is not None: @@ -1571,71 +1538,74 @@ def _load_camera_meta(camera: str = "agentview", nn: int | None = None) -> dict: else: raise ValueError("camera must be 'agentview' or 'wrist' with nn") if not path.exists(): - raise FileNotFoundError(f"{path.name} not found in {out_dir}") + raise FileNotFoundError(f"{path.name} not found in {state.output_dir}") with open(path) as f: return json.load(f) -def _load_depth(camera: str, nn: int) -> np.ndarray: - out_dir = get_output_dir() - if camera not in ("agentview", "wrist"): +def _load_depth(state: EnvState, camera: str, nn: int) -> np.ndarray: + if camera == "agentview": + stream = "depth" + elif camera == "wrist": + stream = "depth_wrist" + else: raise ValueError("camera must be 'agentview' or 'wrist'") - path = artifact_path(out_dir, "depth", step=nn, camera=camera, resolution="low") - if not path.exists(): - raise FileNotFoundError(f"{path.name} not found in {out_dir}") - depth = np.load(path) + depth = state.load_depth(nn, stream) if depth.ndim == 3: depth = depth[..., 0] return depth -def view_driver_state(step: int | None = None) -> dict: - latest = _latest_step() +def view_driver_state(step: int | None = None, *, state: EnvState) -> dict: + latest = state.latest_step if latest is None: return {"error": "no state entries; env not ready"} nn = latest if step is None else int(step) try: - data = _load_step(nn) + record = state.get(nn) except Exception as e: return {"error": f"step {nn} not present in state trace: {e}"} - out: dict = {"step": nn} - out["task_language"] = data.get("task_language") - # Default to {} (not the entire blob) when "state" is missing — otherwise - # command/result/vla_desync would bleed into the "state" field, confusing - # the LLM about what robot state actually contains. - out["state"] = data.get("state", {}) - out["libero_terminated"] = data.get("libero_terminated") - out["episode_truncated"] = data.get("episode_truncated") - out["world_map"] = data.get("world_map") - out["wrist_world_map"] = data.get("wrist_world_map") - out["world_map_hi"] = data.get("world_map_hi") - out["wrist_world_map_hi"] = data.get("wrist_world_map_hi") + extras = record.extras + out: dict = {"step": nn, "state": record.state} + out["task_language"] = extras.get("task_language") + out["libero_terminated"] = extras.get("libero_terminated") + out["episode_truncated"] = extras.get("episode_truncated") + out["world_map"] = extras.get("world_map") + out["wrist_world_map"] = extras.get("wrist_world_map") + out["world_map_hi"] = extras.get("world_map_hi") + out["wrist_world_map_hi"] = extras.get("wrist_world_map_hi") out["log"] = { - "command": data.get("command"), - "result": data.get("result"), - "elapsed_s": data.get("elapsed_s"), + "command": record.command, + "result": record.result, + "elapsed_s": record.elapsed_s, } for field, kind in ( ("image_path", "agent"), ("image_cam_path", "camera"), ("image_wrist_path", "wrist"), ): - image_path = _load_image_path(nn, kind) + image_path = _load_image_path(state, nn, kind) if image_path: out[field] = image_path for field, camera in ( ("image_cam_hi_path", "agentview"), ("image_wrist_hi_path", "wrist"), ): - image_path = artifact_path(get_output_dir(), "image", step=nn, camera=camera, resolution="high") + image_path = artifact_path( + state.output_dir, + "image", + step=nn, + camera=camera, + resolution="high", + ) if image_path.exists(): out[field] = str(image_path) return out -def _select_segment_artifacts(nn: int, camera: str): - out_dir = get_output_dir() +def _select_segment_artifacts(state: EnvState, nn: int, camera: str): + out_dir = state.output_dir if camera not in ("agentview", "wrist"): raise ValueError(f"unknown segment camera: {camera}") pairs = [ @@ -1645,7 +1615,6 @@ def _select_segment_artifacts(nn: int, camera: str): ) for resolution in ("high", "low") ] - for image_path, world_path in pairs: if image_path.exists() and world_path.exists(): return image_path, world_path, pairs @@ -1737,19 +1706,24 @@ def _write_segment_overlay(image_path: Path, mask: np.ndarray, return False -def view_camera_meta(camera: str = "agentview", step: int | None = None) -> dict: +def view_camera_meta( + camera: str = "agentview", + step: int | None = None, + *, + state: EnvState, +) -> dict: """Read camera calibration metadata for localization.""" if camera not in ("agentview", "wrist"): return {"error": f"bad camera '{camera}' (use 'agentview' or 'wrist')"} nn = None if camera == "wrist": - nn = _latest_step() if step is None else int(step) + nn = state.latest_step if step is None else int(step) if nn is None: return {"error": "no wrist metadata available"} try: - meta = _load_camera_meta(camera, nn) + meta = _load_camera_meta(state, camera, nn) except Exception as e: return {"error": f"{camera} camera metadata not found: {e}"} @@ -1768,6 +1742,8 @@ def back_project( col_range: list | None = None, z_min: float | None = None, z_max: float | None = None, + *, + state: EnvState, ) -> dict: """Look up a pixel's world XYZ in the precomputed world map.""" if camera not in ("agentview", "wrist"): @@ -1784,22 +1760,21 @@ def back_project( ) } - latest = _latest_step() - nn = latest if step is None else int(step) + nn = state.latest_step if step is None else int(step) if nn is None: return {"error": "no depth/world-map files available"} try: - data = _load_step(nn) + record = state.get(nn) except Exception as e: return {"error": f"step {nn} not present in state trace: {e}"} if camera == "agentview": - hi_artifact = data.get("world_map_hi") - low_artifact = data.get("world_map") + hi_artifact = record.extras.get("world_map_hi") + low_artifact = record.extras.get("world_map") else: - hi_artifact = data.get("wrist_world_map_hi") - low_artifact = data.get("wrist_world_map") + hi_artifact = record.extras.get("wrist_world_map_hi") + low_artifact = record.extras.get("wrist_world_map") source_artifact = hi_artifact if resolution == "high" else low_artifact if not source_artifact: return { @@ -1810,8 +1785,7 @@ def back_project( } try: - world_path = artifact_path(get_output_dir(), "world", step=nn, camera=camera, resolution=resolution) - world_map = np.load(world_path) + world_map = np.load(state.output_dir / source_artifact) except Exception as e: return { "error": ( @@ -1897,7 +1871,7 @@ def back_project( depth_m = None if source_artifact == low_artifact: try: - depth = _load_depth(camera, nn) + depth = _load_depth(state, camera, nn) except Exception as e: return {"error": f"{camera} depth not found for step {nn}: {e}"} depth_m = float(depth[row, col]) diff --git a/rpent/tools/common.py b/rpent/tools/common.py index fd959566..54e02ce9 100644 --- a/rpent/tools/common.py +++ b/rpent/tools/common.py @@ -3,10 +3,16 @@ import os from pathlib import Path +from typing import Any + +import numpy as np from rpent.utils.config import get_repo_root from rpent.utils.logging import get_output_dir +_BACKPROJECT_RADIUS = 6 +_DEPTH_BAND_M = 0.02 + TOOLS_SPEC: list[dict] = [ { "name": "read_text_file", @@ -134,6 +140,88 @@ def finish(status: str, summary: str) -> dict: return {"_finish": True, "status": status, "summary": summary} +def backproject_points(K, rows, cols, depths) -> np.ndarray: + """Back-project pixel coords + depths to camera-frame XYZ, shape (N, 3).""" + K = np.asarray(K, dtype=np.float64) + rows = np.asarray(rows, dtype=np.float64) + cols = np.asarray(cols, dtype=np.float64) + depths = np.asarray(depths, dtype=np.float64) + fx, fy = K[0, 0], K[1, 1] + cx, cy = K[0, 2], K[1, 2] + return np.stack( + [(cols - cx) * depths / fx, (rows - cy) * depths / fy, depths], axis=1 + ) + + +def robust_surface_centroid( + depth: np.ndarray, + K, + T_base_cam, + row: int, + col: int, + *, + radius: int = _BACKPROJECT_RADIUS, + band: float = _DEPTH_BAND_M, +) -> dict: + """Back-project a pixel neighbourhood to a robust 3D point. + + Back-projects every valid pixel in a ``(2*radius+1)`` window, keeps those on + the dominant surface (depth within ``band`` of the window median, rejecting + background / table / dropouts), and returns the median point + diagnostics. + Returns world ``xyz`` when ``T_base_cam`` is given, otherwise camera-frame + ``xyz_cam``. + """ + row, col = int(row), int(col) + radius = max(0, int(radius)) + height, width = depth.shape[:2] + if not (0 <= row < height and 0 <= col < width): + return { + "error": ( + f"pixel ({row},{col}) out of bounds; image is {height}x{width}" + ) + } + row_start, row_end = max(0, row - radius), min(height, row + radius + 1) + col_start, col_end = max(0, col - radius), min(width, col + radius + 1) + rows, cols = np.mgrid[row_start:row_end, col_start:col_end] + depths = depth[row_start:row_end, col_start:col_end].reshape(-1).astype( + np.float64 + ) + rows = rows.reshape(-1).astype(np.float64) + cols = cols.reshape(-1).astype(np.float64) + valid = np.isfinite(depths) & (depths > 0) + if not np.any(valid): + return {"error": f"no valid depth near ({row},{col}); pick another pixel"} + depths, rows, cols = depths[valid], rows[valid], cols[valid] + median_depth = float(np.median(depths)) + surface = np.abs(depths - median_depth) <= band + depths, rows, cols = depths[surface], rows[surface], cols[surface] + if depths.size == 0: + return {"error": f"no dominant surface depth near ({row},{col})"} + + camera_points = backproject_points(K, rows, cols, depths) + camera_point = np.median(camera_points, axis=0) + out: dict[str, Any] = { + "pixel": [row, col], + "radius": radius, + "n_points": int(camera_points.shape[0]), + "depth_m": round(median_depth, 4), + "xyz_cam": [round(float(value), 4) for value in camera_point], + } + if T_base_cam is not None: + transform = np.asarray(T_base_cam, dtype=np.float64) + base_points = camera_points @ transform[:3, :3].T + transform[:3, 3] + base_point = np.median(base_points, axis=0) + out["xyz"] = [round(float(value), 4) for value in base_point] + out["xy_spread_m"] = round( + float(np.hypot(*base_points[:, :2].std(axis=0))), 4 + ) + else: + out["xy_spread_m"] = round( + float(np.hypot(*camera_points[:, :2].std(axis=0))), 4 + ) + return out + + TOOL_HANDLERS: dict = { "read_text_file": read_text_file, "write_text_file": write_text_file, diff --git a/rpent/tools/state.py b/rpent/tools/state.py new file mode 100644 index 00000000..573fde6c --- /dev/null +++ b/rpent/tools/state.py @@ -0,0 +1,348 @@ +"""EnvState: the per-run state-trace owner. + +A **step** is one motion-primitive tool call that produced a dumped state +snapshot, indexed from ``0`` where ``0`` is the post-``reset()`` baseline +(no command). Non-motion tool calls (``get_ee_pose``, ``back_project``, +``view_driver_state``, file/memory tools) are NOT steps and leave the trace +untouched. + +``EnvState`` replaces the per-env ``_append_state`` / ``_load_states`` / +``_latest_step`` / ``_load_step`` / ``_load_image`` / ``_load_depth`` free +functions (and the toolkit's ``_next_step`` counter) with one explicit owner +constructed with an ``output_dir`` (no process-global). The toolkit and the +reader tools (``view_driver_state``, ``back_project``) hold a non-owning +reference to one ``EnvState`` per run; the composition root owns its lifecycle. + +Artefact naming follows the LIBERO layout. Each saved artefact is a named +*stream* turned into a path by a fixed rule:: + + stream "image" -> images/image_NN.png + stream "image_wrist" -> images_wrist/image_wrist_NN.png + stream "depth" -> depths/depth_NN.npy + stream "depth_wrist" -> depths_wrist/depth_wrist_NN.npy + stream "world" -> world/world_NN.npy (libero) + stream "wrist_meta" -> wrist_meta/wrist_meta_NN.json + +i.e. the stream name is the file prefix; the directory pluralises the +artefact type (``image``->``images``, ``depth``->``depths``; ``world`` and +``wrist_meta`` stay singular, matching libero) and appends the camera suffix. +Franka maps scene->``image``/``depth`` and wrist->``image_wrist``/ +``depth_wrist``; lerobot maps scene->``image``/``depth`` and +arm->``image_arm``. No per-env layout class is needed: the env's thin +``dump_state`` wrapper just picks the stream names. +""" +from __future__ import annotations + +import json +import os +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +import imageio.v2 as imageio +import numpy as np + +from rpent.utils.logging import get_logger + +logger = get_logger("env_state") + +# Artefact type (the leading segment of a stream name) -> file extension. +_ARTIFACT_EXT: dict[str, str] = { + "image": ".png", + "depth": ".npy", + "world": ".npy", + "wrist_meta": ".json", +} + +# Artefact type -> directory name (plural where libero pluralises). +_PLURAL: dict[str, str] = { + "image": "images", + "depth": "depths", +} + +def _split_stream(stream: str) -> tuple[str, str]: + for artifact_type in sorted(_ARTIFACT_EXT, key=len, reverse=True): + if stream == artifact_type: + return artifact_type, "" + prefix = artifact_type + "_" + if stream.startswith(prefix): + return artifact_type, stream[len(prefix):] + art, sep, suffix = stream.partition("_") + return art, (suffix if sep else "") + + +def stream_dir(output_dir: Path, stream: str) -> Path: + """Directory for a stream's artefacts (e.g. ``images_wrist``).""" + art, suffix = _split_stream(stream) + base = _PLURAL.get(art, art) + return output_dir / (base if not suffix else f"{base}_{suffix}") + + +def stream_path(output_dir: Path, stream: str, step_idx: int, ext: str | None = None) -> Path: + """Full path for one stream artefact at ``step_idx``.""" + art, _ = _split_stream(stream) + if ext is None: + ext = _ARTIFACT_EXT.get(art, ".bin") + return stream_dir(output_dir, stream) / f"{stream}_{step_idx:02d}{ext}" + + +# --------------------------------------------------------------------------- +# StepRecord +# --------------------------------------------------------------------------- + + +@dataclass +class StepRecord: + """One dumped step: the unit the LLM reads back via ``view_driver_state``. + + ``step_idx`` is the motion-primitive index (0 = post-reset baseline, which + has ``command``/``result``/``elapsed_s`` = None). ``frames``/``depth`` are + the stream names saved for this step (e.g. ``["image", "image_wrist"]``). + ``camera_meta`` is the per-camera metadata dict at capture time (intrinsics + + extrinsics + calibration status). ``extras`` absorbs env-specific fields + (libero's world maps, ``libero_terminated``, task_language, ...). + """ + + step_idx: int + state: dict + command: dict | None = None + result: dict | None = None + elapsed_s: float | None = None + frames: list[str] = field(default_factory=list) + depth: list[str] = field(default_factory=list) + camera_meta: dict | None = None + extras: dict[str, Any] = field(default_factory=dict) + + def to_blob(self) -> dict[str, Any]: + blob: dict[str, Any] = { + "step_idx": self.step_idx, + "state": self.state, + "frames": self.frames, + "depth": self.depth, + } + if self.camera_meta is not None: + blob["camera_meta"] = self.camera_meta + if self.command is not None: + blob["command"] = self.command + if self.result is not None: + blob["result"] = self.result + if self.elapsed_s is not None: + blob["elapsed_s"] = self.elapsed_s + if self.extras: + blob["extras"] = self.extras + return blob + + @classmethod + def from_blob(cls, blob: dict) -> "StepRecord": + return cls( + step_idx=int(blob.get("step_idx", -1)), + state=blob.get("state", {}), + command=blob.get("command"), + result=blob.get("result"), + elapsed_s=blob.get("elapsed_s"), + frames=list(blob.get("frames", [])), + depth=list(blob.get("depth", [])), + camera_meta=blob.get("camera_meta"), + extras=dict(blob.get("extras") or {}), + ) + + +# --------------------------------------------------------------------------- +# EnvState +# --------------------------------------------------------------------------- + + +class EnvState: + """Owns the on-disk state trace for one run. + + Constructed with an explicit ``output_dir`` (the composition root passes + it; ``EnvState`` never reaches into a process-global). Owns the step + counter, the ``states.json`` trace (atomic append), and the image/depth + artefact layout. The toolkit and the reader tools (``view_driver_state``, + ``back_project``) hold a non-owning reference to one ``EnvState`` per run. + """ + + def __init__(self, output_dir: Path | str): + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + self._steps: list[dict] = self._read_states() + self._next_step = self._derive_next_step() + + # -- internal: states.json I/O ---------------------------------------- + + def _states_path(self) -> Path: + return self.output_dir / "states.json" + + def _read_states(self) -> list[dict]: + path = self._states_path() + if not path.exists(): + return [] + try: + with open(path) as f: + arr = json.load(f) + return [s for s in arr if isinstance(s, dict)] if isinstance(arr, list) else [] + except Exception as e: + logger.warning("could not parse %s: %s; starting fresh", path, e) + return [] + + def _write_states_atomically(self, steps: list[dict]) -> None: + path = self._states_path() + tmp = path.with_name(path.name + ".tmp") + with open(tmp, "w") as f: + json.dump(steps, f, indent=2, default=str) + os.replace(tmp, path) + + def _derive_next_step(self) -> int: + if not self._steps: + return 0 + return max(int(s["step_idx"]) for s in self._steps if "step_idx" in s) + 1 + + # -- lifecycle -------------------------------------------------------- + + def reset(self, *, wipe_streams: Iterable[str] = ()) -> None: + """Wipe stale artefact dirs + ``states.json`` for a fresh run. + + Resets the in-memory trace and counter. Step 0 (baseline) is dumped by + the caller right after via :meth:`append`. + """ + self.output_dir.mkdir(parents=True, exist_ok=True) + for stream in wipe_streams: + d = stream_dir(self.output_dir, stream) + if d.exists(): + shutil.rmtree(d) + states = self._states_path() + if states.exists(): + states.unlink() + self._steps = [] + self._next_step = 0 + + # -- counter ---------------------------------------------------------- + + @property + def next_step_idx(self) -> int: + """The step_idx the next dumped step will get (does NOT advance).""" + return self._next_step + + @property + def latest_step(self) -> int | None: + """The highest step_idx successfully written (None if trace is empty).""" + if not self._steps: + return None + return int(self._steps[-1]["step_idx"]) + + # -- writing ---------------------------------------------------------- + + def save_image(self, step_idx: int, stream: str, frame) -> str | None: + """Save one RGB frame under ``_.png``; return ``stream`` on success.""" + path = stream_path(self.output_dir, stream, step_idx) + path.parent.mkdir(parents=True, exist_ok=True) + arr = np.asarray(frame) + if arr.dtype != np.uint8: + arr = arr.astype(np.uint8) + try: + imageio.imwrite(path, arr) + return stream + except Exception as e: + logger.warning("frame dump failed for stream %s: %s", stream, e) + return None + + def save_depth(self, step_idx: int, stream: str, depth) -> str | None: + """Save one depth map under ``_.npy``; return ``stream`` on success.""" + path = stream_path(self.output_dir, stream, step_idx) + path.parent.mkdir(parents=True, exist_ok=True) + try: + np.save(path, np.asarray(depth, dtype=np.float32)) + return stream + except Exception as e: + logger.warning("depth dump failed for stream %s: %s", stream, e) + return None + + def artifact_path( + self, + step_idx: int, + stream: str, + *, + ext: str | None = None, + ) -> Path: + """Return the canonical path for one step artifact.""" + return stream_path(self.output_dir, stream, step_idx, ext=ext) + + def append(self, record: StepRecord) -> StepRecord: + """Atomically append ``record`` to ``states.json`` and advance the counter. + + The counter and ``states.json`` are kept in sync: this write is the + single source of truth for "latest step". If the write raises, the + counter is NOT advanced (so the next caller reuses the same + ``step_idx``) -- no desync between the in-memory counter and disk. + """ + blob = record.to_blob() + self._write_states_atomically(self._steps + [blob]) + self._steps.append(blob) + self._next_step = max(self._next_step, record.step_idx + 1) + return record + + # -- reading ---------------------------------------------------------- + + def get(self, step_idx: int) -> StepRecord: + """Look up the step record for ``step_idx``.""" + for blob in self._steps: + if int(blob.get("step_idx", -1)) == step_idx: + return StepRecord.from_blob(blob) + raise KeyError(f"step {step_idx} not present in states.json") + + def records(self) -> list[StepRecord]: + """Return the persisted step records in trace order.""" + return [StepRecord.from_blob(blob) for blob in self._steps] + + def load_image_bytes(self, step_idx: int, stream: str) -> bytes | None: + path = stream_path(self.output_dir, stream, step_idx) + if not path.exists(): + return None + return path.read_bytes() + + def load_depth(self, step_idx: int, stream: str) -> np.ndarray: + path = stream_path(self.output_dir, stream, step_idx) + return np.load(path) + + # -- LLM-facing view (ex-view_driver_state) --------------------------- + + def view(self, step: int | None = None, *, image_slots: dict[str, str] | None = None) -> dict: + """Build the ``view_driver_state`` dict for one step. + + ``image_slots`` maps a ToolResult image slot (``_image_bytes``, + ``_image_cam_bytes``, ``_image_wrist_bytes``) to a stream name whose + saved PNG should be embedded as bytes. The env decides which cameras + map to which slots. + """ + latest = self.latest_step + if latest is None: + return {"error": "no driver state entries; driver not ready"} + nn = latest if step is None else int(step) + try: + rec = self.get(nn) + except Exception as e: + return {"error": f"step {nn} not present in driver state trace: {e}"} + out: dict[str, Any] = { + "step": nn, + "state": rec.state, + "frames": rec.frames, + "depth": rec.depth, + "camera_meta": { + name: {k: v for k, v in meta.items() if k not in {"K", "T_base_cam"}} + for name, meta in (rec.camera_meta or {}).items() + }, + "log": { + "command": rec.command, + "result": rec.result, + "elapsed_s": rec.elapsed_s, + }, + } + if rec.extras: + out["extras"] = rec.extras + if image_slots: + for slot, stream in image_slots.items(): + b = self.load_image_bytes(nn, stream) + if b: + out[slot] = b + return out diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 31a7cbca..a747020f 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -106,10 +106,16 @@ class Toolkit: :meth:`close` to release env-side primitives / servers at the end of the run. """ - def __init__(self, *, dashboard_events: DashboardEventSink) -> None: + def __init__( + self, + *, + dashboard_events: DashboardEventSink, + state: Any = None, + ) -> None: # name -> (spec, handler) self._tools: dict[str, tuple[dict[str, Any], Callable[..., dict[str, Any]]]] = {} self._dashboard_events = dashboard_events + self._state = state self._operation_lock = threading.Lock() self._active_operation: _ToolOperation | None = None self._register_common_tools() From c3a371700bef80347b209353195e58894db023a4 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Tue, 4 Aug 2026 08:59:09 +0000 Subject: [PATCH 02/25] refactor(state): centralize environment artifact lifecycle in EnvState Signed-off-by: Jiaxing Qiu --- .../rst_source/development/add_primitive.rst | 6 +- .../rst_source/development/add_robot.rst | 24 +- docs/source-en/rst_source/quickstart.rst | 15 +- docs/source-en/rst_source/usage/libero.rst | 6 +- .../rst_source/development/add_primitive.rst | 4 +- .../rst_source/development/add_robot.rst | 25 +- docs/source-zh/rst_source/quickstart.rst | 6 +- docs/source-zh/rst_source/usage/libero.rst | 6 +- robots/libero/__init__.py | 2 - robots/libero/guides/env_calibration.md | 44 +- robots/libero/guides/pro_hybrid_guide.md | 619 ++++---------- robots/libero/guides/strict_hybrid_guide.md | 783 +++++------------- robots/libero/prompts/system.py | 117 +-- robots/libero/prompts/user.py | 10 +- robots/libero/toolkit.py | 80 +- robots/libero/tools.py | 710 ++++++---------- rpent/cli/main.py | 1 - rpent/dashboard/server.py | 15 +- rpent/dashboard/state.py | 24 +- rpent/planner/api_loop.py | 6 +- rpent/tools/state.py | 635 ++++++++------ rpent/utils/sam3_client.py | 10 +- 22 files changed, 1173 insertions(+), 1975 deletions(-) diff --git a/docs/source-en/rst_source/development/add_primitive.rst b/docs/source-en/rst_source/development/add_primitive.rst index dcefdd5b..1148f94c 100644 --- a/docs/source-en/rst_source/development/add_primitive.rst +++ b/docs/source-en/rst_source/development/add_primitive.rst @@ -167,8 +167,10 @@ Design principles for a new primitive the state dump reflecting the post-action world. Don't let the primitive return before the render finishes. - **Return small dicts.** Tool return values are fed back to the LLM - as text. Store larger content, such as images, depth data, and - ``states.json``, in the state dump instead. + as text. Save larger observations through ``EnvState.save``; ``EnvState`` + automatically records each logical base name in its owned + ``StepRecord.artifacts`` set. Expose images through ``view_driver_state`` and + geometry through environment tools rather than returning raw paths. - **Guardrails belong in env_server**, not in the toolkit. The LLM can and will call any tool with any arguments; workspace bounds and safety clamps must be enforced on the server side. diff --git a/docs/source-en/rst_source/development/add_robot.rst b/docs/source-en/rst_source/development/add_robot.rst index 2111a9f7..0a2b5497 100644 --- a/docs/source-en/rst_source/development/add_robot.rst +++ b/docs/source-en/rst_source/development/add_robot.rst @@ -222,21 +222,24 @@ Anthropic-style tool definitions (``name``, ``description``, ``input_schema``), plus any module-level functions referenced by the toolkit (e.g. ``view_driver_state``, ``back_project``, ``finish``). -**Per-step state dump** — ``dump_state(primitives, output_dir, step_idx, log)`` -serializes whatever state the agent will read back via the ``view_*`` tools -(images, depths, JSON state, camera meta) into ``output_dir``. +**Per-step state dump** — ``dump_state(driver, env_state, log)`` opens +``env_state.record_step(...)`` and receives the allocated step index. Save +large observations through ``env_state.save(..., step=step_idx)``. ``EnvState`` +owns and commits the ``StepRecord`` and adds every successfully saved base name +to its flat ``artifacts`` set automatically. Readers use the canonical artifact +filenames rather than maintaining a parallel observation index. **Toolkit class** — subclass ``rpent.tools.toolkit.Toolkit``: -- build the primitives in ``__init__`` through a custom initialization - helper (named ``init_primitives_clean`` in LIBERO; it wipes stale - ``images/`` etc., constructs the primitives, and dumps step 0), +- build the primitive driver in ``__init__`` through a custom initialization + helper (named ``init_primitives_clean`` in LIBERO; it calls + ``EnvState.reset()``, constructs the primitives, and dumps step 0), - register each tool with ``self.add_tool(name, spec, handler)`` — stateless readers (``view_driver_state``, ``finish``, …) bind directly to module-level functions; primitive tools route through ``_step(name, **kwargs)`` which calls ``getattr(self._primitives, name)(**kwargs)`` and re-renders state, -- override ``close()`` to write any remaining agent-side artifacts (e.g. the - LIBERO toolkit saves the agentview MP4 there). +- override ``close()`` to save remaining agent-side artifacts through + ``EnvState`` (for example ``state.save("episode.mp4", frames, step=None)``). ``primitives_kwargs`` (forwarded from ``__init__.py:get_toolkit``) is the dict the toolkit passes verbatim to your primitives' ``__init__`` — typically @@ -246,8 +249,9 @@ Conventions worth keeping ------------------------- - ``output_dir`` is the working directory that the runner creates for each - run. Images, depths, ``states.json``, transcripts, ``episode.mp4``, and other - artifacts go there. + run. Environment observations are owned by ``EnvState``; callers use logical + base names and never construct storage paths. Transcripts and other + run-management outputs share the same run directory. - Tool definitions use the Anthropic format (``name`` / ``description`` / ``input_schema``). Every tool registered with ``self.add_tool(...)`` is exposed to all planners. diff --git a/docs/source-en/rst_source/quickstart.rst b/docs/source-en/rst_source/quickstart.rst index b9ba30b5..8fb27a7d 100644 --- a/docs/source-en/rst_source/quickstart.rst +++ b/docs/source-en/rst_source/quickstart.rst @@ -103,14 +103,11 @@ A successful run: by the elapsed time, token usage, and path to the run record. 3. With the Dashboard enabled, also streams agent output, camera views, the action timeline, and clip replays to the Dashboard. -4. By default, artifacts are saved under - ``logs/__t_s/``. They include - ``transcript_*.json`` (run record), ``states.json`` (one record per - environment step), ``recipe_*.jsonl`` (action sequence), and - ``episode.mp4`` (episode video). - -After the run, inspect the final record in ``states.json``: -``libero_terminated`` set to ``true`` means LIBERO judged the task complete. -You can also open ``episode.mp4`` to review the run. +4. By default, artifacts are saved under ``logs/__t_s/``. They include ``transcript_*.json`` (run record), ``states.json`` (the versioned ``EnvState`` manifest), ``recipe_*.jsonl`` (action sequence), and ``episode.mp4`` (episode video). Step artifact files use flat, zero-padded step prefixes with a minimum width of two digits and are managed internally by ``EnvState``. + +Inspect the final state through the Dashboard or +``view_driver_state(step=-1)``. Its top-level ``libero_terminated`` value is the +benchmark outcome. ``states.json`` is internal ``EnvState`` storage and should +not be parsed by callers. You can also open ``episode.mp4`` to review the run. If something goes wrong, inspect the four log files described at the bottom of :doc:`installation`. diff --git a/docs/source-en/rst_source/usage/libero.rst b/docs/source-en/rst_source/usage/libero.rst index 4fd25442..b2f7682b 100644 --- a/docs/source-en/rst_source/usage/libero.rst +++ b/docs/source-en/rst_source/usage/libero.rst @@ -147,8 +147,10 @@ Physical action tools advance the environment and record new state and images. coordinates. - ``segment(prompt=... / point=..., ...)`` — use SAM3 to segment an existing image with a text or point prompt. -- ``view_driver_state(step=None)`` — read an existing state and image record. -- ``view_camera_meta(camera=..., step=None)`` — read existing camera metadata. +- ``view_driver_state(step=-1)`` — read a recorded state and its embedded + observation images. Step ``0`` is initial; ``-1`` is latest. +- ``view_camera_meta(camera=..., step=-1)`` — read camera metadata for a + recorded step. Step ``-1`` is latest. - ``finish(status, summary)`` — end the current run. These tools do not advance the environment. diff --git a/docs/source-zh/rst_source/development/add_primitive.rst b/docs/source-zh/rst_source/development/add_primitive.rst index 909a7e82..2ba6ce24 100644 --- a/docs/source-zh/rst_source/development/add_primitive.rst +++ b/docs/source-zh/rst_source/development/add_primitive.rst @@ -152,7 +152,9 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 - **每个工具执行结束后都要保存新的状态快照。** 下一轮需要读取动作执行后的 环境状态，因此原语不能在渲染完成前返回。 - **工具只返回简短的字典。** 返回值会以文本形式提供给 LLM；图像、深度数据和 - ``states.json`` 等较大的内容则通过状态快照提供。 + 其他大型观测应通过 ``EnvState.save`` 保存；``EnvState`` 会把每个逻辑基础 + 文件名自动加入其持有的 ``StepRecord.artifacts`` 集合。图像通过 + ``view_driver_state`` 提供，几何数据通过环境工具访问，不返回原始路径。 - **安全限制由 ``env_server`` 强制执行。** LLM 可能使用任意参数调用工具， 因此工作空间边界和安全限制不能只依赖 toolkit。 diff --git a/docs/source-zh/rst_source/development/add_robot.rst b/docs/source-zh/rst_source/development/add_robot.rst index 995a13e7..0f76750c 100644 --- a/docs/source-zh/rst_source/development/add_robot.rst +++ b/docs/source-zh/rst_source/development/add_robot.rst @@ -209,21 +209,23 @@ Anthropic API 的工具定义格式，包含 ``name``、``description`` 和 ``input_schema``），以及 toolkit 引用的模块级函数，例如 ``view_driver_state``、``back_project`` 和 ``finish``。 -**每步状态 dump** —— ``dump_state(primitives, output_dir, step_idx, log)`` 把 agent -之后会通过 ``view_*`` 工具读回的所有状态 (图像、深度、JSON 状态、camera meta) -序列化到 ``output_dir``。 +**每步状态 dump** —— ``dump_state(driver, env_state, log)`` 通过 +``env_state.record_step(...)`` 创建由 ``EnvState`` 持有的步骤，并取得分配的 +step index。大型观测通过 ``env_state.save(..., step=step_idx)`` 保存。每次保存 +成功后，``EnvState`` 会自动把基础文件名加入该 ``StepRecord`` 的扁平 +``artifacts`` 集合并最终提交记录；读取方直接使用规范化的工件文件名。 **Toolkit 类** 继承 ``rpent.tools.toolkit.Toolkit``： -- 在 ``__init__`` 中通过自定义的初始化辅助方法构建 primitives（LIBERO - 中的方法名为 ``init_primitives_clean``；它会清理过期的 ``images/`` 等目录、 - 构造原语并 dump 第 0 步）, +- 在 ``__init__`` 中通过自定义的初始化辅助方法构建 primitive driver（LIBERO + 中的方法名为 ``init_primitives_clean``；它会调用 ``EnvState.reset()``、构造 + 原语并 dump 第 0 步）, - 用 ``self.add_tool(name, spec, handler)`` 注册每个工具。无状态的读取工具 （如 ``view_driver_state``、``finish``）直接绑定模块级函数；原语工具通过 ``_step(name, **kwargs)`` 调用。``_step`` 使用 - ``getattr(self._primitives, name)(**kwargs)`` 调用 primitives 方法并重新渲染状态； -- 重写 ``close()``，将 agent 侧生成的文件写入磁盘（例如 LIBERO toolkit - 在这里保存 agentview MP4）。 + ``getattr(self._primitives, name)(**kwargs)`` 调用 driver 方法并重新渲染状态； +- 重写 ``close()``，通过 ``EnvState`` 保存 agent 侧剩余工件（例如 + ``state.save("episode.mp4", frames, step=None)``）。 ``primitives_kwargs`` 由 ``__init__.py:get_toolkit`` 转发给 toolkit，再原样传入 primitives 的 ``__init__``。其中通常包含 @@ -232,8 +234,9 @@ primitives 的 ``__init__``。其中通常包含 建议遵循的约定 -------------- -- ``output_dir`` 是 runner 为单次运行创建的临时目录。图像、深度数据、 - ``states.json``、transcript 和 ``episode.mp4`` 等工件都写入该目录。 +- ``output_dir`` 是 runner 为单次运行创建的工作目录。环境观测由 + ``EnvState`` 管理；调用方只使用逻辑基础文件名，不自行拼接存储路径。 + transcript 等运行管理输出与环境工件共享该目录。 - 工具定义使用 Anthropic API 格式（``name`` / ``description`` / ``input_schema``）。 每个用 ``self.add_tool(...)`` 注册的工具都会暴露给所有 planner。 diff --git a/docs/source-zh/rst_source/quickstart.rst b/docs/source-zh/rst_source/quickstart.rst index b9cf71c7..9d870b55 100644 --- a/docs/source-zh/rst_source/quickstart.rst +++ b/docs/source-zh/rst_source/quickstart.rst @@ -95,7 +95,9 @@ LIBERO-PRO 仿真资源。下面以 LIBERO-PRO 和 ``claude_code`` planner 1. 终端会先显示 ``env_server``、``vla_server`` 和 ``sam3_server`` 的启动信息。 2. 智能体的逐轮输出和工具调用会显示在终端中；运行结束时还会显示耗时、token 用量和运行记录的路径。 3. 启用 Dashboard 后，智能体的输出、相机视图、动作时间线和片段回放也会实时显示在 Dashboard 中。 -4. 默认输出目录为 ``logs/__t_s/``，其中包含 ``transcript_*.json``\ （运行记录）、``states.json``\ （每个环境步的记录）、``recipe_*.jsonl``\ （动作序列）和 ``episode.mp4``\ （回合录像）。 +4. 默认输出目录为 ``logs/__t_s/``，其中包含 ``transcript_*.json``\ （运行记录）、``states.json``\ （带版本号的 ``EnvState`` 清单）、``recipe_*.jsonl``\ （动作序列）和 ``episode.mp4``\ （回合录像）。每步工件文件采用至少两位、零填充的步骤前缀扁平命名，并由 ``EnvState`` 在内部管理。 -运行结束后，查看 ``states.json`` 的最后一条记录：``libero_terminated`` 为 ``true`` 表示 LIBERO 已判定任务完成；也可以打开 ``episode.mp4`` 复核运行过程。 +通过 Dashboard 或 ``view_driver_state(step=-1)`` 查看最终状态；其顶层 +``libero_terminated`` 即为基准任务结果。``states.json`` 是 ``EnvState`` 的内部 +存储，调用方不应直接解析。也可以打开 ``episode.mp4`` 复核运行过程。 出问题时，参考 :doc:`installation` 页底部提到的四份日志文件。 diff --git a/docs/source-zh/rst_source/usage/libero.rst b/docs/source-zh/rst_source/usage/libero.rst index 2675458b..b66e06ac 100644 --- a/docs/source-zh/rst_source/usage/libero.rst +++ b/docs/source-zh/rst_source/usage/libero.rst @@ -141,8 +141,10 @@ LIBERO 工具分为物理动作工具和只读工具。 - ``back_project(row, col, ...)`` —— 将图像像素反投影到世界坐标。 - ``segment(prompt=... / point=..., ...)`` —— 通过 SAM3 对已有图像进行文本或 点提示分割。 -- ``view_driver_state(step=None)`` —— 读取已有的状态和图像记录。 -- ``view_camera_meta(camera=..., step=None)`` —— 读取已有的相机元数据。 +- ``view_driver_state(step=-1)`` —— 读取已记录的状态和内嵌观测图像；第 0 步为 + 初始状态，``-1`` 表示最新状态。 +- ``view_camera_meta(camera=..., step=-1)`` —— 读取指定步骤的相机元数据； + ``-1`` 表示最新状态。 - ``finish(status, summary)`` —— 结束当前运行。 这些工具不会推进环境。 diff --git a/robots/libero/__init__.py b/robots/libero/__init__.py index c5f31923..48b1bcde 100644 --- a/robots/libero/__init__.py +++ b/robots/libero/__init__.py @@ -45,7 +45,6 @@ def get_toolkit( *, primitives_kwargs: dict[str, Any], dashboard_events: DashboardEventSink, - video_path: str | None = None, ): """Return the LIBERO toolkit (common tools + LIBERO primitives).""" from robots.libero.toolkit import LiberoToolkit @@ -53,7 +52,6 @@ def get_toolkit( return LiberoToolkit( primitives_kwargs=primitives_kwargs, dashboard_events=dashboard_events, - video_path=video_path, ) diff --git a/robots/libero/guides/env_calibration.md b/robots/libero/guides/env_calibration.md index 1dbab00d..886fd711 100644 --- a/robots/libero/guides/env_calibration.md +++ b/robots/libero/guides/env_calibration.md @@ -2,12 +2,15 @@ ## Current LIBERO MCP Runtime Contract -Use this file as a calibration reference only. For current MCP-based runs, use -structured MCP tools, do not issue file-based protocol commands, and do not manually manage -`env_server.py`. Do not read BDDL files or hidden task definition files to infer -coordinates. Do not expect object world coordinates in `states.json`; localize -objects through images_cam + depth/back_project, segment, and wrist/high-res -artifacts when available. +Use this file as a calibration reference for structured-tool runs. The runner +owns the environment server and the `EnvState` lifecycle. Do not issue +file-based driver commands, inspect observation storage directly, or read BDDL +files for coordinates. + +Start with `view_driver_state({"step": 0})`. It returns the initial robot state, +top-level task language, logical observation references, and embedded camera +images. Use `back_project` or `segment` for geometry and `view_camera_meta` for +calibration. Step `-1` selects the latest record. Measured 2026-05-20 on `libero_10_with_mug` t0 (LIVING_ROOM frame) and t8 (KITCHEN frame). All probes use `move_to` with `gripper=-1` and tight @@ -17,9 +20,9 @@ Measured 2026-05-20 on `libero_10_with_mug` t0 (LIVING_ROOM frame) and t8 Each task scene uses one of the table fixtures below, which sets the entire world-frame z origin. The OSC workspace and all pick/place altitudes shift -accordingly. **Check `states.json[0].state.robot0_eef_pos[2]` in the initial -state and branch on it.** Do not read BDDL files for this; use runtime state and -visual evidence. +accordingly. **Check `state.robot0_eef_pos[2]` in the result of +`view_driver_state({"step": 0})` and branch on it.** Do not read BDDL files for +this; use runtime state and visual evidence. | Fixture | eef home z | Table top z | Used by tasks | |---|---|---|---| @@ -119,7 +122,7 @@ limit 1.15). My libero_10 t0 used z=0.95 for travel — safe and consistent. ## Practical rules going forward -1. **Always read `states.json[0].state.robot0_eef_pos[2]` before computing any z target.** +1. **Always inspect the initial returned `state.robot0_eef_pos[2]` before computing any z target.** ≈ 0.68 → LIVING_ROOM; ≈ 1.17 → KITCHEN; ≈ 0.26 → OBJECT. Use the matching frame table above. 2. **Never command an eef z below the per-frame floor.** Going to z=0.42 @@ -143,24 +146,25 @@ limit 1.15). My libero_10 t0 used z=0.95 for travel — safe and consistent. `move_to` + `set_gripper` (last resort; unreliable for objects <6 cm — see `resources/libero/memory/feedback_scripted_pick_limits.md`). -## Calibration log files +## Calibration Records + +Each calibration motion returns its state, command result, elapsed time, and +embedded observation. Use that tool result immediately. To revisit a recorded +step, call `view_driver_state({"step": N})`; use `-1` for the latest step. -Raw probe logs are preserved in: -- `{output_dir}/states.json` (one step entry per command — the per-command - audit; each entry has `command`, `result`, `state`, `elapsed_s`). -- Only kept for the most recent agent session; reproduce by re-running - the calibration with the snippet in the next section. +The internal `states.json` file is a versioned manifest owned by `EnvState`, not +a list for manual indexing. Calibration analysis should use structured tool +results rather than parsing storage files. ## Reproducer -Legacy calibration notes below describe the old file-protocol flow. In the -current MCP runtime, use the runner-managed environment and call structured MCP -tools instead. +Use the runner-managed environment and call structured tools: ```bash # For each z in 0.65 .. 0.42, call the MCP tool: move_to({"xyz": [-0.20, 0.10, 0.65], "gripper": -1, "tol": 0.008, "step_clip": 0.010, "max_steps": 80}) -# Then read states.json entry NN for final_eef_pos & final_dist_m. +# Inspect the returned result for final_eef_pos and final_dist_m. +# Call view_driver_state({"step": -1}) only when the latest state is needed again. ``` diff --git a/robots/libero/guides/pro_hybrid_guide.md b/robots/libero/guides/pro_hybrid_guide.md index d7999405..7da40264 100644 --- a/robots/libero/guides/pro_hybrid_guide.md +++ b/robots/libero/guides/pro_hybrid_guide.md @@ -1,502 +1,197 @@ -# LIBERO-Pro Hybrid (Pi0.5 + LLM-in-the-loop) — Perception-Isolated Guide - -You are picking up the **LIBERO-Pro** evaluation track in **perception-isolated** -mode — the only mode in this repository (the legacy oracle-state mode is not -included here). - -> **Pi0.5 only does the grasp (`pi0_pick`). The LLM (you) handles every motion -> (`move_to`), every release, sequencing, retries — and you do not get GT object -> coordinates. You localize objects yourself from the depth + camera calibration -> the runtime dumps each step.** - -This document layers on the base playbook. Read it first: - -- [`strict_hybrid_guide.md`](./strict_hybrid_guide.md) — the perception protocol: - back-projection localization, the perception artifacts - (`images_cam/image_cam_NN.png` / `depths/depth_NN.npy` / `camera_meta.json`), - the primitive vocabulary, the Rules (0/1/2/4/5), and the `strict_perception` - audit format. **This is the source of truth for *how you localize*.** -- **This file** — LIBERO-Pro–specific setup, the four perturbation axes, the Pi0 - fullshot baseline, the frame split, and how perception isolation changes the - P2-swap story. - -> Your task comes from `states.json[0].task_language`; the BDDL is FORBIDDEN. -> Read the authoritative instruction (the BDDL `:language` tag, coord-free) that -> the runtime injects and obey it verbatim. Do **not** scrape the BDDL or import -> the benchmark: that is error-prone (wrong task-map index → wrong task) and a -> perception-isolation breach (the `:init` block holds the GT coordinates this -> mode withholds). For swap fixtures, localize them **visually** (§3.5), never -> from `:init`. You never hand-roll projection math — `back_project` applies -> `K⁻¹` + the cam→world extrinsic and returns the surface `world_xyz` under a -> pixel; just use it. - -Whenever this guide says "see the perception protocol" or "the localization -snippet", it means `strict_hybrid_guide.md`. **Every rule there applies here**; -this guide only *adds* PRO constraints and tooling. Runner contract: call the -structured MCP tools (the runner owns the env server — do not start/stop it, and -issue no file-based commands); it is a single-episode run — no `reset`/`exit`, -recover in place or write an honest failure audit and `finish`. - -## 0. What's different from oracle PRO mode (read this first) - -| | oracle mode (not in this repo) | **perception (this guide)** | -|---|---|---| -| how launched | oracle-state run | `rpent/cli/main.py --libero-type pro` (or `LIBERO_TYPE=pro`); perception artifacts always dumped, coords withheld | -| `states.json` objects | full `objects:{name:[x,y,z]}` | **`object_names:[…]` only — NO coords** | -| how you learn the task | env prompt / scrape BDDL | **`states.json[0].task_language`** (authoritative `:language`, coord-free) — never read the BDDL | -| extra obs artifacts | agentview RGB only | **+ `images_cam/`, `depths/`, `world/` (agentview); `images_wrist/`, `depths_wrist/`, `world_wrist/` (wrist); hi-res pairs; `camera_meta.json`** — all via `back_project` | -| cameras | agentview only | **agentview (fixed, ~1m → ±8-13cm) + eye-in-hand wrist (moves with gripper, ±1-2cm when <20cm to target)** — see §3.6 | -| how P2 swap is solved | read swapped coords from `states.json[0]` | **localize the swapped objects by `back_project`** | -| how a swapped *fixture* site is found | read the swap BDDL `:init` block | **localize the fixture visually** (see §3.5) | -| run budget | short | **larger** — perception localization + manipulation is slower (raise `--max-turns` / `--planner-timeout-s`) | -| audit `regime` | `strict` | `strict_perception` | -| which image you pick pixels in | agentview RGB | `images_cam/image_cam_NN.png` (or hi-res) for **pixel-picking**; `images/image_NN.png` only for a sanity glance | - -**The single most important conceptual shift.** In oracle PRO mode the headline -is "the hybrid beats Pi0 on P2 because it reads the *swapped* coordinates straight -out of `states.json[0]`, while Pi0 is prompt-/memory-blind." In **perception** -mode there are no coordinates to read — so the hybrid's P2 win now comes from -**seeing where the object is and back-projecting it**. This is a *stronger* claim -(no oracle state at all), and it is the whole point of running PRO in perception -mode: P1 (Task) is still won by reading the *language*; P2 (Position) is now won by -*perception*, not by an oracle. - -## 1. Why LIBERO-Pro - -LIBERO-Pro -([paper](https://arxiv.org/pdf/2510.03827), [repo](https://github.com/Zxy-MLlab/LIBERO-PRO)) -perturbs each base task along five axes; all end-to-end VLAs (OpenVLA / Pi0 / -Pi0.5 / UniVLA) collapse on the two strongest: - -| Axis | Suffix | Paper column | What changes | Headline result | -|---|---|---|---|---| -| **Task** | `_task` | **P1** | Instruction + goal predicate inverted | All VLAs ≈ 0.0 | -| **Position** | `_swap` | **P2** | Object **and fixture** initial positions swapped | All VLAs 0.0–0.4 | -| Semantic | `_lan` | — | Instruction paraphrased; goal unchanged | VLAs handle (memorize visual) | -| Object | `_object` | — | Object appearance / colour / scale | VLA visual policy stressed | -| Environment | `_environment` | — | Table / scene swapped | Visual policy stressed | - -The agentic hybrid wins on **P1 and P2** by routing the language channel and the -*perceived* spatial-state channel through the LLM. Object and Environment -perturbations enter through the Pi0 vision channel and the hybrid inherits the -VLA's weakness there — declare that upfront, don't oversell. **In perception -mode, the Object/Environment axes also stress your own localization** (a -recolored or rescaled object is harder to pixel-pick), so lean on the -multi-pixel-median tip from the perception protocol. - -## 2. Setup (do these once on a fresh checkout) - -Run the idempotent installer from the repo root: - -```bash -bash scripts/install_libero_pro_plus.sh -``` +# LIBERO-Pro Hybrid Perception Guide + +This guide extends [strict_hybrid_guide.md](./strict_hybrid_guide.md) for the +LIBERO-Pro evaluation tracks. Read the strict guide first; its runtime contract, +localization discipline, and single-attempt rules apply unchanged. -It does all four steps below: the liberopro editable install, applies the -benchmark-registration patch, syncs the authoritative HF dataset snapshot, and -verifies with the `get_benchmark(...).get_task(0).language` check. The perception -observables (depth + both cameras + hi-res) are unconditional once the runner -launches with `--libero-type pro`; there is nothing perception-specific to -install beyond the PRO setup itself. +LIBERO-Pro perturbs object placement, fixture placement, spatial relations, and +task language. The agent must solve the observed scene rather than replaying a +seed-specific command sequence. -### 2.1. LIBERO-PRO repo +## Start Of Run -Cloned at `${LIBERO_PRO_PATH:-/path/to/LIBERO-PRO}/` from -`https://github.com/RLinf/LIBERO-PRO.git` and installed editable into the openpi -venv: +Call: -```bash -python -m pip show liberopro -# Name: liberopro Version: 0.1.0 Location: ${LIBERO_PRO_PATH:-/path/to/LIBERO-PRO} +```json +{"step": 0} ``` -### 2.2. Apply the benchmark-registration patch +through `view_driver_state`. From the returned tool result: -The upstream `__init__.py` does **not** expose the 16 perturbation suites through -`get_benchmark()`. Our patch -[`scripts/liberopro_register_perturbations.patch`](../../../../scripts/liberopro_register_perturbations.patch) -adds them and overrides `Task.language` to read each BDDL's actual `:language` -tag (so the perturbed instruction reaches Pi0 / hybrid). +- read top-level `task_language` verbatim; +- inspect `state.robot0_eef_pos` to identify the scene frame; +- inspect object names only as a scene inventory, never as coordinates; +- inspect the embedded agentview image for semantic identity and relations; +- inspect the wrist image only when it contains useful close geometry. -```bash -cd ${LIBERO_PRO_PATH:-/path/to/LIBERO-PRO} -git apply /scripts/liberopro_register_perturbations.patch -``` +Step `-1` means the latest record. Do not inspect or index `states.json` +directly; it is an internal manifest. -If already applied (likely), `git status -s` shows clean. If you reinstall -liberopro, re-apply. +## Perturbation Axes -### 2.3. Huggingface dataset (authoritative) +### Object perturbation -The LIBERO-PRO git repo ships **incomplete / broken** init files for several -perturbation suites (e.g. `libero_spatial_swap` has 0 BDDLs; some -`libero_spatial_task` `.pruned_init` files are 0 bytes). Treat the git repo as -unreliable for perturbation data. The full, correct set lives on Huggingface -([`zhouxueyang/LIBERO-Pro`](https://huggingface.co/datasets/zhouxueyang/LIBERO-Pro)), -persisted locally at: +Objects move relative to their seed positions. Re-localize every target and +destination from the current images. Seed recipes remain useful for primitive +ordering, prompt wording, safe heights, and known failure modes, but never for +coordinates. -``` -${LIBEROPRO_DATASET_PATH:-/path/to/liberopro_hf}/ -├── bddl_files/ 16 perturbation suites, 10 BDDLs each -└── init_files/ 16 perturbation suites, 10 init files each -``` +### Spatial perturbation -Covers `{libero_spatial, libero_object, libero_goal, libero_10} × {swap, task, -lan, object}`. The installer syncs this into the liberopro install (overwriting -the broken upstream files); if the persistent copy is gone, re-download with: +Relations such as left/right, on-top-of, or next-to may select a different +instance than in the reference scene. Identify the candidate satisfying the +current visual relation, then localize it with `back_project` or `segment`. -```bash -python -c " -from huggingface_hub import snapshot_download -snapshot_download(repo_id='zhouxueyang/LIBERO-Pro', repo_type='dataset', - local_dir='${LIBEROPRO_DATASET_PATH:-/path/to/liberopro_hf}', - allow_patterns=['bddl_files/**','init_files/**'])" -``` +### Goal perturbation -### 2.4. Verify +The destination or requested interaction changes. Re-read `task_language` and +classify the destination semantically. Do not infer the goal from the suite, +task index, object list, or a sibling result file. -```bash -LIBERO_TYPE=pro python -c " -import liberopro.liberopro.benchmark as bench -for n in ['libero_spatial_task','libero_spatial_swap','libero_spatial_lan']: - b = bench.get_benchmark(n)(); t = b.get_task(0) - print(f'{n} t0: {t.language!r} trials={len(b.get_task_init_states(0))}')" -``` +### Task-language perturbation -Expected: -``` -libero_spatial_task t0: 'Pick the akita black bowl not between the plate and the ramekin and place it on the plate' trials=50 -libero_spatial_swap t0: 'Pick the akita black bowl between the plate and the ramekin and place it on the plate' trials=50 -libero_spatial_lan t0: 'lift the black bowl between the plate and ramekin and set it on the plate' trials=50 -``` +The authoritative instruction is the current top-level `task_language`. Do not +read BDDL files. They combine language with hidden initialization data and +therefore violate perception isolation. -## 3. PRO-specific environment gotchas +## Scene Frame Selection -Everything in `strict_hybrid_guide.md` applies. The following are **additional** -PRO constraints (most live in detail at [`env_calibration.md`](./env_calibration.md)). +The initial end-effector height distinguishes the principal scene frames: -### 3.1. Three scene frames, picked per-task +| Initial EEF z | Frame | Typical fixtures | +|---|---|---| +| approximately 0.26 m | object/low-table | grocery and basket tasks | +| approximately 0.68 m | living-room table | plates, baskets, pudding | +| approximately 1.17 m | kitchen table | stove, cabinet, drawer, microwave | -PRO scenes use one of three table fixtures; the eef home z differs by up to -~0.9 m. +Use `view_driver_state({"step": 0})["state"]["robot0_eef_pos"][2]` as the +measurement. Then use the matching safe-height guidance from +[env_calibration.md](./env_calibration.md). -| Fixture | eef home z | Table top z | xy reachable | Where | -|---|---|---|---|---| -| `living_room_table` | ≈ 0.68 | ≈ 0.43 | `(x∈±0.30, y∈±0.30)` | basket / plate / pudding | -| `kitchen_table` | ≈ 1.17 | ≈ 0.90 | `(x∈±0.30, y∈±0.30)` | stove / cabinet / drawer / microwave | -| `object` (low table) | ≈ 0.26 | ≈ 0.0 | `(x∈±0.30, y∈±0.30)` | `libero_object` grocery-into-basket | +## Mandatory Perception Table -**Mandatory check at session start: read `states.json[0].state.robot0_eef_pos[2]`** -(via `view_driver_state({"step": 0})`). ≈ 0.68 → LIVING_ROOM; ≈ 1.17 → KITCHEN; -≈ 0.26 → OBJECT. Pick `pre_pos_z` / `carry_z` / `release_z` accordingly (per-item -OBJECT-frame altitudes are in -`resources/libero/memory/project_libero_object_pro_done.md`). Sending a -wrong-frame z (e.g. KITCHEN coordinates while the env is in LIVING_ROOM frame) -crashes the env worker (EOFError, silent state loss). +Before manipulating, create one row for every task-relevant entity: -> **Perception note.** This proprioceptive z is *not* an object coordinate — it's -> the robot's own pose, which `states.json` still gives you in perception mode. -> Reading it to pick the frame is fine. It also doubles as a free depth sanity -> check: your back-projected table z should sit ≈0.25 m below eef home z (≈0.43 in -> LIVING_ROOM, ≈0.90 in KITCHEN). If a back-projection lands far from that, you -> picked a wrong pixel. +| field | meaning | +|---|---| +| role | target, destination, support, fixture, or relation landmark | +| semantic evidence | visual properties and relation establishing identity | +| agentview pixels | several interior pixels or a verified segment mask | +| agentview xyz | robust projection from the global camera | +| wrist refinement | accepted, rejected, basket-confirmed, or unnecessary | +| final xyz | coordinate used for planning | +| uncertainty | duplicates, occlusion, rim bias, label ambiguity, etc. | -For `libero_spatial_*` you are always in **KITCHEN frame**. Standard heights: +Do not begin manipulation while a required row lacks a defensible identity or +coordinate. -``` -pre_pos_z = 1.05 # ~7 cm above objects at z≈0.97 -carry_z = 1.10 # safe traversal, well under upper limit 1.15 -release_z = 1.01 # ~7 cm above plate top at z≈0.90 -``` +## Swapped Objects And Fixtures -These are *altitude* knobs (robot-frame), not object positions — you still derive -the object's **xy** by `back_project` every time. - -### 3.2. xy single-step ±0.30 cap - -OSC flips IK branches if you command `|x|>0.30` or `|y|>0.30` in a single -`move_to` (the eef lands in the wrong half-space and corrupts the run). **Never -command beyond ±0.30 in a single move** — split into carry-z waypoints. (Detail -in `env_calibration.md`.) - -### 3.3. Slow long-distance carry for swap variants - -P2 swap can move a large object 15 cm across the table. `step_clip=0.025` lets it -slip in the gripper and the object ends up centimetres off target. Mitigation -(proven on `_swap` t0, carries over directly): - -- `carry_z = 1.15` (higher than usual 1.10) -- `step_clip = 0.020` (slower) -- **Re-localize mid-travel** instead of trusting a cached `object_xyz - eef_xyz` - offset: `Read images_cam/image_cam_NN.png` (or the hi-res `images_cam_hi/`) - mid-carry and call `back_project` on the carried object's pixel; if the - perceived offset drifted >5 mm from post-pick, re-pre-position and re-`pi0_pick`. - -### 3.4. Task language is from BDDL, not filename - -After the patch, `get_task(i).language` returns the perturbed `:language` tag, and -the env passes it to Pi0 as the prompt (surfaced to you as -`states.json[0].task_language`). For `_task` and `_lan` this is the perturbed -instruction. **Don't override it** — falsifying the VLA's prompt-blindness is the -point. You read the same language to decide *which* object to localize and place. - -### 3.5. ⚠ Swap moves FIXTURES too — and you must localize them visually - -This is the biggest perception-mode trap, and it is **specific to `_swap`**. The -P2 perturbation does not only swap loose objects; for `libero_goal_swap` it swaps -entire **fixtures** (stove ↔ cabinet ↔ wine_rack), so a goal predicate like -`On(bowl, flat_stove_1_cook_region)` now points at wherever the *stove* was -relocated to, and there are no coordinates in `states.json` to read. See -`resources/libero/memory/feedback_swap_perturbs_fixtures.md`. - -In **oracle** mode the documented fix is to read the swap BDDL `:init` block and -recompute the fixture site's world coordinates. **That is forbidden here** — the -`:init` block is ground-truth geometry. In **perception** mode you instead: - -1. Identify the target fixture by name from the (perturbed) task language and - `state.object_names`. -2. **Localize the fixture's predicate site visually** in - `images_cam_hi/image_cam_hi_NN.png`: pick pixels on the stove's cook region / - cabinet top surface / rack top shelf, then `back_project` (sample 3–5 pixels, - median the xy — fixtures are large and the surface you want is the *placement* - surface, not the nearest edge). -3. Carry target xy = perceived site xy; descend to the perceived site z + a small - clearance, `release`, then **retreat with gripper open** — the predicate often - fires *during* the settle, not at the release step itself. - -So the swap-fixture problem becomes "find the fixture in the image" rather than -"read where the BDDL put it." Loose-object swaps (e.g. `libero_spatial_swap`, -where only bowls/plates move) are simpler: just localize each object by -`back_project` as usual; their new positions fall straight out of the depth. - -### 3.6. Two cameras — agentview = IDENTITY, wrist = GEOMETRY - -Do **not** restate the two-camera protocol here — the strict guide's **First-step -perception protocol** is the source of truth: agentview / agentview-hi is the -semantic identity authority (decides *which* object/surface satisfies the task -language + relation); wrist / wrist-hi refines geometry for the *same* candidate -(accept only within ~3–5 cm of the agentview anchor, never average, basket/cavity -excepted). Both maps share one world frame, read via -`back_project({"camera":"wrist"})`; the optional `segment` honours both cameras -(`{"camera":"agentview"|"wrist"}`, `min_score` default 0.2, or `point:[row,col]`). - -**PRO implication:** `_swap` (and `_task`) can *invert* target or destination -semantics, so **never reuse base-task or recipe coordinates** — the object -satisfying the relation may now sit where the base task never put it. Let -agentview decide *what*; the near-vertical wrist is BAD at identity (locks onto -look-alikes) and only refines *where*. - -### 3.6c. Mandatory pre-task perception pass - -Also owned by the strict guide's First-step perception protocol: before ANY -pick/place, build the localization table (one row per task-relevant entity) and -pass the FINAL READY CHECK. **PRO implication:** in single-attempt mode a -wrong-target first grab is unrecoverable, and under `_swap` the "right" target is -exactly the one you'd get wrong by habit — so localizing all entities up front is -cheap insurance, not optional. - -### 3.6b. Hi-res perception channel (1024×1024) - -Also owned by the strict guide's Hi-res perception channel section (1024×1024 -pairs each step; prefer the hi image for identification; `back_project` defaults -`resolution="high"`; never mix pixel grids). **PRO implication:** hi-res fixes -*which* object you point at — decisive for the recolored/rescaled Object axis and -for telling swapped same-shape groceries apart — but does NOT change metric -accuracy or replace the wrist coarse→fine refinement. **Identity caveat:** SAM3 -gives two same-shaped, different-brand objects the SAME category (it labelled the -tomato-sauce and alphabet-soup cans identically) — use its mask only as a category -candidate, then READ the label yourself in the hi-res crop to assign identity; -never let SAM3's category settle a brand choice. - -## 4. The four-cell experiment per (base task, seed) - -For each base task you claim coverage on, generate four runs — every run is a -perception cell: - -| Suite | Variant | Perception-mode expectation | -|---|---|---| -| `libero_spatial` | base sanity | Pi0 and hybrid both pass | -| `libero_spatial_task` | **P1 Task** | Pi0 ✗ (picks base target); hybrid ✓ (LLM flips target from instruction, then localizes it) | -| `libero_spatial_swap` | **P2 Position** | Pi0 mixed; hybrid ✓ — **by localizing the swapped object/fixture from depth, not from state** | -| `libero_spatial_lan` | Semantic | Both pass (paraphrase invariant) | +For swap-style perturbations: -Replace `spatial` with `object`, `goal`, or `10` for the other base suites. +1. Identify both swapped entities in the embedded agentview image. +2. Classify each by appearance and current relation, not expected seed layout. +3. Localize each independently. +4. Verify the chosen target still satisfies `task_language`. +5. Re-localize after any contact that could move either entity. -### 4.1. Hybrid run — the MCP runner +The runtime withholds privileged coordinates, so there is no coordinate field +to fall back on. The current images and geometry tools are the source of truth. -Launch a cell with the CLI; the runner owns `env_server.py`, exposes the -structured tools, and runs single-attempt: +## Destination Classification -```bash -python rpent/cli/main.py --env libero --suite --task --seed \ - --libero-type pro --planner claude_code --model claude-opus-4-8 +Pro scenes frequently contain look-alike surfaces. Before placement, explicitly +classify all plausible destinations: -# e.g. --suite libero_spatial_task 0 (P1) · libero_spatial_swap 0 (P2) · -# libero_goal_swap 2 (P2 fixture swap) · libero_10_task 5 (long horizon) -``` +- plate versus stove burner; +- cabinet top versus drawer opening; +- basket interior versus rim; +- microwave cavity versus door or surrounding counter; +- movable lid versus fixed fixture surface. -`--libero-type pro` may be given as `LIBERO_TYPE=pro` instead. +Only after semantic classification should you call `back_project` or `segment` +for coordinates. -Audit + recipe land in the run's `output_dir` (`output_dir` and `recipe_tag` -arrive in your first message): +## Mid-Carry Re-Localization -``` -{output_dir}/{recipe_tag}.json <- you write this (write_text_file) -{output_dir}/recipe_{recipe_tag}.jsonl <- exported automatically by the runner -``` +Long-horizon Pro tasks often move or occlude objects during earlier steps. +Before each new pick or placement: -Do NOT write into `resources/libero/results_*_pert/` — that tree is a **read-only -seed-0 reference corpus**, not a write target. - -### 4.2. Environment server is runner-owned - -There is no manual server to launch and no REPL to drive: the MCP runner starts, -manages, and tears down `env_server.py` for you and blocks each tool call until -the next `states.json` entry is dumped. Do not start, stop, or background it, and -do not poll for readiness. - -### 4.3. Pi0 fullshot baseline - -The baseline is Pi0.5 driving the task end-to-end with the runtime's own -(perturbed) `task_language`. There is **no standalone baseline CLI in this repo**; -the numbers to compare against are the recorded full-shot results (the team's -`SUCCESS_RATES` table). **Do not invent a `pi0_baseline.py` path.** - -Pi0 never sees object coords in either mode, so there is no perception variant of -the baseline. Expected behavior: - -- **P1 (task):** Pi0 "succeeds at the wrong task" — it picks the *base*-task - target object, places it on the plate, and `libero_terminated=False` because the - goal predicate names a different object. This is exactly the gap the hybrid - closes. -- **P2 (position):** Pi0 picks / places at the *base* (un-swapped) location. - -### 4.4. Audit JSON — PRO + perception fields - -Start from the `strict_perception` audit schema in the perception protocol, and -add the PRO fields: - -```jsonc -{ - "suite": "libero_spatial_swap", - "task_id": 0, - "seed": 0, - "regime": "strict_perception", - "perturbation_type": "swap (P2 Position perturbation)", - "perturbed_task_language": "", - "perturbation_semantics": "", - "expected_baseline_behavior": "", - "strategy_notes": "HOW you localized — which pixel(s) in images_cam, back-projected world xyz; for swap, how you found the relocated object/fixture", - "pick_result": { /* the pi0_pick step's result */ }, - "final_state": { /* latest states.json entry's `state` field */ }, - "libero_terminated": true -} -``` +1. call `view_driver_state({"step": -1})` if the previous primitive result is + no longer in context; +2. inspect the newest embedded images; +3. re-localize any entity that may have moved; +4. update the working perception table; +5. verify the remaining primitive order still matches `task_language`. -`strategy_notes` **must** describe the localization (pixel → `back_project` → -world xyz). For swap cells, explicitly note that the relocated object/fixture was -found by perception, not by reading coords. If unrecoverable after honest -exploration, write `libero_terminated: false` with what you tried and which step -failed — never warp (teleport primitives are deleted; see Rule 4 in the -perception protocol). Write the audit with `write_text_file` to -`{output_dir}/{recipe_tag}.json`, then call `finish`. - -## 5. Perception-protocol rules — PRO clarifications - -- **Rule 0 (use images for reasoning).** Even more critical under PRO: P2 swap can - move a large object/fixture clear across the table, and you have *no* - coordinates to fall back on. `images_cam/image_cam_NN.png` + depth are your only - spatial truth — open `images_cam/` and describe the scene before deciding - targets. -- **Rule 1 (no `pi0_end_to_end`).** Pi0 does the grasp via `pi0_pick`; the LLM - scripts every motion + release. Under PRO this is doubly important — handing - back to Pi0 means handing back to the prompt-blind / memorized-place habit you - are trying to falsify. -- **Rule 2 (single-episode current run).** This is a one-shot eval: do not call - `reset` or `exit`. Recover *within* the episode when safe (re-localize, - re-pre-position, re-`pi0_pick`, walk the prompt ladder, - `rotate_pitch`/`move_pose`); otherwise write an honest stuck/failure audit and - `finish`. `_swap` typically needs an in-episode retry — document it. -- **Rule 4 (no teleport).** `set_object_pose`, `articulate_to`, `js_move_to`, - `carry_object` are deleted. A goal past OSC reach with no physical approach → - honest `libero_terminated:false`. -- **Rule 5 (assume solvable).** A localization that moves the gripper into thin - air means you picked a wrong pixel (a reflection, a rim, a decoy object under - the Object perturbation), not that the cell is unreachable. Re-look, re-pick, - re-`back_project` before concluding failure. - -## 6. Existing corpus +Never assume the initial coordinate remains valid after contact, release, or a +fixture interaction. -``` -robots/libero/guides/ -├── pro_hybrid_guide.md <- this file -├── strict_hybrid_guide.md <- perception protocol + Rules (source of truth) -└── env_calibration.md <- OSC frame bounds + safe altitudes -scripts/ -└── liberopro_register_perturbations.patch -resources/libero/memory/ <- MEMORY.md index + feedback_*/project_* notes -resources/libero/results_spatial_pert/ <- read-only seed-0 reference corpus -resources/libero/results_{object,goal,10}_pert/ <- same, other suites (seed-0) -``` +## Contact Tasks -Before you start, **read the auto-memory**: `resources/libero/memory/MEMORY.md` -(one-line hooks, auto-injected via CLAUDE.md). For perception PRO cells always -open `feedback_no_teleport_rule.md` and — for any `_swap` cell — -`feedback_swap_perturbs_fixtures.md` (what swaps, and why you re-find the -relocated fixture visually). For bowl→plate spatial tasks also read -`feedback_bowl_eef_y_offset.md`; for cluttered picks, `feedback_pi0_pick_full_prompt.md`; -after two failed retries, `feedback_failure_forensics.md`. The -`resources/libero/results_*_pert/` recipes are **inputs** (technique priors) — -consult them for prompt ladders, staging, and target zones, but never reuse their -coordinates (re-derive every xyz from THIS scene) and never write there. - -## 7. What to do next (priority order) - -1. **Extend spatial to all 10 tasks at seed 0**, four perception cells each - (base / `_task` / `_swap` / `_lan`). For hybrid runs, use the seed-0 reference - recipes in `resources/libero/results_spatial_pert/` as *technique* starting - points — the pick step is usually identical; the place target changes for - `_swap`, the target object changes for `_task`. Never reuse their coordinates. -2. **Scale to seeds beyond 0** (50 trials per task). Recipes must re-localize per - scene; a perception recipe is *data-flow* (perceive → plan → act), never - hard-coded xyz. -3. **Replicate on `libero_object`, `libero_goal`, `libero_10`.** Frame split - applies (read `states.json[0].state.robot0_eef_pos[2]`). For - `libero_goal_swap`, apply §3.5 — localize the **swapped fixture** visually. -4. **Aggregate into a main table** `(suite × perturbation × {Pi0, hybrid})`. The - headline number is the conditional: of the seeds Pi0 fails on, what fraction - does the *perception* hybrid solve? Because there is no oracle state anywhere in - the hybrid's reasoning here, this is the strongest single statistic the agentic - decomposition can claim. - -## 8. Quick reference for a brand-new session - -```bash -# 1. Sanity-check the liberopro patch (perturbed language must show) -LIBERO_TYPE=pro python -c \ - "import liberopro.liberopro.benchmark as b; print(b.get_benchmark('libero_spatial_task')().get_task(0).language)" -# -> must read 'Pick the akita black bowl not between ...' (the perturbed text) - -# 2. Read the auto-memory: resources/libero/memory/MEMORY.md - -# 3. Launch a perception cell (runner owns env_server; single-attempt) -python rpent/cli/main.py --env libero --suite libero_spatial_swap --task --seed 0 \ - --libero-type pro --planner claude_code --model claude-opus-4-8 -``` +Use `pi0_doubled` for learned contact behavior such as turning a knob or +opening/closing a drawer. Its success flag mirrors the benchmark termination +predicate, so an intermediate contact can be useful even when success is false. +Inspect state and image evidence after every contact attempt. + +Use short scripted alignment motions around contact skills. Avoid long blind +pushes, which can destabilize MuJoCo or move the end effector into an invalid IK +branch. + +## Planning Across Multiple Objects + +For multi-object tasks: + +- process objects in an order that minimizes collision and occlusion; +- preserve already-correct placements; +- route later carries around placed objects rather than over them; +- re-confirm each source and destination immediately before use; +- verify intermediate relations visually rather than assuming they held. + +When order is specified by the task, obey it even if another order appears +easier. + +## Reference Results And Memory + +Reference results and memory entries provide reusable strategy: + +- prompt ladders; +- manipulation ordering; +- safe approach and carry heights; +- fixture-specific contact patterns; +- known failure and recovery modes. + +They do not provide valid coordinates for the current run. Re-derive all +positions through perception. + +## Outcome And Audit + +The current benchmark outcome is top-level `libero_terminated` in the latest +environment tool result. It is not stored inside `state`. + +The audit should record: + +- exact perturbed `task_language`; +- perturbation type when known; +- semantic identification evidence; +- agentview and accepted wrist localization results; +- primitive sequence and recovery decisions; +- memory files consulted; +- final state and `libero_terminated`. + +Recipe export is runtime-managed from recorded primitives and successful +segmentation events. Artifact identifiers returned by tools are for audit and +traceability, not manual file access. + +## Quick Checklist -Then, inside the run: - -4. `view_driver_state({"step": 0})` → read `state.robot0_eef_pos[2]` to pick the - frame (§3.1). `Read images_cam/image_cam_00.png` (or hi-res) and - `view_camera_meta`. Localize the target (and, for `_swap`, the relocated - object/fixture) with `back_project` — run the mandatory pre-task perception pass - (§3.6c). Plan, then execute one structured tool at a time. -5. `write_text_file` the audit to `{output_dir}/{recipe_tag}.json` - (`regime: strict_perception`) before `finish`; the recipe - `{output_dir}/recipe_{recipe_tag}.jsonl` is exported automatically by the runner. - -When in doubt about *how to localize* or a primitive, the source of truth is -[`strict_hybrid_guide.md`](./strict_hybrid_guide.md); about *PRO setup / -perturbation semantics*, see -[`scripts/install_libero_pro_plus.sh`](../../../../scripts/install_libero_pro_plus.sh) -and §2. +- Read strict guide and relevant memory. +- Call `view_driver_state({"step": 0})`. +- Select the scene frame from initial EEF z. +- Read top-level `task_language`. +- Build the complete perception table. +- Re-derive all coordinates for this scene. +- Use agentview for identity and wrist for consistent refinement. +- Re-localize after contacts and placements. +- Check top-level `libero_terminated` after each relevant action. +- Write an honest audit and call `finish` without resetting. diff --git a/robots/libero/guides/strict_hybrid_guide.md b/robots/libero/guides/strict_hybrid_guide.md index 683feabe..3518a0a7 100644 --- a/robots/libero/guides/strict_hybrid_guide.md +++ b/robots/libero/guides/strict_hybrid_guide.md @@ -1,611 +1,206 @@ -# Strict Hybrid LLM + Pi0.5 — Perception-Isolated Guide - -You are taking over a hybrid LIBERO experiment in **perception-isolated** mode -— the only mode in this repository (the legacy oracle-state mode, where the -state JSON carried GT object coordinates, is not included here). - -> **Pi0.5 only does the grasp (`pi0_pick`). The LLM (you) handles every motion -> (`move_to`), every release, sequencing, retries — and you do not get GT -> object coordinates. You localize objects yourself from the depth + camera -> calibration the toolkit dumps each step.** - -## What's different from legacy oracle mode (read this first) - -| | legacy oracle mode (not in this repo) | **perception (this guide)** | -|---|---|---| -| how object coords are withheld | (none — full GT coords in `state`) | **the runner withholds them: `states.json` carries `object_names` only, no coordinates** | -| `states.json` objects | full `objects:{name:[x,y,z]}` | **`object_names:[…]` only — NO coords** | -| how you learn the task | env prompt / BDDL | **`states.json[NN].task_language`** (authoritative `:language`, coord-free) — never scrape the BDDL | -| extra obs artifacts | `images/image_NN.png` only | **+ `images_cam/`, `depths/`, `world/` (agentview); `images_wrist/`, `depths_wrist/`, `world_wrist/`, `wrist_meta/` (wrist); top-level `camera_meta.json`; hi-res `images_cam_hi/`, `world_hi/`, `images_wrist_hi/`, `world_wrist_hi/`** | -| cameras | agentview only | **agentview (fixed, ~1m → ±8–13 cm) + eye-in-hand wrist (moves with gripper, ±1–2 cm when <20 cm to target)** — coarse→fine, see below | -| how you get an object's xyz | read `state["objects"][name]` | **pick the object pixel, then `back_project({"row":ROW,"col":COL,"step":NN})` (K⁻¹+extrinsic already done); refine with the wrist map up close** | -| how you confirm the grasp | GT object-lift oracle | **grasp-only, no oracle — you judge the grasp from gripper width + wrist cam (Rule 1 / 1b)** | -| how you pick which of two identical objects | read their distinct coords | **by SPATIAL RELATION from `task_language` (elevation / left-right), never by `_1`/`_2` name** | -| cell budget | 600 (short suites) | **1200** — perceptual localization + manipulation is slower | -| audit `regime` | `strict` | `strict_perception` | - -How localization works (the core of this mode): - -The toolkit already back-projects EVERY pixel for you into world coordinates and -saves them as `world/world_NN.npy` (agentview) and `world_wrist/world_wrist_NN.npy` -(wrist) — both in the SAME world frame. **You do NOT write back-projection math; -you pick a pixel and call `back_project`, which indexes the map for you.** - -> **Path convention:** below, a bare file name refers to the file inside its own -> `output_dir` subdirectory — e.g. `image_cam_hi_NN.png` lives at -> `images_cam_hi/image_cam_hi_NN.png`, `world_NN.npy` at `world/world_NN.npy`, -> `image_wrist_hi_NN.png` at `images_wrist_hi/image_wrist_hi_NN.png` (see the obs -> artifacts row above for the full directory list). `NN` is the zero-padded step -> index. In practice you rarely hand-build these paths — each primitive tool -> already returns the resolved `image_cam_hi_path` etc. for the new step. - -1. Read `image_cam_hi_NN.png` — agentview RGB **in the calibration frame** - (vertical-flip of the raw buffer). This is the image you pick pixels in. - `image_NN.png` is the Pi0-rotation frame; **do not pick pixels there**. -2. Find the target object visually → pixel `(row, col)`. -3. Read its world xyz directly: `back_project({"row":ROW,"col":COL,"step":NN})`. - Sample 3–5 pixels on the object's top surface and median the returned xy - (robust to one mis-picked rim/edge pixel). That's the object's surface point - in world frame. - -### Hi-res perception channel (ON BY DEFAULT, 1024×1024) - -The toolkit dumps, every step, a 1024×1024 pair per camera IN ADDITION to the -256 files: `images_cam_hi/image_cam_hi_NN.png` + `world_hi/world_hi_NN.npy` -(agentview) and `images_wrist_hi/image_wrist_hi_NN.png` + -`world_wrist_hi/world_wrist_hi_NN.npy` (wrist). -**PREFER the hi pair for looking and identification** — a far object spans 4× -more pixels, and package text/label art is actually legible (e.g. "Cream -Cheese" vs "BUTTER" boxes are readable at 1024 and indistinguishable smears -at 256). If the hi files are absent, everything below works unchanged at 256. - -- A hi-res pixel `(row,col)` indexes ONLY the hi-res world map (`back_project` - default `resolution:"high"`; float16, same world frame). Never index a 1024 - pixel into the 256 map or vice versa — pass `resolution:"low"` only for a - pixel taken from a 256 image (convert by dividing/multiplying by 4 if needed). -- The `segment` command automatically uses the hi-res frame when present - (its `centroid_pixel`/`box` are then in 1024 coords). -- Metric accuracy of mask-median localization is the SAME at both resolutions - (the residual ~2 cm error is the surface-vs-center offset, not pixel size) — - the hi channel is for **identification**, not for replacing the wrist-cam - fine-localization protocol. -- Disk note: hi files keep only the LAST ~5 steps (rolling window); the 256 - history is complete. If the hi files are absent, everything below works - unchanged at 256. - -### Two cameras — agentview = IDENTITY, wrist = GEOMETRY - -**Roles are NOT symmetric (this is the core discipline, from the 80-task -localization sweep):** -- **agentview** (`image_cam_hi_NN.png` + `world_hi/world_hi_NN.npy`, ~1 m away): - the **semantic IDENTITY authority** — decides WHICH object/surface satisfies - the task language + spatial relation. At 1024 a far object spans enough pixels - to read labels/shape. Its metric precision is only ±8–13 cm, but identity is - its job, not millimetres. -- **wrist** (`image_wrist_hi_NN.png` + `world_wrist_hi/world_wrist_hi_NN.npy`, - <20 cm → ±1–2 cm): a **GEOMETRY refinement** camera for the SAME candidate - agentview already chose. It is near-vertical and **bad at identity** — it - cannot read side labels or tell duplicate/similar items apart, and in failed - probes it locked onto a look-alike hundreds of pixels away. **NEVER let the - wrist freely re-identify a non-basket target.** - -Protocol (non-basket objects/surfaces): -1. **Identity (agentview)** — in `image_cam_hi_NN.png` choose the target by RGB / - label / shape / global spatial relation; sample 3–8 pixels on it and median - `back_project` → the **identity anchor** (rough xy ok). -2. **Approach** — `move_to` ~15–20 cm directly above that anchor xy (toolkit - re-renders both cams after every primitive). -3. **Geometry refine (wrist)** — pick the SAME candidate's pixel in - `image_wrist_hi_NN.png`, `back_project` it with `camera:"wrist"`. **Accept - this corrected xy ONLY if it is within ~3–5 cm of the agentview anchor**; if - it jumps >5 cm, REJECT it (it hit a look-alike/background) and keep the - agentview xyz. The wrist may sharpen coordinates but may NOT override the - agentview semantic choice. Never average the two. -- **Basket / cavity special case:** for `basket`/cavity the wrist MAY also - confirm/refine the true interior centre (basket failures are rim/edge bias, - not semantic confusion). - -### Mandatory pre-task perception pass — localize EVERYTHING, THEN act - -Before ANY pick/place, build a localization table in your reasoning — one row -per task-relevant entity (every movable target, every destination/support/ -fixture, every relation landmark named by `task_language`), each with: -`name_or_role · agentview_evidence (why this candidate) · agentview_pixels · -agentview_xyz (median back_project on hi-res) · wrist_refine -accepted|rejected|basket_confirmed · final_xyz · uncertainty`. Do the -**FINAL READY CHECK** (every entity has a final xyz; non-basket wrist -refinements are spatially consistent with agentview; basket points are -interior-centred) — only then start manipulating. Rationale: in single-attempt -mode a **wrong-target first grab is unrecoverable** (it tips/displaces both the -grabbed object and the target zone), so the cheap insurance is to identify all -entities up front instead of recovering later. - -> The underlying math (already done for you): `cam = [(col−cx)·z/fx, -> (row−cy)·z/fy, z]` with `z=depth[row,col]`, then `P_world = -> extrinsic_cam2world @ [cam,1]`. You must invert `K` BEFORE the extrinsic — the -> old `E @ [col·z, row·z, z, 1]` recipe (no `K⁻¹`) is **wrong** (metres off). -> The `world/` and `world_wrist/` maps bake in the correct `K⁻¹`+extrinsic, and -> `back_project` reads them for you — that is precisely why `back_project` -> exists; you never write this math yourself. Forward world→pixel was verified -> 5/5 vs GT at design time (plate Δ=6 mm). The wrist map may be all-table/null -> early (the wrist only sees gripper + table until you move it over a target). - -## Before you start: READ THE AUTO-MEMORY - -Operating wisdom lives in the in-repo memory: +# Strict Hybrid LLM + Pi0.5 Perception Guide +This guide describes the perception-isolated LIBERO runtime. The agent receives +robot proprioception, object names, task language, and camera observations. It +does not receive privileged object coordinates. + +Pi0.5 performs grasping through `pi0_pick`. The LLM owns semantic perception, +localization, scripted motion, release, verification, and recovery within the +current episode. + +## Runtime Contract + +Every motion primitive appends a `StepRecord`. The record contains: + +- `step_idx`: sequential motion-step number; +- `state`: robot proprioception and coordinate-free scene information; +- `artifacts`: sorted logical base names for all files captured at the step; +- `command`, `result`, and `elapsed_s`; +- `extras`: LIBERO outcome fields such as `task_language`, + `libero_terminated`, and `episode_truncated`. + +Artifact storage is private to `EnvState`. Do not construct filenames or read +observation files manually. Use the structured tools: + +- `view_driver_state`: state, artifact names, log, and embedded images; +- `view_camera_meta`: calibration data for one camera and step; +- `back_project`: matching image pixel or region to world coordinates; +- `segment`: text- or point-prompted mask plus projected `world_xyz`. + +Step `0` is the initial observation. Step `-1` means the latest observation. +When a tool omits `step`, its schema default is `-1`. + +`states.json` is an internal versioned manifest. It is not an agent-facing state +API and should not be indexed directly. + +## Camera Roles + +`view_driver_state` embeds the best available images for the selected step: + +- **policy image**: the Pi0-oriented agentview input; +- **agentview image**: fixed global view, high resolution when available; +- **wrist image**: eye-in-hand view, high resolution when available. + +Use agentview for semantic identity and global relationships. Use wrist for +close-range geometry after the target identity has already been anchored in +agentview. The wrist view must not freely replace the semantic decision when +similar objects are nearby. + +Pixels passed to `back_project` must come from the same camera and resolution +specified in the call. The tool selects the matching world map internally. + +## Initial Perception Pass + +Before the first manipulation primitive: + +1. Call `view_driver_state({"step": 0})`. +2. Read the returned top-level `task_language` verbatim. +3. Identify every movable target, destination, support, and relation landmark + named by the task. +4. Inspect the embedded agentview image and classify candidates by color, + shape, label, container type, and spatial relation. +5. Localize each chosen candidate with several interior pixels and + `back_project`, or use `segment` when a stable text/point prompt exists. +6. Record a working table with semantic evidence, sampled pixels, projected + coordinates, uncertainty, and the selected final coordinate. + +Do not start manipulation until every required target and destination has a +defensible identity and coordinate estimate. + +## Semantic Identity Before Geometry + +Depth and world coordinates cannot distinguish visually similar surfaces. A +plate, stove burner, pot lid, and cabinet top can all be flat circular or planar +regions at similar heights. + +Classify the destination in RGB before localizing it: + +- **plate**: ceramic disc with a clean rim, often white or ringed; +- **stove region**: darker fixture surface, coil, grate, or burner pattern; +- **basket**: open container whose interior center differs from the rim; +- **cabinet or drawer**: fixture geometry associated with the task noun. + +When duplicate objects exist, choose by the relation in `task_language`, not by +internal object suffixes. + +## Coarse-to-Fine Localization + +For each non-basket object: + +1. Select the semantic candidate in agentview. +2. Project three to eight interior pixels and take a robust median. +3. Move approximately 15-20 cm above the agentview anchor. +4. Inspect the wrist image and refine the same physical candidate. +5. Accept the wrist estimate only when it remains within roughly 3-5 cm of the + agentview anchor. Otherwise reject it and retain the global estimate. + +For baskets and cavities, use agentview for global identity and wrist for the +interior center. Avoid rim pixels. `back_project` region mode can summarize a +bounded pixel window with optional `z_min` and `z_max` filtering. + +## Segmentation + +`segment` does not move the robot. Supply exactly one of: + +```json +{"prompt": "the black bowl on the stove", "camera": "agentview", "step": -1} ``` -resources/libero/memory/MEMORY.md + +```json +{"point": [420, 615], "camera": "agentview", "step": -1} ``` -Scan the ~30 one-line hooks. For perception cells **always** open: -- `feedback_no_teleport_rule.md` — the deleted primitives. -- `feedback_redo_cell_timeout_1200.md` — why the cell budget is long here. -(The grasp is oracle-free too: there is no GT-lift oracle — you judge the grasp -from gripper width + the wrist cam, see Rule 1 / 1b.) +The result includes the mask score, projected `world_xyz`, logical artifact +identifiers for audit, and an embedded overlay image when available. Inspect +the overlay before trusting the projection. -For bowl→plate spatial tasks always also read `feedback_bowl_eef_y_offset.md` -(bowl-eef y-offset 4.5 cm: place at `eef_y = plate_y + 0.045`, not `plate_y`). +Use plain visual phrases. Remove benchmark or brand names that the segmenter +cannot ground. If segmentation fails, choose pixels manually and call +`back_project`. -## Rule 1 — `pi0_pick` is grasp-only (no oracle) +## Grasping -`pi0_end_to_end` is FORBIDDEN. `pi0_pick` is for the grasp only; you script every -`move_to` and every `release`. Use: +Pre-position above the localized object before calling Pi0.5: -``` -pi0_pick({ - "prompt": "", +```json +{ + "prompt": "pick up the short red-label can", "max_chunks": 20, "lift_thresh": 0.05, "gripper_closed_thresh": 0.06 -}) +} ``` -`pi0_pick` takes **no object-tracking / oracle argument** — it is grasp-only -and reads NO GT object pose. Passing a name would do nothing even if you tried. -You judge "did I grab the target?" yourself (Rule 1b). NEVER let Pi0 finish the -place — YOU do every `move_to` and the `release`. - -## Rule 1b — JUDGE THE GRASP from perception, NOT from a name - -After a pick, decide "did I grab the target?" from two coord-free signals: - -- **Gripper** (`state.robot0_gripper_qpos` from the latest `states.json` - entry): fingers closed but NOT fully shut (~0.01–0.05 gap) ⇒ holding an - object; fully closed (~0.0) ⇒ grasped air. -- **Wrist cam**: after the lift, read `image_wrist_hi_NN.png` — the target - should be raised into the gripper and the spot it came from now empty (its - surface z jumps up by your lift distance). If needed, `back_project` wrist - pixels to confirm the surface z jumped. - -`pi0_pick`'s returned `success` (an eef-lift + gripper-closure heuristic, pure -proprioception) is a HINT, not proof — confirm with the wrist cam before -carrying. The only authoritative TASK-success signal is -`state.libero_terminated` (the benchmark predicate), which is name-independent. - -## Rule 4 — NO TELEPORT primitives (physics-only) - -`set_object_pose`, `articulate_to`, `js_move_to`, `carry_object` are **deleted -from the codebase**. They are not callable and are not in the tool list. If a -goal is past OSC reach and no physical approach works, write an honest -`libero_terminated:false` audit — never warp. - -## Rule 0 — Use images for reasoning, not just JSON state - -After every primitive tool call, inspect the returned state and image paths -(the tool returns them — no separate read needed). Read the new -`image_cam_hi_NN.png` path (calibration frame — the one you pick pixels in) -and, when close to a target, the `image_wrist_hi_NN.png` path. Even more -important here than in oracle mode: this is *the* signal you use to find -objects. - -`image_cam_hi_NN.png` (and its 256 counterpart `image_cam_NN.png`) is the -calibration-frame RGB — same scene as `image_NN.png` but vertically flipped so -that pixel coordinates align with the camera matrices in `camera_meta.json`. -Pick object pixels from these; `states.json` alone gives only proprioception + -object names. - -## Rule 2 — SINGLE EPISODE, NO RESET - -This is a **one-shot** evaluation: you get **exactly ONE episode**. Do NOT reset -and do NOT restart the episode. You MAY recover *within* this one episode -(re-localize — objects may have moved; re-pre-position; re-`pi0_pick` a missed -grasp; walk the next rung of the Pi0 prompt ladder in Rule 3; re-firm the grip; -`rotate_pitch`/`move_pose`) — that is all one continuous attempt. But the -instant you would want to start over, **STOP instead and write the audit** -(success or an honest `libero_terminated:false`), then call `finish`. Never -warp; never reset. - -## Rule 5 — Assume every task is physically solvable - -Same as oracle mode. A localization that "looks right" but moves the gripper -into thin air usually means your pixel was on a wrong surface (e.g. picked the -bowl's reflection on the table). Re-look at `image_cam_hi_NN.png`, pick a -different pixel firmly on the target's top, re-`back_project`. Don't conclude -"unreachable" until you've validated localization. - -## Rule 6 — Your task is `task_language`; the BDDL is FORBIDDEN - -Each `states.json[NN]` entry carries a **`task_language`** field — the -authoritative task instruction (the BDDL's `:language` tag, which contains -**no** coordinates). **Read it and obey it verbatim.** Do not infer the task -from object names, from sibling recipes, or by guessing a `task_map` index — -that produced *wrong-task* runs (an agent solving a task it was never assigned). - -> ⚠️ **Never read the BDDL files, import the benchmark, or query env object -> poses.** The BDDL is the one place the task language *and* the `:init` -> ground-truth coordinates live together — reading it to get the language also -> leaks the coordinates this mode exists to withhold (a perception-isolation -> breach; observed on several swap cells in earlier runs). You already have the -> task from `task_language`; you get object **positions** ONLY by depth -> back-projection. The runtime strips coords from `state` but does **not** -> sandbox the BDDL — that discipline is on you. - -## Rule 7 — Ground the target by its spatial RELATION, not its name - -When the task names a relation ("the bowl **ON THE COOKIES BOX**", "the mug -**LEFT OF** the plate"), the target is whichever object SATISFIES that relation -in the scene — find it by perception. Identical objects (`akita_black_bowl_1` vs -`_2`) carry NO perceptual difference in their names, so the name can't choose -for you; the RELATION disambiguates: - -- "on the cookies box" ⇒ the bowl that is **elevated** (~0.03–0.06 m above the - table, on the box) — distinguish it by its higher world-z from `back_project` - vs the table-level bowl. -- "left/right/front/back of X" ⇒ compare back-projected world xy to X's xy. - -Pick the target purely from where things ARE. (You never need to know which -`_N` name the target is — no primitive in this mode asks for one.) - -## Mental model - -1. **The runner (`rpent/cli/main.py`) owns a long-lived env server** — Pi0.5 + a - single-env LIBERO sim. It launches and manages the server; you do NOT start, - stop, or restart it. -2. **You call one structured MCP tool per step.** The tool BLOCKS until the - toolkit runs that one primitive and dumps the new step, then RETURNS the new - state + log + image paths. There is no file bus and no polling — the tool's - return value IS your signal. Each dumped step appends an entry to - `states.json` (entry `[NN]`) and writes `images/image_NN.png` + - `images_cam/image_cam_NN.png` + `depths/depth_NN.npy` + `world/world_NN.npy` - (+ the wrist and hi-res dirs) and, once, top-level `camera_meta.json`. -3. **You read the returned state, localize via `back_project`, decide the next - move, and call the next tool.** - -## Launch a session - -The runner (`rpent/cli/main.py`) launches and owns the env server (Pi0.5 + single-env -sim) — do not start/stop it. You call MCP tools; begin by reading step 0 via -`view_driver_state({"step":0})`. - -## The perception artifacts you read each step - -| artifact | what's in it | -|---|---| -| `states.json` (entry `[NN]`) | `step_idx`, `libero_terminated`, **`task_language` (your authoritative task instruction — the BDDL `:language` tag, coord-free; obey it verbatim)**, `state.{robot0_eef_pos, robot0_eef_quat, robot0_gripper_qpos, object_names}`, and the merged `command`/`result`/`elapsed_s` for that step. **No object coordinates.** Read via `view_driver_state({"step": NN})` (omit `step` = latest). | -| `images/image_NN.png` | RGB in Pi0 frame (180° rotated). *Do not pick pixels here for back-projection.* | -| `images_cam/image_cam_NN.png` | agentview RGB in **calibration frame** (vertical flip). Pick object pixels HERE (256 grid → `back_project` with `resolution:"low"`). | -| `images_cam_hi/image_cam_hi_NN.png` | **HI-RES 1024×1024** agentview RGB, calibration frame. **PREFER this for looking / identification**; `back_project` its pixels at the default `resolution:"high"`. | -| `depths/depth_NN.npy` | `(256, 256) float32` agentview metric depth (m), calibration frame. Same row/col as `image_cam_NN.png`. | -| `world/world_NN.npy` | `(256, 256, 3) float32` — **precomputed agentview world xyz per pixel** (K⁻¹+extrinsic done). Prefer `back_project`; read this manually only for debugging. Fixed cam (~1 m). | -| `world_hi/world_hi_NN.npy` | `(1024, 1024, 3) float16` — precomputed agentview world xyz per hi-res pixel. | -| `images_wrist/image_wrist_NN.png` (+ `images_wrist_hi/`) | **wrist (eye-in-hand) RGB**, calibration frame. Moves with the gripper. | -| `depths_wrist/depth_wrist_NN.npy` | wrist metric depth (m). | -| `world_wrist/world_wrist_NN.npy` (+ `world_wrist_hi/`) | `(256, 256, 3)` / `(1024, 1024, 3)` **precomputed wrist world xyz per pixel**, SAME world frame as agentview. ±1–2 cm when <20 cm to target. May be all-table/null until you move over the target. | -| `wrist_meta/wrist_meta_NN.json` | wrist intrinsics + extrinsic **for THAT step only** (the wrist cam moves). Read via `view_camera_meta({"camera":"wrist","step":NN})`. | -| `camera_meta.json` (top-level) | agentview `intrinsic_K` (3×3), `extrinsic_cam2world` (4×4), `depth_near/far`, projection recipe. Read via `view_camera_meta({"camera":"agentview"})`. | -| `segments/segment_NN_XX.json` | (after a completed `segment` response) `{found, mode, prompt|point, camera, source_step, score, box, mask_shape, centroid_pixel, world_xyz, n_pixels}` — SAM3's top mask back-projected via the matching world map. `world_xyz` is a robust MEDIAN over the whole mask. A valid no-detection response carries `{found:false, error}`. `XX` is a per-step index. | -| `segments/segment_overlay_NN_XX.png` | (only after a successful `segment` call) the segmented mask tinted red on the source image — read it to confirm SAM3 grabbed the right object. | - -The command + its result + `elapsed_s` are merged INTO the `states.json[NN]` -entry (and echoed in each primitive tool's return value) — there is no separate -per-step log file to read. - -## Localization with `back_project` (coarse agentview → fine wrist) - -You index the precomputed map through `back_project` — no K⁻¹ math. Coarse -(agentview) then, once the gripper is parked over the target, fine (wrist): +Treat `pi0_pick.success` as a hint. Verify the grasp from: -``` -# COARSE: agentview hi-res (default resolution:"high") — choose which object / rough xy -back_project({"row": ROW, "col": COL, "step": NN}) +- end-effector lift; +- nonzero gripper opening consistent with holding an object; +- wrist or agentview evidence that the correct object moved with the gripper; +- an empty or changed source location. -# FINE: wrist — after move_to ~15-20cm above the target, pick its pixel in -# image_wrist_hi_NN.png and back_project it (±1-2cm; refines the SAME candidate). -back_project({"row": ROW, "col": COL, "step": NN, "camera": "wrist"}) -``` +If the grasp failed, recover in the same episode by re-localizing, +re-positioning, and improving the visual prompt. Do not reset in single-attempt +evaluation mode. -(Pass `"resolution":"low"` only when `ROW,COL` came from a 256 image.) - -Tips: - -- Sample 3–5 pixels on the object (centre + a couple of edge pixels) and - median the back-projected xy — robust to a single mis-picked pixel. Avoid - pixels on the thin rim/edge or the gap to the table (those index a - background/edge depth and give a world point metres away). -- The returned point is the **visible surface** under your chosen pixel. - For a flat object (plate, basket) that surface ≈ the place target. For a - bowl/bottle, the surface is the top of the object; the rim's xy is what - you want for `release`, not the grasp's eef_y (apply - `feedback_bowl_eef_y_offset` — bowl `eef_y_target = perceived_plate_y + 0.045`). -- For the **table z** (when you need a known floor to compare against), - `back_project` a pixel on bare table near the object. - -## The command vocabulary - -Call one structured tool per step. The full primitive set (only the *control -signals* are listed here) — every one blocks until the step is dumped and -returns the new state + log + image paths: - -```jsonc -// === physics-only primitives (the entire allowed set) ===================== - -// Scripted EEF servo. action_scale=0.05 is the env's units; step_clip caps -// per-step Δxyz (m) BEFORE division by action_scale — smaller = slower. -move_to({"xyz": [x, y, z], "gripper": -1, - "tol": 0.012, "step_clip": 0.025, "max_steps": 80, - "action_scale": 0.05, "target_yaw": null}) // gripper: -1 open, +1 close - -// Pi0.5 closed-loop pick. Grasp-only — no oracle/tracking arg (Rule 1). -// You judge the grasp from gripper width + wrist cam (Rule 1b). -pi0_pick({"prompt": "pick up the X", "max_chunks": 20, - "lift_thresh": 0.05, "gripper_closed_thresh": 0.06}) - -// Pi0.5 for a contact skill (knob turn, drawer/door open-close — rare here). -// success mirrors libero_terminated only; inspect image/state for intermediates. -pi0_doubled({"prompt": "turn off the stove", "max_chunks": 20}) - -// Open gripper to place. Triggers libero termination if the On/In predicate met. -release({"max_steps": 20}) - -// Hold pose + drive gripper. Use to firm a grip mid-carry. -set_gripper({"gripper": 1, "steps": 5}) // -1 open, +1 close - -// Wrist yaw (world-z). Provide target_yaw (absolute) OR delta_yaw (relative). -rotate_wrist({"target_yaw": 0.0, "gripper": 1, - "max_steps": 40, "tol": 0.02, "step_clip": 0.10}) - -// Tilt eef pitch (axis-angle X). Cavity entry / micro-aiming. target_pitch OR -// delta_pitch. Use before threading a narrow opening whose face normal is ±y. -rotate_pitch({"target_pitch": 0.9, "gripper": 1, - "max_steps": 40, "tol": 0.02, "step_clip": 0.10}) - -// Co-vary xyz + pitch + yaw — threads cabinet-front IK singularity that move_to -// walls at. gripper defaults to -1 (OPEN) — pass gripper:1 while holding. -move_pose({"xyz": [x, y, z], "target_pitch": 0.0, "target_yaw": 0.0, - "gripper": 1, "step_clip": 0.02, "pitch_step": 0.08, "yaw_step": 0.08, - "tol": 0.012, "ori_tol": 0.05, "max_steps": 150}) - -// SAM3-grounded localization — does NOT move the robot and does NOT -// replace manual back-projection. Segments the most-recent dumped image for the -// prompt, back-projects the mask through the matching world map, and writes -// segments/segment_NN_XX.json {score, box, centroid_pixel, world_xyz (robust MEDIAN over -// the whole mask), n_pixels} + segments/segment_overlay_NN_XX.png. Read world_xyz -// directly instead of eyeballing a pixel. camera "wrist" uses the wrist world -// map for fine refinement (move the eef over the target first). Pass -// "point":[row,col] for a point prompt instead of text; prompt and point are -// mutually exclusive. min_score default 0.2. -// RPent manages the SAM3 service. If one call fails or finds nothing, the result is -// an {"error":...,"fallback":...} dict — fall back to picking a pixel in -// image_cam_hi_NN.png and calling back_project. -segment({"prompt": "the black bowl on the stove", "camera": "agentview", - "point": null, "min_score": 0.2}) - -// === FORBIDDEN — DELETED FROM THE CODE — DO NOT EMIT ============ -// set_object_pose, articulate_to, js_move_to, carry_object -``` +## Scripted Motion + +Use short, staged `move_to` commands. Do not issue a single horizontal move +larger than approximately 0.30 m. Long commands can switch IK branches and move +the end effector into the wrong half-space. + +After every motion: + +1. Inspect the returned `result` and final distance. +2. Inspect the new state and embedded images. +3. Re-localize anything that may have moved. +4. Continue only when the observed state matches the plan. + +Use `rotate_wrist`, `rotate_pitch`, or `move_pose` when orientation matters. +Use `pi0_doubled` for short learned contact interactions such as knobs, +buttons, doors, or drawers. Alternate it with short, capped scripted alignment +motions; never use one long blind push. + +## Placement + +Carry objects at a collision-safe height. Reconfirm the destination before +release, especially when plate/burner or rim/interior confusion is possible. + +For open containers: + +- localize the interior rather than the nearest rim; +- place above the interior center; +- lower enough to avoid a high-energy drop; +- inspect the post-release observation and outcome flag. + +If the task predicate does not fire, reclassify the destination before assuming +the grasp failed. Wrong-surface placement is a common cause. + +## Completion And Audit + +The current outcome is the top-level `libero_terminated` value returned by the +latest environment tool. Do not look for it inside `state`. + +On completion, write the requested audit with: + +- suite, task, seed, and evaluation regime; +- exact memory files consulted; +- semantic identification and localization strategy; +- grasp and placement evidence; +- final state and `libero_terminated`; +- honest failure details when the task remains incomplete. + +Then call `finish`. Recipe generation is handled by the runtime from recorded +primitive and successful segmentation events. + +## Final Checklist -### How to use `segment` (SAM3) — practical tips - -`segment` is the fastest way to localize: one call gives you a `world_xyz` -that is a robust MEDIAN over the whole object mask (hundreds of pixels), which -beats eyeballing 3–5 pixels. Workflow: - -1. Read `image_cam_hi_NN.png`, decide the target by the task's spatial RELATION. -2. Call `segment` with a **plain visual phrase + the relation**, then read the - returned `world_xyz` and the `segments/segment_overlay_NN_XX.png` to confirm the mask - is on the right object before you move. -3. **Two camera views — YOUR choice via the `"camera"` field** (`segment` works - on either; default `"agentview"`): - - `"camera":"agentview"` — the fixed ~1 m cam (±8–13 cm). Use it to choose - WHICH object (global layout / spatial relation) and get a rough xy. - - `"camera":"wrist"` — the eye-in-hand cam (±1–2 cm), reads the wrist world - map in the SAME world frame. Use it to PRECISELY localize once the gripper - is parked over the target. ⚠ It is null / all-table until you `move_to` - ~15–20 cm over the target (the wrist only sees the gripper+table before - that), so segment agentview first, approach, THEN segment wrist. - Typical coarse→fine: agentview select → `move_to` above → wrist refine → grasp. - You decide when the wrist refinement is worth it (small / closely-spaced / - identical objects benefit most; a big isolated plate may not need it). - -**Prompt phrasing (important — SAM3 is sensitive):** -- ✅ Use plain colour + shape + spatial relation: `"the black bowl on the stove"`, - `"the white plate"`, `"the bowl on the cookies box"`. -- ❌ Do NOT use the object's internal/brand NAME from `object_names`/BDDL: - `"the akita black bowl"` scores ~0.03 (no detection) because SAM3 can't ground - "akita"; the same object as `"the black bowl on the stove"` scores ~0.76. Strip - proper nouns (`akita`, `glazed_rim_porcelain_…`) — say what it *looks like*. -- For two identical objects, the relation in the prompt (`…on the stove`, - `…left of the plate`) usually steers SAM3 to the right instance; verify via the - overlay and the world-z (an object *on* a fixture has a higher z than a - table-level twin). The two-camera relation protocol still applies when SAM3 - can't disambiguate from text alone. -- If `segment` returns `{"error":...}` (low score / service down), walk the - prompt (drop the brand word, simplify, add the relation) or fall back to - manual pixel → `back_project`. - -## The strict-hybrid recipe (perception variant) - -A typical bowl→plate cell looks like: - -1. `view_driver_state({"step":0})`; inspect `task_language`, - `image_cam_hi_00.png`, and `camera_meta.json`. -2. Identify the target by the task's spatial RELATION, not its name (Rule 7). -3. Localize it — pick its pixel in `image_cam_hi_00.png`, - `back_project({"row":ROW,"col":COL,"step":0})` (coarse). Refine with the - wrist cam after pre-positioning (fine). -4. **Pre-position** ~15–20 cm above it: `move_to([obj_x, obj_y, carry_z])`, - gripper open. Re-`back_project` a wrist pixel here and correct the xy before - grasping. -5. `pi0_pick` with the right prompt (start "pick up the {object}", escalate per - the Pi0 ladder — see below). Confirm the grasp via gripper + wrist (Rule 1b). -6. `set_gripper({"gripper":1,"steps":5})` to firm the grip. -7. Localize the placement region (basket / plate / drawer slot) the same way. -8. `move_to([place_x, place_y, carry_z])` to traverse at constant height. -9. Optionally descend `move_to([place_x, place_y, place_z])`. -10. `release` — predicate (`On`/`In`) checks → `libero_terminated=True` if hit. -11. Light retreat (`move_to` upward) so the next step's image is clean. - -> **Predicate fire timing.** Most LIBERO `On(X, Y)` predicates fire on -> `release` if `X` is above `Y`'s region. `In(X, container)` needs `X` to -> have actually entered the container's volume before release. If a release -> fires `term=False` but the object is on top of the right region, descend -> 1–2 cm more and re-`release`. - -## Pi0 prompt ladder (Rule 3 — Pi0 IS the delivery service) - -Try in order; each rung uses a slightly more specific prompt or a re-pre-pos. - -1. `"pick up the {object}"` — generic sub-instruction. -2. The full `task_language` verbatim (e.g. `"Pick the akita black bowl on the - cookies box and place it on the plate"`). -3. Add a spatial qualifier (`"…on the wooden cabinet"`, `"…next to the - basket"`). -4. Re-pre-position 5 cm lower or shifted, then retry rung 2 or 3. - -Empirically Pi0 sometimes needs the **full prompt with the spatial -qualifier** for elevated picks (stove, cabinet-top, drawer). See -`feedback_pi0_pick_full_prompt.md` and the prompt-ladder note in MEMORY. - -## Key hyperparameters - -- Single-step `xyz` within ±0.30 m of current eef or OSC flips IK. Split - long traversals into 2–3 carry-z waypoints. -- `lift_thresh`: 0.05 (flat/stable) / 0.08 (slippery tall bottles). -- `step_clip`: 0.025 (empty / box) / 0.015 (cans) / 0.012 (tall bottles). -- Frame z (from `state.robot0_eef_pos[2]` at step 00): - ≈ 0.68 → LIVING_ROOM, ≈ 1.17 → KITCHEN, ≈ 0.26 → OBJECT. -- BOWL: `eef_y_target = perceived_plate_y + 0.045` (bowl-eef y-offset). -- TALL BOTTLES: carry at `z=0.30`, release without descending. -- Approach high-then-vertical; recover by re-`pi0_pick`, not by hovering. - -## Reading state - -After every primitive tool call, the return value already carries the new -`state`, the merged `command`/`result`/`elapsed_s` log, and the image paths — -you do **not** need a separate read. When you need an older step, call -`view_driver_state({"step":NN})` (omit `step` for the latest): - -1. Check `result.success`, `final_dist_m`, `peak_lift_m`, etc. in the returned - log. -2. Read the returned `image_cam_hi_NN.png` path → visual confirmation; pick - pixels for any new localization. -3. Check `state.robot0_eef_pos` / `robot0_gripper_qpos` / `libero_terminated` - in the returned `state`. - -You **do not** open the depth maps yourself — feed a pixel to `back_project`. -Don't call `view_driver_state` immediately after a primitive that already -returned the new state. - -## Common failure modes - -- **Pick missed (gripper closed empty).** `result.peak_lift_m` < `lift_thresh`, - `min_gripper_opening` ≈ 0. Re-pre-position 1–2 cm lower or shifted; retry - Pi0 with next prompt-ladder rung. -- **Object slipped mid-carry.** `release` returns `term=False` and the object - is no longer where you `release`d it. `release`, re-pre-pos above it, - `pi0_pick` again, traverse again. -- **OSC stuck.** `move_to` returns `final_dist_m > 0.05` at `max_steps` and - same xy twice → try `rotate_pitch`, split into more waypoints, or - `move_pose` (co-varying) for the cabinet-front singularity. -- **Placement off because localization was wrong.** The release puts the - object on bare table instead of on the plate. Re-read `image_cam_hi_NN.png`, - pick a different pixel on the target region (sample 3 pixels on the plate's - flat top, median the back-projected xy), redo. Tip: if depth at your chosen - pixel is much closer than the table z, you picked the camera's near edge / - an object rim — pick again. - -## Verifying strict compliance - -Before saving the audit, confirm your command history is physics-only. The -teleport primitives are not even in the tool list, but audit it anyway: read -the `command` field of every `states.json` entry (via -`view_driver_state({"step":NN})` per step, or `read_text_file` on -`{output_dir}/states.json`) and confirm every `command.action` is one of the -allowed physics primitives (`move_to`, `pi0_pick`, `pi0_doubled`, `release`, -`set_gripper`, `rotate_wrist`, `rotate_pitch`, `move_pose`) — no -`set_object_pose` / `articulate_to` / `js_move_to` / `carry_object` appears. - -## Persisting successful runs as audit JSONs - -When `state.libero_terminated == true`: - -a. The working command recipe (`{output_dir}/recipe_{recipe_tag}.jsonl`) is - **auto-exported by the runner** from the non-error primitive commands in - `states.json` plus successful segment calls recorded in - `segments/segment_*.json`, merged in execution order — you do NOT - hand-write it. -b. Write a minimal audit JSON with `write_text_file` to - `{output_dir}/{recipe_tag}.json` with at least: `suite`, `task_id`, `seed`, - `regime: "strict_perception"`, `strategy_notes` (mention HOW you localized — - which pixel, depth, back-projected world xyz), `pick_result` (the `result` - from your `pi0_pick`), `final_state` (the latest `states.json` entry's - `state` field), `libero_terminated: true`. -c. Call `finish({"status":"success","summary":"…"})`. - -If unrecoverable after honest exploration in this one episode, write -`{output_dir}/{recipe_tag}.json` with `libero_terminated: false` + -`strategy_notes` describing what you tried, the back-projected xyz you used, and -which step failed. Then call `finish` (NO reset, NO second attempt). - -> The `regime: "strict_perception"` tag distinguishes these audits from the -> oracle-state `strict` regime in mixed-mode datasets. - -## Iteration heuristics - -- After 2 failed retries on the same step, **stop tuning numerics** and - inspect images: read `image_cam_hi_NN.png` at pick / pre-release / - post-release. The visual disagreement is usually the bug (you picked the - wrong pixel, or Pi0 grabbed the decoy). This is the lesson of - `feedback_failure_forensics.md` — applies even more strongly here. -- If a release reports `term=False` but the object is visibly on the target - region, descend 1–2 cm more and `release` again; predicate often needs - contact, not just hover. -- Once you use `back_project`, trust it: if a localization "feels off", it's - because you picked the wrong pixel — not because the depth / calibration is - wrong. (`back_project` inverts `K` before the extrinsic; the raw - `E @ [col·z, row·z, z, 1]` form skips it and is wrong — which is why you use - `back_project` and never hand-roll the math.) - -## What "strict_perception" means concretely - -- **No GT object coordinates anywhere in your reasoning.** The `state` you - read has none; the only legitimate sources of object xyz are the camera - images + depth + the precomputed `world/` and `world_wrist/` maps (via - `back_project`). -- **No teleport primitives.** The four are deleted. -- **Pi0 only does the grasp.** You script every motion + release. -- **Fully oracle-free, including the grasp.** There is no GT-lift oracle — - `pi0_pick` reads NO GT object pose and takes no tracking argument. You judge - the grasp from gripper width + the wrist cam, and TASK success from - `state.libero_terminated` (the benchmark predicate). -- **Single attempt.** One episode, no reset (Rule 2). -- The expected audit `regime` is `strict_perception`. - -## Reference cases - -The seed-0 sweep results live under `resources/libero/results_*_pert/` -(`results_10_pert`, `results_object_pert`, `results_spatial_pert`, -`results_goal_pert` — PRO swap+task, t0–t9). Each solved cell has an audit JSON -+ a `recipe_{tag}.jsonl` command sequence. The consistent winning pattern: -localize → pre-pos → `pi0_pick` → `set_gripper` → move → `release` in 6–12 -commands. - -When you write a new audit, browse a sibling cell's `recipe_{tag}.jsonl` as a -*technique* template — but never paste its xyz; re-derive every position via -`back_project` from THIS scene's depth. - -Begin by reading `resources/libero/memory/MEMORY.md`, then call -`view_driver_state({"step":0})` and inspect the returned `image_cam_hi_00.png` -(+ `camera_meta.json` via `view_camera_meta`); localize the target object via -`back_project`, then plan and execute. +- Initial state read with `step: 0`. +- Task language copied from the returned tool result. +- All targets and destinations semantically identified in agentview. +- Coordinates obtained through `back_project` or `segment`. +- Wrist refinements checked against the agentview anchor. +- Grasp confirmed visually and proprioceptively. +- Long motion split into safe waypoints. +- Destination reclassified immediately before release. +- Latest result checked for top-level `libero_terminated`. +- Audit written before `finish`. diff --git a/robots/libero/prompts/system.py b/robots/libero/prompts/system.py index 16bd1e93..70591e30 100644 --- a/robots/libero/prompts/system.py +++ b/robots/libero/prompts/system.py @@ -10,7 +10,7 @@ > persistence / up to N attempts" instruction anywhere below).** This is a > ONE-SHOT evaluation: you get **exactly ONE episode**. You MUST NOT call > `reset`, and you must not restart the episode. Plan carefully, then execute -> your single best manipulation sequence toward `state.libero_terminated == true`. +> your single best manipulation sequence toward `libero_terminated == true`. > You MAY recover *within* this one episode (re-pre-position, re-`pi0_pick` a > missed grasp, walk the Pi0 prompt ladder, `rotate_pitch`/`move_pose`) — that is > all one continuous attempt — but the instant you would want to reset/start over, @@ -70,7 +70,7 @@ weak at reading side labels or distinguishing similar grocery items (ketchup/BBQ/tomato sauce, soup cans, cream cheese/butter). Do NOT let the wrist freely re-identify a non-basket target; it often locks onto a look-alike. - Instead: choose the target from `image_cam_hi_NN.png`, compute its agentview + Instead: choose the target from the high-resolution agentview image, compute its xyz, move over that candidate, project/track that SAME candidate in wrist, and refine only its surface/center coordinates. SAM3 scores ~0.02-0.06 on brand nouns ("alphabet soup", "tomato sauce") — prompt by colour+shape ("the short @@ -146,58 +146,28 @@ - Under some runtimes these same tools may appear namespaced; call the actual tool name shown in your tool list, preserving the same arguments and semantics. -The toolkit writes artifacts in `{{output_dir}}/`: - -- `{{output_dir}}/states.json` — top-level JSON array; each entry has - `step_idx`, `task_language`, `libero_terminated`, `state` (robot - proprioception + object_names; NO object coordinates), `command`, `result`, - `elapsed_s`, and world-map path fields when available. -- `{{output_dir}}/images/image_NN.png` — agentview RGB, 180°-rotated (Pi0 frame; - do NOT use for back-projection). -- `{{output_dir}}/images_cam/image_cam_NN.png` — agentview RGB in the CALIBRATION - frame; use for low-resolution pixel checks. -- `{{output_dir}}/depths/depth_NN.npy` — agentview metric depth (meters), - calibration frame. -- `{{output_dir}}/world/world_NN.npy` — HxWx3 precomputed world xyz per 256px - agentview pixel. Prefer `back_project`; read this manually only for debugging - or if the tool is unavailable. -- `{{output_dir}}/images_wrist/image_wrist_NN.png` — wrist RGB, calibration frame. -- `{{output_dir}}/depths_wrist/depth_wrist_NN.npy` — wrist metric depth (meters). -- `{{output_dir}}/world_wrist/world_wrist_NN.npy` — wrist world xyz map in the - SAME world frame as agentview. -- `{{output_dir}}/wrist_meta/wrist_meta_NN.json` — wrist intrinsics + extrinsic - FOR THAT STEP ONLY (the wrist cam moves, so it changes every step). -- `{{output_dir}}/images_cam_hi/image_cam_hi_NN.png` — HI-RES (1024x1024) - agentview RGB in calibration frame. USE THIS to inspect the scene and identify - objects — a far object spans 4x more pixels than at 256. -- `{{output_dir}}/world_hi/world_hi_NN.npy` — 1024x1024x3 float16 precomputed - world xyz per hi-res agentview pixel. Prefer `back_project`; if you manually - inspect it, never index a low-res pixel into this grid or vice versa. -- `{{output_dir}}/images_wrist_hi/image_wrist_hi_NN.png` / - `{{output_dir}}/world_wrist_hi/world_wrist_hi_NN.npy` — same hi-res pair for - the WRIST cam. - ⚠ Hi-res pixel (row,col) indexes ONLY the hi-res world map (and 256 pixel -> - 256 map). Don't mix grids; if you must convert, divide hi coords by 4. - ⚠ Hi-res files keep only the LAST 5 STEPS (disk); for older before/after - comparisons use the 256 files or `states.json` history. -- `{{output_dir}}/camera_meta.json` — agentview intrinsics K, cam->world - extrinsic, projection recipe. -- `{{output_dir}}/action_videos/step_NN_.mp4` — per-action clips generated - when the Dashboard is enabled. - -NN is zero-padded sequential (`00`, `01`, `02`, ...). Initial state step `00` is -dumped before you begin. Use `view_driver_state({"step": 0})` to read it.""" - -GOAL = """YOUR GOAL: produce `state.libero_terminated == true` in ONE episode. ⛔ NO +The driver records a state and an `observation` dictionary for every motion +step. Observation entries name the available policy, agentview, wrist, depth, +world-map, and metadata artifacts, but storage paths are internal to the +runtime. Do not construct or read artifact paths manually. + +Use `view_driver_state` to retrieve a state. It embeds the policy image and the +best available agentview and wrist images, preferring high resolution. Use +`view_camera_meta`, `back_project`, and `segment` to consume metadata, depth, +and world maps. These tools guarantee that the selected camera, resolution, +and step use matching artifacts. + +Step `0` is the initial state. Step `-1` selects the latest state.""" + +GOAL = """YOUR GOAL: produce top-level `libero_terminated == true` in ONE episode. ⛔ NO `reset`, NO retry (SINGLE-ATTEMPT MODE — see the override at the very top; it supersedes any reset/retry wording in the Rules below).""" RULES = """Rule 0 — USE IMAGES. After every primitive tool call, inspect the returned state - and image paths. If you need a state again, call `view_driver_state`. Read the - new `image_cam_hi_NN.png` path (calibration frame — the one you pick pixels in) - and, when close to a target, the `image_wrist_hi_NN.png` path. The image is - your spatial-reasoning input; `states.json` only gives proprioception + object - names. + and embedded images. If you need a state again, call `view_driver_state`. + Use agentview for global layout and wrist for close-range geometry. The image + is your spatial-reasoning input; the JSON state only gives proprioception + + object names. Rule 1 — Pi0 is ONLY for the grasp. Use: pi0_pick({ @@ -215,19 +185,19 @@ Rule 1b — JUDGE THE GRASP from perception, NOT from a name. After a pick, decide "did I grab the target?" from two coord-free signals: • GRIPPER (proprioception): `state.robot0_gripper_qpos` from the latest - `states.json` entry — fingers closed but NOT fully shut (~0.01–0.05 gap) + state record — fingers closed but NOT fully shut (~0.01–0.05 gap) ⇒ holding an object; fully closed (~0.0) ⇒ grasped air. - • WRIST CAM: Read `image_wrist_hi_NN.png` after lifting. The target should + • WRIST CAM: inspect the returned wrist image after lifting. The target should now be raised into the gripper, and the spot it came from should be EMPTY. Compare before/after wrist or agentview evidence; if needed, use `back_project` on wrist pixels to confirm the target surface z jumped up. `pi0_pick.success` (eef-lift + gripper-closure heuristic) is a HINT, not proof — always confirm with the wrist cam before carrying. -Rule 2 — Inspect THEN act. Call `view_driver_state({"step": 0})`, read the - returned high-resolution image path(s), and inspect the relevant memory/guides - BEFORE your first primitive. **Your task is `states.json[0]["task_language"]` - — read it and obey it verbatim.** This is the authoritative instruction (the BDDL +Rule 2 — Inspect THEN act. Call `view_driver_state({"step": 0})`, inspect the + returned high-resolution images, and inspect the relevant memory/guides + BEFORE your first primitive. **Your task is the returned `task_language`; + read it and obey it verbatim.** This is the authoritative instruction (the BDDL `:language` tag). Do NOT infer the task from object names, from sibling recipes, or by guessing a task_map index — those caused wrong-task runs in the past. @@ -257,7 +227,7 @@ plate, a stove burner/cook-region, a wooden-cabinet top, and a pot lid all read as "flat disc at table height" in back-projected coordinates. They are only separable in the RGB. So before you carry-and-release onto a surface, look at - `image_cam_hi_NN.png` (and the wrist `image_wrist_hi_NN.png` once close) and + returned agentview image (and the wrist image once close) and NAME each candidate surface: • PLATE ⇒ ceramic disc, usually white, with a clean raised rim (often colored concentric rings). This is the place target for "place it on the plate". @@ -292,8 +262,7 @@ LOCALIZATION = """This is the core of perception-isolated mode. To find where an object is: -1. Look at `image_cam_hi_NN.png` (1024x1024 — PREFER THIS; fall back to the - 256 `image_cam_NN.png` only if the hi file is absent) and find the target +1. Look at the returned agentview image (high resolution when available) and find the target object's pixel (row, col). (row = vertical/y from top, col = horizontal/x from left.) 2. Call `back_project` on that pixel: @@ -316,7 +285,7 @@ metres away. Pick pixels firmly on the object's top surface.) ALWAYS apply the manipulation offsets from memory to the PERCEIVED position -(e.g. BOWL: eef_y = plate_y + 0.045). Verify visually in image_cam after moving.""" +(e.g. BOWL: eef_y = plate_y + 0.045). Verify visually in agentview after moving.""" PERCEPTION_ALGORITHM = """This is the default perception algorithm for EVERY cell (from the 80-task localization sweep: `agentview_identity_wrist_geometry_except_basket`). @@ -335,11 +304,11 @@ ALGORITHM (run this BEFORE manipulating): -1. From `states.json[0]["task_language"]` + `image_cam_hi_00.png` + +1. From the initial `task_language` + returned agentview image + object_names, infer the task-relevant TARGETS and DESTINATIONS (language only; never BDDL/poses). -2. GLOBAL SEMANTIC PASS (agentview hi-res): in `image_cam_hi_NN.png` choose each +2. GLOBAL SEMANTIC PASS (agentview hi-res): in the returned agentview image choose each target/destination candidate by RGB, label/shape, and global spatial relation. For duplicates (two bowls/plates/mugs) pick by RELATION (on stove, on cookie box, left/right/front/back), not `_1/_2`. For sauce/can/box groceries use the @@ -348,7 +317,7 @@ burner vs cabinet/drawer vs basket) semantically in RGB here. 3. COARSE XYZ (agentview): pick 3-8 pixels firmly on the chosen candidate in - `image_cam_hi_NN.png`, call `back_project` on the SAME pixels, take the median. + agentview image, call `back_project` on the SAME pixels, take the median. Avoid edges/holes/shadows/table-gaps. This median is the IDENTITY ANCHOR for that entity. @@ -431,8 +400,8 @@ command lists. Re-derive every coordinate from THIS scene. """, """INSPECT INITIAL STATE: call `view_driver_state({"step": 0})`; inspect -`task_language`, object_names, eef pose, `image_cam_hi_00.png`, -`image_wrist_hi_00.png` if useful, and `camera_meta.json`. Identify ALL target + `task_language`, object_names, eef pose, the returned agentview and wrist images, + and call `view_camera_meta` if needed. Identify ALL target objects, destination surfaces, and relation landmarks named by task_language. """, """RUN THE MANDATORY PRE-TASK PERCEPTION PASS (FIRST-STEP ALGORITHM above) — @@ -450,10 +419,9 @@ pi0_pick({"prompt": "...", "max_chunks": 20, ...}) release({}) -Each primitive tool blocks until the next `states.json` entry is dumped and -returns the new state view + log + image paths. Then inspect the returned state -+ high-resolution image paths (+ `back_project` as needed), decide, repeat -with NN=02, 03, ... +Each primitive tool blocks until the next state record is dumped and returns the +new state view, log, and embedded images. Inspect them, use `back_project` as +needed, decide, and repeat. """, """ALLOWED PRIMITIVES (physics-only; full schemas in the tool list/guides): `move_to`, `pi0_pick`, `pi0_doubled`, `release`, `set_gripper`, @@ -475,16 +443,17 @@ SAM3 localization aid — `segment` (no robot motion): instead of eyeballing a pixel, call `segment({"prompt":"the black bowl on the cookies box", "camera":"agentview"})`. It runs SAM3 on the current image, back-projects the -mask via the matching world map, and writes `segments/segment_NN_XX.json` with a robust -median `world_xyz` (+ a `segments/segment_overlay_NN_XX.png` to confirm the right -object). Use `camera":"wrist"` (after parking the eef ~15–20 cm over the +mask via the matching world map, and returns a robust median `world_xyz` plus +an embedded overlay image for visual confirmation. The logical +`segment_artifact` and `overlay_artifact` names are audit references, not paths +to open manually. Use `camera":"wrist"` (after parking the eef ~15–20 cm over the target) for ±1–2 cm refinement, or `"point":[row,col]` for a point prompt. Text `prompt` and `point` are mutually exclusive; provide exactly one. ⚠ PROMPT PHRASING (SAM3 is sensitive): use a plain colour+shape+RELATION phrase, NEVER the internal/brand name from `object_names`/BDDL. `"the akita black bowl"` scores ~0.03 (SAM3 can't ground "akita") whereas `"the black bowl on the stove"` scores ~0.76. Strip proper nouns (akita, glazed_rim_porcelain_…) — say what it -LOOKS LIKE + where it is. Always inspect the returned overlay path to confirm +LOOKS LIKE + where it is. Always inspect the returned overlay image to confirm the mask landed on the right object before moving. This is a CONVENIENCE alternative to manual back-projection — if it returns `{"error":..., "fallback":...}` (server down / low score / no detection), walk @@ -499,7 +468,7 @@ unrecoverable within this one episode, do NOT reset — write an honest stuck-audit (`libero_terminated:false`) and call `finish`. Never warp. """, - """WHEN state.libero_terminated == True: + """WHEN `libero_terminated == true` in the latest tool result: a. Write audit `{{output_dir}}/{{recipe_tag}}.json` with: suite, task_id, seed, regime:"strict_perception", strategy_notes (incl. how you localized), pick_result, final_state (latest state's `state`), diff --git a/robots/libero/prompts/user.py b/robots/libero/prompts/user.py index 668a4db4..63d171ac 100644 --- a/robots/libero/prompts/user.py +++ b/robots/libero/prompts/user.py @@ -10,9 +10,11 @@ - recipe: {{output_dir}}/recipe_{{recipe_tag}}.jsonl""" -MODE = """Use the high-resolution image paths returned by view_driver_state and -back_project to localize objects before motion.""" +MODE = """Inspect the embedded high-resolution images returned by +view_driver_state, then use back_project or segment to localize objects before +motion.""" -BEGIN = """read MEMORY.md, the guides, then `view_driver_state({"step":0})` and the -returned high-resolution images. Localize the target, then plan and execute.""" +BEGIN = """Read MEMORY.md and the guides, then call +`view_driver_state({"step": 0})` and inspect its embedded images. Localize every +task-relevant entity before planning and execution.""" diff --git a/robots/libero/toolkit.py b/robots/libero/toolkit.py index 71345caa..b6ae17d5 100644 --- a/robots/libero/toolkit.py +++ b/robots/libero/toolkit.py @@ -19,22 +19,6 @@ class LiberoToolkit(Toolkit): """Toolkit for the LIBERO environment.""" - _WIPE_STREAMS = ( - "image", - "image_cam", - "depth", - "world", - "image_wrist", - "depth_wrist", - "world_wrist", - "wrist_meta", - "image_cam_hi", - "world_hi", - "image_wrist_hi", - "world_wrist_hi", - "segments", - "action_videos", - ) # Tool schemas keyed by name (built once from the canonical ordered list # in libero_tools.TOOLS_SPEC) so each tool registers with its own spec. _SPECS = {spec["name"]: spec for spec in libero_tools.TOOLS_SPEC} @@ -44,11 +28,9 @@ def __init__( *, primitives_kwargs: dict[str, Any], dashboard_events: DashboardEventSink, - video_path: str | None = None, ) -> None: state = EnvState(get_output_dir()) super().__init__(dashboard_events=dashboard_events, state=state) - self._video_path: str | None = video_path self.init_primitives_clean(primitives_kwargs=primitives_kwargs) self._register_libero_tools() @@ -99,28 +81,36 @@ def _step(self, name: str, **kwargs) -> dict: else: result_dict = {"value": result} - step_idx = self._state.next_step_idx + record = libero_tools.dump_state( + self._primitives, + self._state, + log={"command": command, "result": result_dict, "elapsed_s": elapsed}, + ) + action_video_name = None if self._dashboard_events.enabled: - video_dir = libero_tools.artifact_path( - self._state.output_dir, "action_videos" - ) - video_path = video_dir / f"step_{step_idx:02d}_{name}.mp4" try: - self._primitives.save_frame_slice(start_frame, str(video_path), fps=20) + frames = self._primitives.frame_slice(start_frame) + if frames: + candidate = f"action_{name}.mp4" + if self._state.save( + candidate, + frames, + step=record.step_idx, + fps=20, + ): + action_video_name = candidate except Exception as e: get_logger("libero_toolkit").warning( - f"failed to save action clip to {video_path}: {e}" + "failed to save action clip for step %s: %s", + record.step_idx, + e, ) - libero_tools.dump_state( - self._primitives, - self._state, - step_idx=step_idx, - log={"command": command, "result": result_dict, "elapsed_s": elapsed}, - ) - out = libero_tools.view_driver_state(step_idx, state=self._state) + out = libero_tools.view_driver_state(record.step_idx, state=self._state) out["agent_elapsed_s"] = elapsed if result_dict.get("interrupted"): out.update(result_dict) + if action_video_name is not None: + out["action_video_artifact"] = action_video_name return out def init_primitives_clean( @@ -129,14 +119,7 @@ def init_primitives_clean( primitives_kwargs: dict[str, Any], ) -> None: """Wipe stale run artifacts, build the LiberoPrimitives, dump step 0.""" - self._state.reset(wipe_streams=self._WIPE_STREAMS) - out_dir = self._state.output_dir - for target in ( - libero_tools.artifact_path(out_dir, "metadata", camera="agentview", resolution="low"), - libero_tools.artifact_path(out_dir, "episode_video"), - ): - if target.exists(): - target.unlink() + self._state.reset() primitives = libero_tools.LiberoPrimitives( check_cancelled=self.raise_if_cancelled, @@ -144,28 +127,25 @@ def init_primitives_clean( ) primitives.reset() primitives.start_recording() - libero_tools.dump_state(primitives, self._state, step_idx=0, log=None) + libero_tools.dump_state(primitives, self._state, log=None) self._dashboard_events.emit( ToolResultEvent( name="view_driver_state", - result=libero_tools.view_driver_state(0, state=self._state), + result=libero_tools.view_driver_state(0, state=self._state), ) ) self._primitives = primitives def close(self) -> None: - """Flush the agent-side video buffer to disk (end-of-run). - """ - if self._video_path is None: - return + """Flush the agent-side video buffer through ``EnvState``.""" try: - self._primitives.stop_recording_and_save(self._video_path) + frames = self._primitives.stop_recording() + if frames: + self._state.save("episode.mp4", frames, step=None, fps=20) except Exception as e: - # The runner is in the cleanup path; never let a video save - # abort it. get_logger("libero_toolkit").warning( - f"failed to save video to {self._video_path}: {e}" + f"failed to save episode video: {e}" ) def write_recipe(self, recipe_tag: str) -> str: diff --git a/robots/libero/tools.py b/robots/libero/tools.py index 6b95f6cf..7f511e8e 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -1,13 +1,9 @@ """LIBERO + OpenPI tool implementation.""" from __future__ import annotations -import json -import os from collections.abc import Callable -from pathlib import Path from typing import Any -import imageio.v2 as imageio import numpy as np from robots.libero.env_client import LiberoEnvClient @@ -18,70 +14,6 @@ logger = get_logger("libero") -ARTIFACT_LAYOUT: dict[tuple[str | None, str | None, str], str] = { - ("agentview", "low", "policy_image"): "images/image_{step:02d}.png", - ("agentview", "low", "image"): "images_cam/image_cam_{step:02d}.png", - ("agentview", "low", "depth"): "depths/depth_{step:02d}.npy", - ("agentview", "low", "world"): "world/world_{step:02d}.npy", - ("agentview", "low", "metadata"): "camera_meta.json", - ("agentview", "high", "image"): "images_cam_hi/image_cam_hi_{step:02d}.png", - ("agentview", "high", "world"): "world_hi/world_hi_{step:02d}.npy", - ("wrist", "low", "image"): "images_wrist/image_wrist_{step:02d}.png", - ("wrist", "low", "depth"): "depths_wrist/depth_wrist_{step:02d}.npy", - ("wrist", "low", "world"): "world_wrist/world_wrist_{step:02d}.npy", - ("wrist", "low", "metadata"): "wrist_meta/wrist_meta_{step:02d}.json", - ("wrist", "high", "image"): "images_wrist_hi/image_wrist_hi_{step:02d}.png", - ("wrist", "high", "world"): "world_wrist_hi/world_wrist_hi_{step:02d}.npy", - (None, None, "states"): "states.json", - (None, None, "episode_video"): "episode.mp4", - (None, None, "segments"): "segments", - (None, None, "action_videos"): "action_videos", -} -ARTIFACT_DIRECTORIES: tuple[str, ...] = ( - "images", - "images_cam", - "depths", - "world", - "images_cam_hi", - "world_hi", - "images_wrist", - "depths_wrist", - "world_wrist", - "wrist_meta", - "images_wrist_hi", - "world_wrist_hi", - "segments", - "action_videos", -) - - -def artifact_path( - output_dir: str | os.PathLike[str], - kind: str, - *, - step: int | None = None, - camera: str | None = None, - resolution: str | None = None, -) -> Path: - """Resolve one artifact path from the shared layout. - - ``kind`` identifies the artifact; the remaining fields are optional - qualifiers and must be passed by keyword to avoid mixing them up. - """ - return Path(output_dir) / _artifact_relative_path(step, camera, resolution, kind) - - -def _artifact_relative_path( - step: int | None, - camera: str | None, - resolution: str | None, - kind: str, -) -> str: - template = ARTIFACT_LAYOUT[(camera, resolution, kind)] - if step is None and "{step" in template: - raise ValueError(f"{kind} artifact requires a step") - return template.format(step=step) - def _normalize_xyz(xyz): """Coerce an LLM-supplied xyz into a length-3 list[float].""" @@ -117,7 +49,7 @@ def __init__( self._last_obs_eef_z = None self._last_obs_gripper = None # Per-env-step frame buffer for diagnostic video rendering. - # Toggled via start_recording() / stop_recording_and_save(). + # Toggled via start_recording() / stop_recording(). self._recording = False self._frames = [] @@ -132,22 +64,14 @@ def record_frame(self, obs): def recorded_frame_count(self) -> int: return len(self._frames) - def stop_recording_and_save(self, path: str, fps: int = 20): - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - n = len(self._frames) - if n > 0: - imageio.mimwrite(path, self._frames, fps=fps) + def stop_recording(self) -> list[np.ndarray]: + frames = list(self._frames) self._recording = False self._frames = [] - return {"path": path, "n_frames": n} + return frames - def save_frame_slice(self, start: int, path: str, fps: int = 20): - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - frames = list(self._frames[int(start):]) - n = len(frames) - if n > 0: - imageio.mimwrite(path, frames, fps=fps) - return {"path": path, "n_frames": n, "fps": fps} + def frame_slice(self, start: int) -> list[np.ndarray]: + return list(self._frames[int(start):]) def set_obs(self, obs): self._last_obs = obs @@ -660,7 +584,7 @@ def segment( self, prompt: str = "", camera: str = "agentview", - step: int | None = None, + step: int = -1, point: list[int] | None = None, min_score: float = 0.2, *, @@ -672,9 +596,11 @@ def segment( artifacts. Errors are structured so the agent can continue with image inspection and ``back_project``. """ - nn = state.latest_step if step is None else int(step) - if nn is None: - return {"error": "no state entries; cannot select segment image"} + try: + record = state.get(step) + except Exception as exc: + return {"error": f"state step not available: {exc}"} + nn = record.step_idx camera = camera or "agentview" prompt = prompt.strip() @@ -683,26 +609,27 @@ def segment( if has_prompt == has_point: return {"error": "segment needs exactly one of prompt or point"} try: - image_path, world_path, artifact_pairs = _select_segment_artifacts( - state, nn, camera + image_name, world_name, artifact_pairs = _select_segment_artifacts( + state, record, camera ) except ValueError as e: return {"error": str(e)} - if image_path is None: + if image_name is None or world_name is None: return { "error": "complete segment artifacts not found", "step": nn, "camera": camera, - "checked_paths": [ - str(path) + "checked_artifacts": [ + name for image, world in artifact_pairs - for path in (image, world) + for name in (image, world) + if name ], } try: data = self._sam3_client.segment( - image_path, + state.load_bytes(image_name, step=nn), text_prompt=prompt if has_prompt else None, point=point, min_score=min_score, @@ -712,36 +639,41 @@ def segment( "error": str(e), "step": nn, "camera": camera, - "image_path": str(image_path), + "image_artifact": image_name, } except Exception as e: return { "error": f"segmentation service call failed: {e}", "step": nn, "camera": camera, - "image_path": str(image_path), + "image_artifact": image_name, "fallback": "Use manual visual localization and back_project.", } - out_dir = state.output_dir - segment_path, overlay_candidate_path, segment_index = ( - _next_segment_artifact_paths(out_dir, nn) - ) - overlay_path = None + segment_index = _next_segment_index(record) + segment_name = f"segment_{segment_index:02d}.json" + overlay_name = f"segment_overlay_{segment_index:02d}.png" + saved_overlay = None mask = data.mask if data.found and isinstance(mask, np.ndarray): - if world_path is None or not world_path.exists(): + try: + world_map = state.load(world_name, step=nn) + except Exception as exc: world_result = { "world_xyz": None, - "world_error": "world map artifact not found for selected image", - "expected_world_path": str(world_path) if world_path else None, + "world_error": f"world map artifact not available: {exc}", + "expected_world_artifact": world_name, } else: - world_result = _mask_to_world(mask, np.load(world_path)) - world_result["world_path"] = str(world_path) - overlay_path = overlay_candidate_path - if not _write_segment_overlay(image_path, mask, overlay_path): - overlay_path = None + world_result = _mask_to_world(mask, world_map) + world_result["world_artifact"] = world_name + overlay = _make_segment_overlay(state.load(image_name, step=nn), mask) + if overlay is not None and state.save( + overlay_name, + overlay, + step=nn, + ): + saved_overlay = overlay_name else: world_result = { "world_xyz": None, @@ -754,7 +686,7 @@ def segment( "camera": camera, "source_step": nn, "segment_index": segment_index, - "image_path": str(image_path), + "image_artifact": image_name, "min_score": min_score, "score": round(float(data.score), 3) if data.score is not None else None, "box": data.box, @@ -767,14 +699,18 @@ def segment( if not data.found: segment_blob["error"] = data.reason or "SAM3 found no mask" segment_blob.update(world_result) - segment_path.write_text(json.dumps(segment_blob, indent=2, default=str)) + state.save( + segment_name, + segment_blob, + step=nn, + ) result = { "found": data.found, "step": nn, "camera": camera, - "image_path": str(image_path), - "segment_path": str(segment_path), + "image_artifact": image_name, + "segment_artifact": segment_name, "score": segment_blob["score"], "box": segment_blob["box"], "world_xyz": segment_blob["world_xyz"], @@ -783,59 +719,57 @@ def segment( if "error" in segment_blob: result["error"] = segment_blob["error"] result["fallback"] = "Use manual visual localization and back_project." - if overlay_path is not None and overlay_path.exists(): - result["overlay_path"] = str(overlay_path) + if saved_overlay is not None: + result["overlay_artifact"] = saved_overlay + result["_image_bytes"] = state.load_bytes(saved_overlay, step=nn) return result def write_recipe_from_states(state: EnvState, recipe_tag: str) -> str: """Find a command sequence that gets ``libero_terminated=True``. - Export non-error LIBERO primitive commands from ``states.json`` and - successful segment calls from ``segments/segment_*.json``. + Export non-error LIBERO primitive commands and successful segment calls. """ command_events = [] for record in state.records(): command = record.command - if command is None: - continue - if command.get("action") not in PRIMITIVE_TOOL_NAMES: - continue result = record.result - if isinstance(result, dict) and result.get("error"): - continue - command_events.append(((record.step_idx, -1), command)) - - output_dir = state.output_dir - for artifact in artifact_path(output_dir, "segments").glob("segment_*.json"): - with artifact.open() as f: - segment = json.load(f) - if segment.get("error"): - continue - if segment["mode"] == "text": - command = { - "action": "segment", - "prompt": segment["prompt"], - "camera": segment["camera"], - } - else: - command = { - "action": "segment", - "point": segment["point"], - "camera": segment["camera"], - } - source_step = int(segment["source_step"]) - event_order = (source_step, int(segment["segment_index"])) - command_events.append((event_order, command)) + if ( + command is not None + and command.get("action") in PRIMITIVE_TOOL_NAMES + and not (isinstance(result, dict) and result.get("error")) + ): + command_events.append(((record.step_idx, -1), command)) + + for name in sorted(record.artifacts): + if not (name.startswith("segment_") and name.endswith(".json")): + continue + segment = state.load(name, step=record.step_idx) + if segment.get("error"): + continue + if segment["mode"] == "text": + segment_command = { + "action": "segment", + "prompt": segment["prompt"], + "camera": segment["camera"], + } + else: + segment_command = { + "action": "segment", + "point": segment["point"], + "camera": segment["camera"], + } + event_order = (record.step_idx, int(segment["segment_index"])) + command_events.append((event_order, segment_command)) - recipe_path = os.path.join(state.output_dir, f"recipe_{recipe_tag}.jsonl") - tmp_path = recipe_path + ".tmp" command_events.sort(key=lambda event: event[0]) - with open(tmp_path, "w") as f: - for _, command in command_events: - f.write(json.dumps(command, separators=(",", ":")) + "\n") - os.replace(tmp_path, recipe_path) - return recipe_path + recipe_name = f"recipe_{recipe_tag}.jsonl" + state.save( + recipe_name, + [command for _, command in command_events], + step=None, + ) + return recipe_name def _metric_depth(depth: Any, camera_meta: dict) -> np.ndarray: @@ -864,41 +798,13 @@ def _world_from_depth(depth_metric: np.ndarray, camera_meta: dict) -> np.ndarray return (camera_points @ extrinsic.T)[..., :3] -def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, - log: dict | None = None) -> StepRecord: - """Dump state snapshot, images, and depth for step *step_idx*. - - Writes: - - ``/images/image_NN.png`` (Pi0-frame agentview) - - ``/images_cam/image_cam_NN.png`` (calibration-frame agentview) - - ``/depths/depth_NN.npy`` (metric depth, meters) - - ``/world/world_NN.npy`` (agentview world xyz map) - - ``/images_wrist/image_wrist_NN.png`` - - ``/depths_wrist/depth_wrist_NN.npy`` - - ``/world_wrist/world_wrist_NN.npy`` - - ``/wrist_meta/wrist_meta_NN.json`` - - high-res ``images_cam_hi`` / ``world_hi`` artifacts - - high-res ``images_wrist_hi`` / ``world_wrist_hi`` artifacts - - ``/camera_meta.json`` (static, once) - - appends the step blob to ``/states.json`` - - If *log* is provided (the return value of :func:`execute`), its - ``command``, ``result``, and ``elapsed_s`` fields are merged into the - step blob so a single entry captures everything. - """ - output_dir = env_state.output_dir - for directory in ARTIFACT_DIRECTORIES: - (Path(output_dir) / directory).mkdir(parents=True, exist_ok=True) - - agent_world_map = None - wrist_world_map = None - agent_world_map_hi = None - wrist_world_map_hi = None - wrist_meta = None - # Reuse one raw observation snapshot for state and per-step artifacts. +def dump_state( + primitives: LiberoPrimitives, + env_state: EnvState, + log: dict | None = None, +) -> StepRecord: + """Save one Libero observation through its owned state record.""" raw = primitives.env.raw_obs() - # Expose robot proprioception and object names, but never privileged - # object coordinates; the agent must localize through visual artifacts. state = { "robot0_eef_pos": [float(x) for x in raw["robot0_eef_pos"]], "robot0_eef_quat": [float(x) for x in raw["robot0_eef_quat"]], @@ -909,9 +815,32 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, if k.endswith("_pos") and "robot0" not in k and "to_robot" not in k ), } - imageio.imwrite( - artifact_path(output_dir, "policy_image", step=step_idx, camera="agentview", resolution="low"), + log = log or {} + with env_state.record_step( + state=state, + command=log.get("command"), + result=log.get("result"), + elapsed_s=log.get("elapsed_s"), + extras={ + "libero_terminated": primitives.env.episode_terminated, + "episode_truncated": primitives.env.episode_truncated, + "task_language": primitives.env.get_task_language(), + }, + ) as step_idx: + _save_observation_artifacts(primitives, env_state, step_idx, raw) + return env_state.get(step_idx) + + +def _save_observation_artifacts( + primitives: LiberoPrimitives, + env_state: EnvState, + step_idx: int, + raw: dict[str, Any], +) -> None: + env_state.save( + "agentview_policy.png", primitives._last_obs["main_images"], + step=step_idx, ) # --- camera calibration (static for agentview): fetch metadata as needed --- @@ -920,8 +849,7 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, height=256, width=256, ) or {} - camera_meta_path = artifact_path(output_dir, "metadata", camera="agentview", resolution="low") - if agentview_meta and not camera_meta_path.exists(): + if agentview_meta: cam_meta_out = dict(agentview_meta) cam_meta_out["projection"] = ( "Prefer the back_project(row, col, step=NN) MCP tool; it " @@ -930,24 +858,30 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, "256x256 calibration-frame image." ) cam_meta_out["note"] = ( - "depth_NN.npy is in this camera frame (vertical-flipped raw " - "buffer). image_NN.png is rotated 180deg (Pi0 convention) and " - "is NOT in the same frame as depth/K." + "The agentview_depth.npy observation is aligned with agentview.png. " + "agentview_policy.png uses the Pi0 orientation and must not supply " + "pixels for back-projection." + ) + env_state.save( + "agentview_metadata.json", + cam_meta_out, + step=step_idx, ) - with open(camera_meta_path, "w") as f: - json.dump(cam_meta_out, f, indent=2) # --- per-step RGB in the depth/K frame (vertical-flip of the raw buffer) --- - # The agent picks object pixels HERE (same frame as depth_NN.npy + K), so - # pixel -> depth -> back-project is direct. (image_NN.png is the 180°-rotated - # Pi0-convention frame and must NOT be used for back-projection.) + # Agentview pixels align with the matching depth and calibration. The policy + # image uses Pi0 orientation and must not supply back-projection pixels. try: ci = raw.get("agentview_image") if ci is not None: ci = np.asarray(ci) if ci.dtype != np.uint8: ci = ci.astype(np.uint8) - imageio.imwrite(artifact_path(output_dir, "image", step=step_idx, camera="agentview", resolution="low"), ci[::-1]) + env_state.save( + "agentview.png", + ci[::-1], + step=step_idx, + ) except Exception as e: logger.warning("image_cam dump failed: %s", e) @@ -960,21 +894,19 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, # depth map in this frame. VERIFIED 5/5: projecting each GT object # world pos via M lands on a pixel whose depth_flip[row,col] matches # the object's surface depth (plate Δ6mm, cookies Δ14mm). So - # pixel(row,col) in depth_NN.npy back-projects correctly with - # camera_meta.json (NOT the same frame as the 180°-rotated - # image_NN.png — see camera_meta note). + # Pixels in agentview.png align with agentview_depth.npy and the + # per-step agentview metadata saved with the record artifacts. d = _metric_depth(d, agentview_meta)[::-1] - np.save( - artifact_path(output_dir, "depth", step=step_idx, camera="agentview", resolution="low"), + env_state.save( + "agentview_depth.npy", d.astype(np.float32), + step=step_idx, ) world = _world_from_depth(d, agentview_meta).astype(np.float32) - np.save( - artifact_path(output_dir, "world", step=step_idx, camera="agentview", resolution="low"), + env_state.save( + "agentview_world.npy", world, - ) - agent_world_map = _artifact_relative_path( - step_idx, "agentview", "low", "world" + step=step_idx, ) except Exception as e: logger.warning("depth dump failed: %s", e) @@ -988,7 +920,11 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, wimg = np.asarray(wimg) if wimg.dtype != np.uint8: wimg = wimg.astype(np.uint8) - imageio.imwrite(artifact_path(output_dir, "image", step=step_idx, camera="wrist", resolution="low"), wimg[::-1]) + env_state.save( + "wrist.png", + wimg[::-1], + step=step_idx, + ) except Exception as e: logger.warning("wrist image dump failed: %s", e) @@ -1008,32 +944,30 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, logger.warning("wrist camera meta missing; skipping wrist depth/world") else: wdpt_metric = _metric_depth(wdpt_arr, wmeta)[::-1] - np.save( - artifact_path(output_dir, "depth", step=step_idx, camera="wrist", resolution="low"), + env_state.save( + "wrist_depth.npy", wdpt_metric.astype(np.float32), + step=step_idx, ) world_w = _world_from_depth(wdpt_metric, wmeta).astype(np.float32) - np.save( - artifact_path(output_dir, "world", step=step_idx, camera="wrist", resolution="low"), + env_state.save( + "wrist_world.npy", world_w, - ) - wrist_world_map = _artifact_relative_path( - step_idx, "wrist", "low", "world" + step=step_idx, ) wmeta_out = dict(wmeta) wmeta_out["note"] = ( "MOVING camera: extrinsic_cam2world is for THIS step " - "only. world_wrist_NN.npy[row,col] gives world " - "(x,y,z) for that pixel, in the SAME world frame as " - "agentview world_NN.npy." + "only. The matching wrist world-map observation gives world " + "(x,y,z) for that pixel in the same world frame as the " + "agentview world-map artifact." + ) + env_state.save( + "wrist_metadata.json", + wmeta_out, + step=step_idx, ) - with open( - artifact_path(output_dir, "metadata", step=step_idx, camera="wrist", resolution="low"), - "w", - ) as f: - json.dump(wmeta_out, f, indent=2) - wrist_meta = wmeta except Exception as e: logger.warning("wrist depth/world dump failed: %s", e) @@ -1047,20 +981,19 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, meta_hi = primitives.env.get_camera_meta("agentview", 1024, 1024) if meta_hi is None: raise RuntimeError("agentview camera metadata missing") - imageio.imwrite( - artifact_path(output_dir, "image", step=step_idx, camera="agentview", resolution="high"), + env_state.save( + "agentview_high.png", np.asarray(rgb_hi)[::-1], + step=step_idx, ) world_hi = _world_from_depth( _metric_depth(depth_hi, meta_hi)[::-1], meta_hi, ).astype(np.float16) - np.save( - artifact_path(output_dir, "world", step=step_idx, camera="agentview", resolution="high"), + env_state.save( + "agentview_world_high.npy", world_hi, - ) - agent_world_map_hi = _artifact_relative_path( - step_idx, "agentview", "high", "world" + step=step_idx, ) except Exception as e: logger.warning("agentview high-res dump failed: %s", e) @@ -1077,70 +1010,23 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, ) if meta_wrist_hi is None: raise RuntimeError("robot0_eye_in_hand camera metadata missing") - imageio.imwrite( - artifact_path(output_dir, "image", step=step_idx, camera="wrist", resolution="high"), + env_state.save( + "wrist_high.png", np.asarray(rgb_wrist_hi)[::-1], + step=step_idx, ) world_wrist_hi = _world_from_depth( _metric_depth(depth_wrist_hi, meta_wrist_hi)[::-1], meta_wrist_hi, ).astype(np.float16) - np.save( - artifact_path(output_dir, "world", step=step_idx, camera="wrist", resolution="high"), + env_state.save( + "wrist_world_high.npy", world_wrist_hi, - ) - wrist_world_map_hi = _artifact_relative_path( - step_idx, "wrist", "high", "world" + step=step_idx, ) except Exception as e: logger.warning("wrist high-res dump failed: %s", e) - for old_step in range(max(0, int(step_idx) - 4)): - for path in ( - artifact_path(output_dir, "image", step=old_step, camera="agentview", resolution="high"), - artifact_path(output_dir, "world", step=old_step, camera="agentview", resolution="high"), - artifact_path(output_dir, "image", step=old_step, camera="wrist", resolution="high"), - artifact_path(output_dir, "world", step=old_step, camera="wrist", resolution="high"), - ): - try: - os.unlink(path) - except FileNotFoundError: - pass - - frame_streams = [ - stream - for stream in ("image", "image_cam", "image_wrist") - if env_state.artifact_path(step_idx, stream).exists() - ] - depth_streams = [ - stream - for stream in ("depth", "depth_wrist") - if env_state.artifact_path(step_idx, stream).exists() - ] - camera_meta = {"agentview": agentview_meta} - if wrist_meta is not None: - camera_meta["wrist"] = wrist_meta - record = StepRecord( - step_idx=step_idx, - state=state, - frames=frame_streams, - depth=depth_streams, - camera_meta=camera_meta, - command=log.get("command") if log else None, - result=log.get("result") if log else None, - elapsed_s=log.get("elapsed_s") if log else None, - extras={ - "libero_terminated": primitives.env.episode_terminated, - "episode_truncated": primitives.env.episode_truncated, - "task_language": primitives.env.get_task_language(), - "world_map": agent_world_map, - "wrist_world_map": wrist_world_map, - "world_map_hi": agent_world_map_hi, - "wrist_world_map_hi": wrist_world_map_hi, - }, - ) - return env_state.append(record) - # --------------------------------------------------------------------------- # Tool schema declarations (Anthropic-shaped canonical schema) @@ -1161,16 +1047,9 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, { "name": "view_driver_state", "description": ( - "Read step NN from `states.json` + the matching " - "state images in {{output_dir}}. If step is " - "null, returns the latest entry. Each entry contains the robot " - "state, libero_terminated flag, command log, and result. Returns " - "available PNG paths in this stable " - "order: 1) `images/image_NN.png` (Pi0-frame agentview), " - "2) `images_cam/image_cam_NN.png` (calibration-frame agentview), " - "3) `images_wrist/image_wrist_NN.png` (calibration-frame wrist). " - "High-resolution calibration-frame images are returned as file " - "paths, not embedded as image bytes. " + "Read one recorded state and its observation artifacts. Step -1 " + "selects the latest entry. Embeds policy, agentview, and wrist " + "images when available. " "Use the calibration-frame images for pixel back-projection; JSON " "state alone is not enough. Use agentview for global tabletop " "layout and object locations; use wrist for close-range details " @@ -1180,8 +1059,9 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, "type": "object", "properties": { "step": { - "type": ["integer", "null"], - "description": "Step number; 0 = initial. Null = latest.", + "type": "integer", + "default": -1, + "description": "Step number; 0 = initial, -1 = latest.", }, }, }, @@ -1372,9 +1252,7 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, { "name": "view_camera_meta", "description": ( - "Read camera calibration metadata from the output dir. " - "camera='agentview' reads static camera_meta.json. " - "camera='wrist' reads the per-step wrist metadata." + "Read per-step camera calibration metadata from recorded artifacts." ), "input_schema": { "type": "object", @@ -1385,8 +1263,9 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, "description": "Camera metadata to read (default agentview).", }, "step": { - "type": ["integer", "null"], - "description": "Wrist metadata step to use (default latest).", + "type": "integer", + "default": -1, + "description": "Metadata step to use; -1 = latest.", }, }, }, @@ -1412,8 +1291,9 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, "description": "Artifact camera to use (default agentview).", }, "step": { - "type": ["integer", "null"], - "description": "Step NN to segment; null = latest.", + "type": "integer", + "default": -1, + "description": "Step to segment; -1 = latest.", }, "point": { "type": ["array", "null"], @@ -1439,7 +1319,7 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, "selected camera's precomputed world map. Row 0 = top of image, " "col 0 = left. Returns world_xyz in meters.\n\n" "USE THIS to find where an object is in the world — look at " - "the high-resolution paths returned by view_driver_state " + "the embedded high-resolution image returned by view_driver_state " "to pick a pixel on the target object, then call back_project. " "The default resolution is high (1024x1024). Pass " "resolution='low' only for pixels from the embedded/standard " @@ -1468,8 +1348,9 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, "description": "Pixel column (0=left) in the selected resolution image.", }, "step": { - "type": ["integer", "null"], - "description": "Depth/world-map step to use (default latest). 0 for initial.", + "type": "integer", + "default": -1, + "description": "Depth/world-map step; 0 = initial, -1 = latest.", }, "camera": { "type": "string", @@ -1509,128 +1390,68 @@ def dump_state(primitives: LiberoPrimitives, env_state: EnvState, step_idx: int, ] -def _load_image_path(state: EnvState, nn: int, kind: str) -> str | None: - """Return the path to a dumped state image. None if not present.""" - out_dir = state.output_dir - if kind == "agent": - path = artifact_path(out_dir, "policy_image", step=nn, camera="agentview", resolution="low") - elif kind == "camera": - path = artifact_path(out_dir, "image", step=nn, camera="agentview", resolution="low") - elif kind == "wrist": - path = artifact_path(out_dir, "image", step=nn, camera="wrist", resolution="low") - else: - raise ValueError(f"unknown image kind: {kind}") - if not path.exists(): - return None - return str(path) - - -def _load_camera_meta( - state: EnvState, - camera: str = "agentview", - nn: int | None = None, -) -> dict: - out_dir = state.output_dir - if camera == "agentview": - path = artifact_path(out_dir, "metadata", camera="agentview", resolution="low") - elif camera == "wrist" and nn is not None: - path = artifact_path(out_dir, "metadata", step=nn, camera="wrist", resolution="low") - else: - raise ValueError("camera must be 'agentview' or 'wrist' with nn") - if not path.exists(): - raise FileNotFoundError(f"{path.name} not found in {state.output_dir}") - with open(path) as f: - return json.load(f) - - -def _load_depth(state: EnvState, camera: str, nn: int) -> np.ndarray: - if camera == "agentview": - stream = "depth" - elif camera == "wrist": - stream = "depth_wrist" - else: - raise ValueError("camera must be 'agentview' or 'wrist'") - depth = state.load_depth(nn, stream) - if depth.ndim == 3: - depth = depth[..., 0] - return depth - - -def view_driver_state(step: int | None = None, *, state: EnvState) -> dict: - latest = state.latest_step - if latest is None: - return {"error": "no state entries; env not ready"} - nn = latest if step is None else int(step) +def view_driver_state(step: int = -1, *, state: EnvState) -> dict: try: - record = state.get(nn) - except Exception as e: - return {"error": f"step {nn} not present in state trace: {e}"} + record = state.get(step) + except Exception as exc: + return {"error": f"state step not available: {exc}"} + nn = record.step_idx extras = record.extras - out: dict = {"step": nn, "state": record.state} + out: dict = { + "step": nn, + "state": record.state, + "artifacts": sorted(record.artifacts), + } out["task_language"] = extras.get("task_language") out["libero_terminated"] = extras.get("libero_terminated") out["episode_truncated"] = extras.get("episode_truncated") - out["world_map"] = extras.get("world_map") - out["wrist_world_map"] = extras.get("wrist_world_map") - out["world_map_hi"] = extras.get("world_map_hi") - out["wrist_world_map_hi"] = extras.get("wrist_world_map_hi") out["log"] = { "command": record.command, "result": record.result, "elapsed_s": record.elapsed_s, } - for field, kind in ( - ("image_path", "agent"), - ("image_cam_path", "camera"), - ("image_wrist_path", "wrist"), - ): - image_path = _load_image_path(state, nn, kind) - if image_path: - out[field] = image_path - for field, camera in ( - ("image_cam_hi_path", "agentview"), - ("image_wrist_hi_path", "wrist"), + for slot, names in ( + ("_image_bytes", ("agentview_policy.png",)), + ("_image_cam_bytes", ("agentview_high.png", "agentview.png")), + ("_image_wrist_bytes", ("wrist_high.png", "wrist.png")), ): - image_path = artifact_path( - state.output_dir, - "image", - step=nn, - camera=camera, - resolution="high", - ) - if image_path.exists(): - out[field] = str(image_path) + name = next((name for name in names if name in record.artifacts), None) + if name: + try: + out[slot] = state.load_bytes(name, step=nn) + except FileNotFoundError: + pass return out -def _select_segment_artifacts(state: EnvState, nn: int, camera: str): - out_dir = state.output_dir +def _select_segment_artifacts( + state: EnvState, + record: StepRecord, + camera: str, +) -> tuple[str | None, str | None, list[tuple[str | None, str | None]]]: if camera not in ("agentview", "wrist"): raise ValueError(f"unknown segment camera: {camera}") pairs = [ - ( - artifact_path(out_dir, "image", step=nn, camera=camera, resolution=resolution), - artifact_path(out_dir, "world", step=nn, camera=camera, resolution=resolution), - ) - for resolution in ("high", "low") + (f"{camera}_high.png", f"{camera}_world_high.npy"), + (f"{camera}.png", f"{camera}_world.npy"), ] - for image_path, world_path in pairs: - if image_path.exists() and world_path.exists(): - return image_path, world_path, pairs + for image_name, world_name in pairs: + if ( + image_name in record.artifacts + and world_name in record.artifacts + and state.exists(image_name, step=record.step_idx) + and state.exists(world_name, step=record.step_idx) + ): + return image_name, world_name, pairs return None, None, pairs -def _next_segment_artifact_paths(out_dir: Path, nn: int): - segments_dir = artifact_path(out_dir, "segments") - segments_dir.mkdir(parents=True, exist_ok=True) +def _next_segment_index(record: StepRecord) -> int: idx = 0 - while True: - segment_path = segments_dir / f"segment_{nn:02d}_{idx:02d}.json" - overlay_path = segments_dir / f"segment_overlay_{nn:02d}_{idx:02d}.png" - if not segment_path.exists() and not overlay_path.exists(): - return segment_path, overlay_path, idx + while f"segment_{idx:02d}.json" in record.artifacts: idx += 1 + return idx def _mask_to_world(mask: np.ndarray, world_map: np.ndarray, @@ -1687,28 +1508,25 @@ def _mask_to_world(mask: np.ndarray, world_map: np.ndarray, return result -def _write_segment_overlay(image_path: Path, mask: np.ndarray, - overlay_path: Path) -> bool: - try: - image = imageio.imread(image_path) - if image.ndim != 3 or image.shape[:2] != mask.shape: - return False - overlay = image.copy() - red = np.zeros_like(overlay) - red[..., 0] = 255 - overlay[mask] = ( - 0.55 * overlay[mask].astype(np.float32) - + 0.45 * red[mask].astype(np.float32) - ).astype(np.uint8) - imageio.imwrite(overlay_path, overlay) - return overlay_path.exists() - except Exception: - return False +def _make_segment_overlay( + image: np.ndarray, + mask: np.ndarray, +) -> np.ndarray | None: + if image.ndim != 3 or image.shape[:2] != mask.shape: + return None + overlay = image.copy() + red = np.zeros_like(overlay) + red[..., 0] = 255 + overlay[mask] = ( + 0.55 * overlay[mask].astype(np.float32) + + 0.45 * red[mask].astype(np.float32) + ).astype(np.uint8) + return overlay def view_camera_meta( camera: str = "agentview", - step: int | None = None, + step: int = -1, *, state: EnvState, ) -> dict: @@ -1716,26 +1534,24 @@ def view_camera_meta( if camera not in ("agentview", "wrist"): return {"error": f"bad camera '{camera}' (use 'agentview' or 'wrist')"} - nn = None - if camera == "wrist": - nn = state.latest_step if step is None else int(step) - if nn is None: - return {"error": "no wrist metadata available"} - try: - meta = _load_camera_meta(state, camera, nn) + record = state.get(step) + metadata_name = f"{camera}_metadata.json" + if metadata_name not in record.artifacts: + raise FileNotFoundError(metadata_name) + meta = state.load(metadata_name, step=record.step_idx) except Exception as e: return {"error": f"{camera} camera metadata not found: {e}"} if camera == "agentview": return {"camera": "agentview", "camera_meta": meta} - return {"camera": "wrist", "step": nn, "camera_meta": meta} + return {"camera": "wrist", "step": record.step_idx, "camera_meta": meta} def back_project( row: int | None = None, col: int | None = None, - step: int | None = None, + step: int = -1, camera: str = "agentview", resolution: str = "high", row_range: list | None = None, @@ -1760,23 +1576,16 @@ def back_project( ) } - nn = state.latest_step if step is None else int(step) - if nn is None: - return {"error": "no depth/world-map files available"} - try: - record = state.get(nn) + record = state.get(step) except Exception as e: - return {"error": f"step {nn} not present in state trace: {e}"} + return {"error": f"state step not available: {e}"} + nn = record.step_idx - if camera == "agentview": - hi_artifact = record.extras.get("world_map_hi") - low_artifact = record.extras.get("world_map") - else: - hi_artifact = record.extras.get("wrist_world_map_hi") - low_artifact = record.extras.get("wrist_world_map") + hi_artifact = f"{camera}_world_high.npy" + low_artifact = f"{camera}_world.npy" source_artifact = hi_artifact if resolution == "high" else low_artifact - if not source_artifact: + if source_artifact not in record.artifacts: return { "error": ( f"{camera} {resolution}-resolution world map not recorded " @@ -1785,7 +1594,7 @@ def back_project( } try: - world_map = np.load(state.output_dir / source_artifact) + world_map = state.load(str(source_artifact), step=nn) except Exception as e: return { "error": ( @@ -1871,7 +1680,12 @@ def back_project( depth_m = None if source_artifact == low_artifact: try: - depth = _load_depth(state, camera, nn) + depth_artifact = f"{camera}_depth.npy" + if depth_artifact not in record.artifacts: + raise FileNotFoundError(depth_artifact) + depth = state.load(depth_artifact, step=nn) + if depth.ndim == 3: + depth = depth[..., 0] except Exception as e: return {"error": f"{camera} depth not found for step {nn}: {e}"} depth_m = float(depth[row, col]) diff --git a/rpent/cli/main.py b/rpent/cli/main.py index 889ea17b..a22bbc30 100644 --- a/rpent/cli/main.py +++ b/rpent/cli/main.py @@ -220,7 +220,6 @@ def main() -> int: toolkit = get_toolkit( env_name, primitives_kwargs=primitives_kwargs, - video_path=str(Path(output_dir) / "episode.mp4"), dashboard_events=dashboard_events, ) diff --git a/rpent/dashboard/server.py b/rpent/dashboard/server.py index d6a3bd0f..060f8603 100644 --- a/rpent/dashboard/server.py +++ b/rpent/dashboard/server.py @@ -257,10 +257,11 @@ def api_frame( @app.get("/api/run/video") def api_video(run: str) -> Response: live = self._resolve(run) - if live is None or not live.has_video(): + video = live.video() if live else None + if video is None: return Response(status_code=404) - return FileResponse( - live.video_path, + return Response( + video, media_type="video/mp4", headers={"Cache-Control": "no-store, max-age=0"}, ) @@ -268,11 +269,11 @@ def api_video(run: str) -> Response: @app.get("/api/run/action-video") def api_action_video(run: str, step: int) -> Response: live = self._resolve(run) - path = live.action_video_path(step) if live else None - if path is None: + video = live.action_video(step) if live else None + if video is None: return Response(status_code=404) - return FileResponse( - path, + return Response( + video, media_type="video/mp4", headers={"Cache-Control": "no-store, max-age=0"}, ) diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py index ae495829..d4075cfa 100644 --- a/rpent/dashboard/state.py +++ b/rpent/dashboard/state.py @@ -572,7 +572,14 @@ def _apply_tool_result(self, event: ToolResultEvent) -> None: result = event.result if not isinstance(result, dict): return - self._apply_frame_paths(result) + frames = { + "camera": result.get("_image_cam_bytes") or result.get("_image_bytes"), + "wrist": result.get("_image_wrist_bytes"), + } + self._update_frames( + step=result.get("step"), + frames={kind: data for kind, data in frames.items() if data}, + ) log = result.get("log") if not isinstance(log, dict): return @@ -593,6 +600,7 @@ def _apply_tool_result(self, event: ToolResultEvent) -> None: action=action, ) action_video = str(action_video_path) if action_video_path is not None else None + action_video_artifact = result.get("action_video_artifact") item = { "step": step, "action": action, @@ -600,8 +608,9 @@ def _apply_tool_result(self, event: ToolResultEvent) -> None: "result": log.get("result"), "elapsed_s": log.get("elapsed_s"), "terminated": terminated, - "has_action_video": action_video_path is not None, "action_video_path": action_video, + "action_video_artifact": action_video_artifact, + "has_action_video": bool(action_video_artifact or action_video_path), } with self._lock: self._timeline.append(item) @@ -773,18 +782,25 @@ def frame(self, kind: str) -> bytes | None: with self._lock: return self._frames.get(kind) - def action_video_path(self, step: int) -> Path | None: + def action_video(self, step: int) -> bytes | None: with self._lock: for item in self._timeline: if int(item.get("step", -1)) != int(step): continue + artifact = item.get("action_video_artifact") + if artifact: + video_path = self.output_dir / f"{int(step):02d}_{artifact}" + return video_path.read_bytes() if video_path.exists() else None raw_path = item.get("action_video_path") if not raw_path: return None video_path = Path(raw_path) - return video_path if video_path.exists() else None + return video_path.read_bytes() if video_path.exists() else None return None + def video(self) -> bytes | None: + return self.video_path.read_bytes() if self.video_path.exists() else None + def has_video(self) -> bool: with self._lock: return self._task_state in TERMINAL_RUN_STATES and self.video_path.exists() diff --git a/rpent/planner/api_loop.py b/rpent/planner/api_loop.py index f7a9c106..71890c94 100644 --- a/rpent/planner/api_loop.py +++ b/rpent/planner/api_loop.py @@ -707,7 +707,11 @@ def _build_tools(toolkit: Toolkit, *, no_images: bool = False) -> list[Tool]: def read_image(path: str) -> ToolReturn: - """Read a local image path returned by an RPent tool as visual input.""" + """Read an explicitly provided local image file as visual input. + + Environment observations are already embedded by ``view_driver_state``; + this helper is only for other user-selected local files. + """ return ToolReturn( return_value=path, content=[BinaryContent.from_path(path)], diff --git a/rpent/tools/state.py b/rpent/tools/state.py index 573fde6c..a24d15d9 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -1,44 +1,15 @@ -"""EnvState: the per-run state-trace owner. - -A **step** is one motion-primitive tool call that produced a dumped state -snapshot, indexed from ``0`` where ``0`` is the post-``reset()`` baseline -(no command). Non-motion tool calls (``get_ee_pose``, ``back_project``, -``view_driver_state``, file/memory tools) are NOT steps and leave the trace -untouched. - -``EnvState`` replaces the per-env ``_append_state`` / ``_load_states`` / -``_latest_step`` / ``_load_step`` / ``_load_image`` / ``_load_depth`` free -functions (and the toolkit's ``_next_step`` counter) with one explicit owner -constructed with an ``output_dir`` (no process-global). The toolkit and the -reader tools (``view_driver_state``, ``back_project``) hold a non-owning -reference to one ``EnvState`` per run; the composition root owns its lifecycle. - -Artefact naming follows the LIBERO layout. Each saved artefact is a named -*stream* turned into a path by a fixed rule:: - - stream "image" -> images/image_NN.png - stream "image_wrist" -> images_wrist/image_wrist_NN.png - stream "depth" -> depths/depth_NN.npy - stream "depth_wrist" -> depths_wrist/depth_wrist_NN.npy - stream "world" -> world/world_NN.npy (libero) - stream "wrist_meta" -> wrist_meta/wrist_meta_NN.json - -i.e. the stream name is the file prefix; the directory pluralises the -artefact type (``image``->``images``, ``depth``->``depths``; ``world`` and -``wrist_meta`` stay singular, matching libero) and appends the camera suffix. -Franka maps scene->``image``/``depth`` and wrist->``image_wrist``/ -``depth_wrist``; lerobot maps scene->``image``/``depth`` and -arm->``image_arm``. No per-env layout class is needed: the env's thin -``dump_state`` wrapper just picks the stream names. -""" +"""Per-run environment state and artifact storage.""" from __future__ import annotations +import copy +import fnmatch import json import os -import shutil +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Iterable +from typing import Any import imageio.v2 as imageio import numpy as np @@ -47,82 +18,47 @@ logger = get_logger("env_state") -# Artefact type (the leading segment of a stream name) -> file extension. -_ARTIFACT_EXT: dict[str, str] = { - "image": ".png", - "depth": ".npy", - "world": ".npy", - "wrist_meta": ".json", -} - -# Artefact type -> directory name (plural where libero pluralises). -_PLURAL: dict[str, str] = { - "image": "images", - "depth": "depths", -} - -def _split_stream(stream: str) -> tuple[str, str]: - for artifact_type in sorted(_ARTIFACT_EXT, key=len, reverse=True): - if stream == artifact_type: - return artifact_type, "" - prefix = artifact_type + "_" - if stream.startswith(prefix): - return artifact_type, stream[len(prefix):] - art, sep, suffix = stream.partition("_") - return art, (suffix if sep else "") - - -def stream_dir(output_dir: Path, stream: str) -> Path: - """Directory for a stream's artefacts (e.g. ``images_wrist``).""" - art, suffix = _split_stream(stream) - base = _PLURAL.get(art, art) - return output_dir / (base if not suffix else f"{base}_{suffix}") - - -def stream_path(output_dir: Path, stream: str, step_idx: int, ext: str | None = None) -> Path: - """Full path for one stream artefact at ``step_idx``.""" - art, _ = _split_stream(stream) - if ext is None: - ext = _ARTIFACT_EXT.get(art, ".bin") - return stream_dir(output_dir, stream) / f"{stream}_{step_idx:02d}{ext}" - - -# --------------------------------------------------------------------------- -# StepRecord -# --------------------------------------------------------------------------- +_MANIFEST_NAME = "states.json" +_MANIFEST_VERSION = 2 +_IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg"} +_TEXT_SUFFIXES = {".txt", ".md"} +_SUPPORTED_SUFFIXES = _IMAGE_SUFFIXES | { + ".npy", + ".json", + ".jsonl", + ".mp4", + ".bin", +} | _TEXT_SUFFIXES + + +def _json_default(value: Any) -> Any: + if isinstance(value, np.ndarray): + return value.tolist() + if isinstance(value, np.generic): + return value.item() + if isinstance(value, Path): + return str(value) + raise TypeError(f"{type(value).__name__} is not JSON serializable") @dataclass class StepRecord: - """One dumped step: the unit the LLM reads back via ``view_driver_state``. - - ``step_idx`` is the motion-primitive index (0 = post-reset baseline, which - has ``command``/``result``/``elapsed_s`` = None). ``frames``/``depth`` are - the stream names saved for this step (e.g. ``["image", "image_wrist"]``). - ``camera_meta`` is the per-camera metadata dict at capture time (intrinsics - + extrinsics + calibration status). ``extras`` absorbs env-specific fields - (libero's world maps, ``libero_terminated``, task_language, ...). - """ + """One motion step and the artifact base names captured for it.""" step_idx: int - state: dict + state: dict[str, Any] + artifacts: set[str] = field(default_factory=set) command: dict | None = None result: dict | None = None elapsed_s: float | None = None - frames: list[str] = field(default_factory=list) - depth: list[str] = field(default_factory=list) - camera_meta: dict | None = None extras: dict[str, Any] = field(default_factory=dict) def to_blob(self) -> dict[str, Any]: blob: dict[str, Any] = { "step_idx": self.step_idx, "state": self.state, - "frames": self.frames, - "depth": self.depth, + "artifacts": sorted(self.artifacts), } - if self.camera_meta is not None: - blob["camera_meta"] = self.camera_meta if self.command is not None: blob["command"] = self.command if self.result is not None: @@ -134,215 +70,382 @@ def to_blob(self) -> dict[str, Any]: return blob @classmethod - def from_blob(cls, blob: dict) -> "StepRecord": + def from_blob(cls, blob: dict[str, Any]) -> "StepRecord": return cls( - step_idx=int(blob.get("step_idx", -1)), - state=blob.get("state", {}), + step_idx=int(blob["step_idx"]), + state=dict(blob.get("state") or {}), + artifacts={str(name) for name in blob.get("artifacts") or []}, command=blob.get("command"), result=blob.get("result"), elapsed_s=blob.get("elapsed_s"), - frames=list(blob.get("frames", [])), - depth=list(blob.get("depth", [])), - camera_meta=blob.get("camera_meta"), extras=dict(blob.get("extras") or {}), ) -# --------------------------------------------------------------------------- -# EnvState -# --------------------------------------------------------------------------- - - class EnvState: - """Owns the on-disk state trace for one run. - - Constructed with an explicit ``output_dir`` (the composition root passes - it; ``EnvState`` never reaches into a process-global). Owns the step - counter, the ``states.json`` trace (atomic append), and the image/depth - artefact layout. The toolkit and the reader tools (``view_driver_state``, - ``back_project``) hold a non-owning reference to one ``EnvState`` per run. - """ + """Own a run's step trace and all state-related files in its output root.""" def __init__(self, output_dir: Path | str): - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - self._steps: list[dict] = self._read_states() - self._next_step = self._derive_next_step() - - # -- internal: states.json I/O ---------------------------------------- - - def _states_path(self) -> Path: - return self.output_dir / "states.json" + self._output_dir = Path(output_dir) + self._output_dir.mkdir(parents=True, exist_ok=True) + self._steps: list[StepRecord] = [] + self._pending_step: StepRecord | None = None + self._run_artifacts: set[str] = set() + self._next_step = 0 + self.reset() + + # -- private file resolution ----------------------------------------- + + @staticmethod + def _validate_name(name: str) -> str: + if not isinstance(name, str) or not name: + raise ValueError("artifact name must be a non-empty string") + path = Path(name) + if path.name != name or path.is_absolute() or name in {".", ".."}: + raise ValueError(f"artifact name must be a base filename: {name!r}") + if name == _MANIFEST_NAME: + raise ValueError(f"{_MANIFEST_NAME!r} is reserved") + if path.suffix.lower() not in _SUPPORTED_SUFFIXES: + raise ValueError(f"unsupported artifact suffix: {path.suffix or ''}") + return name + + def _artifact_file(self, name: str, step: int | None) -> Path: + name = self._validate_name(name) + if step is None: + return self._output_dir / name + if step < 0: + raise ValueError("artifact writes require a nonnegative step") + return self._output_dir / f"{step:02d}_{name}" + + def _manifest_file(self) -> Path: + return self._output_dir / _MANIFEST_NAME + + def _temporary_file(self, destination: Path) -> Path: + return destination.with_name( + f".{destination.stem}.tmp{destination.suffix}" + ) - def _read_states(self) -> list[dict]: - path = self._states_path() - if not path.exists(): - return [] + def _record_for_write(self, step: int) -> tuple[StepRecord, bool]: + if self._pending_step is not None and self._pending_step.step_idx == step: + return self._pending_step, False + for record in self._steps: + if record.step_idx == step: + return record, True + raise KeyError(f"step {step} not present in state trace") + + # -- manifest -------------------------------------------------------- + + def _write_manifest(self) -> None: + destination = self._manifest_file() + temporary = self._temporary_file(destination) + manifest = { + "version": _MANIFEST_VERSION, + "run_artifacts": sorted(self._run_artifacts), + "steps": [record.to_blob() for record in self._steps], + } try: - with open(path) as f: - arr = json.load(f) - return [s for s in arr if isinstance(s, dict)] if isinstance(arr, list) else [] - except Exception as e: - logger.warning("could not parse %s: %s; starting fresh", path, e) - return [] - - def _write_states_atomically(self, steps: list[dict]) -> None: - path = self._states_path() - tmp = path.with_name(path.name + ".tmp") - with open(tmp, "w") as f: - json.dump(steps, f, indent=2, default=str) - os.replace(tmp, path) - - def _derive_next_step(self) -> int: - if not self._steps: - return 0 - return max(int(s["step_idx"]) for s in self._steps if "step_idx" in s) + 1 - - # -- lifecycle -------------------------------------------------------- - - def reset(self, *, wipe_streams: Iterable[str] = ()) -> None: - """Wipe stale artefact dirs + ``states.json`` for a fresh run. - - Resets the in-memory trace and counter. Step 0 (baseline) is dumped by - the caller right after via :meth:`append`. - """ - self.output_dir.mkdir(parents=True, exist_ok=True) - for stream in wipe_streams: - d = stream_dir(self.output_dir, stream) - if d.exists(): - shutil.rmtree(d) - states = self._states_path() - if states.exists(): - states.unlink() + with temporary.open("w") as file: + json.dump(manifest, file, indent=2, default=_json_default) + os.replace(temporary, destination) + finally: + temporary.unlink(missing_ok=True) + + # -- lifecycle and counters ----------------------------------------- + + def reset(self) -> None: + """Remove state-owned artifacts and start a fresh trace.""" + self._output_dir.mkdir(parents=True, exist_ok=True) + records = list(self._steps) + if self._pending_step is not None: + records.append(self._pending_step) + for record in records: + for name in record.artifacts: + self._artifact_file(name, record.step_idx).unlink(missing_ok=True) + for name in self._run_artifacts: + self._artifact_file(name, None).unlink(missing_ok=True) + self._manifest_file().unlink(missing_ok=True) self._steps = [] + self._pending_step = None + self._run_artifacts = set() self._next_step = 0 - # -- counter ---------------------------------------------------------- - @property def next_step_idx(self) -> int: - """The step_idx the next dumped step will get (does NOT advance).""" return self._next_step @property def latest_step(self) -> int | None: - """The highest step_idx successfully written (None if trace is empty).""" if not self._steps: return None - return int(self._steps[-1]["step_idx"]) + return self._steps[-1].step_idx - # -- writing ---------------------------------------------------------- - - def save_image(self, step_idx: int, stream: str, frame) -> str | None: - """Save one RGB frame under ``_.png``; return ``stream`` on success.""" - path = stream_path(self.output_dir, stream, step_idx) - path.parent.mkdir(parents=True, exist_ok=True) - arr = np.asarray(frame) - if arr.dtype != np.uint8: - arr = arr.astype(np.uint8) - try: - imageio.imwrite(path, arr) - return stream - except Exception as e: - logger.warning("frame dump failed for stream %s: %s", stream, e) + def _resolve_read_step(self, step: int | None) -> int | None: + if step is None: return None - - def save_depth(self, step_idx: int, stream: str, depth) -> str | None: - """Save one depth map under ``_.npy``; return ``stream`` on success.""" - path = stream_path(self.output_dir, stream, step_idx) - path.parent.mkdir(parents=True, exist_ok=True) + if step == -1: + latest = self.latest_step + if latest is None: + raise LookupError("no steps available") + return latest + if step < 0: + raise ValueError("step must be -1, None, or nonnegative") + return step + + # -- generic artifact operations ------------------------------------ + + def save( + self, + name: str, + value: Any, + *, + step: int | None, + **options: Any, + ) -> str | None: + """Serialize ``value`` according to ``name`` and return its base name.""" + destination = self._artifact_file(name, step) + temporary = self._temporary_file(destination) + suffix = destination.suffix.lower() + record: StepRecord | None = None + committed = False + if step is not None: + record, committed = self._record_for_write(step) try: - np.save(path, np.asarray(depth, dtype=np.float32)) - return stream - except Exception as e: - logger.warning("depth dump failed for stream %s: %s", stream, e) + if suffix in _IMAGE_SUFFIXES: + array = np.asarray(value) + if array.dtype != np.uint8: + array = array.astype(np.uint8) + imageio.imwrite(temporary, array) + elif suffix == ".npy": + np.save(temporary, np.asarray(value)) + elif suffix == ".json": + with temporary.open("w") as file: + json.dump(value, file, indent=2, default=_json_default) + elif suffix == ".jsonl": + with temporary.open("w") as file: + if isinstance(value, str): + file.write(value) + else: + for item in value: + file.write( + json.dumps(item, default=_json_default) + "\n" + ) + elif suffix == ".mp4": + if isinstance(value, (bytes, bytearray, memoryview)): + temporary.write_bytes(bytes(value)) + else: + imageio.mimwrite( + temporary, + list(value), + fps=int(options.get("fps", 20)), + ) + elif suffix in _TEXT_SUFFIXES: + temporary.write_text(str(value)) + elif suffix == ".bin": + temporary.write_bytes(bytes(value)) + else: + raise ValueError(f"unsupported artifact suffix: {suffix}") + os.replace(temporary, destination) + if record is not None: + record.artifacts.add(name) + if committed: + self._write_manifest() + else: + self._run_artifacts.add(name) + self._write_manifest() + return name + except Exception as exc: + logger.warning("failed to save artifact %s: %s", name, exc) return None + finally: + temporary.unlink(missing_ok=True) + + def load(self, name: str, *, step: int | None = -1) -> Any: + """Load an artifact; ``step=-1`` selects the latest recorded step.""" + resolved_step = self._resolve_read_step(step) + source = self._artifact_file(name, resolved_step) + suffix = source.suffix.lower() + if suffix in _IMAGE_SUFFIXES: + return imageio.imread(source) + if suffix == ".npy": + return np.load(source) + if suffix == ".json": + with source.open() as file: + return json.load(file) + if suffix == ".jsonl": + with source.open() as file: + return [json.loads(line) for line in file if line.strip()] + if suffix in _TEXT_SUFFIXES: + return source.read_text() + return source.read_bytes() + + def load_bytes(self, name: str, *, step: int | None = -1) -> bytes: + resolved_step = self._resolve_read_step(step) + return self._artifact_file(name, resolved_step).read_bytes() + + def exists(self, name: str, *, step: int | None = -1) -> bool: + try: + resolved_step = self._resolve_read_step(step) + except LookupError: + return False + return self._artifact_file(name, resolved_step).exists() + + def remove(self, name: str, *, step: int | None) -> bool: + destination = self._artifact_file(name, step) + if not destination.exists(): + return False + record: StepRecord | None = None + committed = False + if step is not None: + record, committed = self._record_for_write(step) + destination.unlink() + if step is None and name in self._run_artifacts: + self._run_artifacts.remove(name) + self._write_manifest() + elif record is not None: + was_recorded = name in record.artifacts + record.artifacts.discard(name) + if was_recorded and committed: + self._write_manifest() + return True + + def list( + self, + pattern: str = "*", + *, + step: int | None = -1, + ) -> list[str]: + resolved_step = self._resolve_read_step(step) + if resolved_step is None: + names = [ + name + for name in self._run_artifacts + if self._artifact_file(name, None).exists() + ] + else: + record = self.get(resolved_step) + names = [ + name + for name in record.artifacts + if self._artifact_file(name, resolved_step).exists() + ] + return sorted(name for name in names if fnmatch.fnmatch(name, pattern)) + + def list_all(self, pattern: str = "*") -> list[tuple[int | None, str]]: + artifacts: list[tuple[int | None, str]] = [ + (None, name) for name in self.list(pattern, step=None) + ] + for record in self._steps: + artifacts.extend( + (record.step_idx, name) + for name in record.artifacts + if fnmatch.fnmatch(name, pattern) + and self._artifact_file(name, record.step_idx).exists() + ) + return sorted( + artifacts, + key=lambda item: (-1 if item[0] is None else item[0], item[1]), + ) - def artifact_path( + # -- step records ---------------------------------------------------- + + @contextmanager + def record_step( self, - step_idx: int, - stream: str, *, - ext: str | None = None, - ) -> Path: - """Return the canonical path for one step artifact.""" - return stream_path(self.output_dir, stream, step_idx, ext=ext) - - def append(self, record: StepRecord) -> StepRecord: - """Atomically append ``record`` to ``states.json`` and advance the counter. - - The counter and ``states.json`` are kept in sync: this write is the - single source of truth for "latest step". If the write raises, the - counter is NOT advanced (so the next caller reuses the same - ``step_idx``) -- no desync between the in-memory counter and disk. - """ - blob = record.to_blob() - self._write_states_atomically(self._steps + [blob]) - self._steps.append(blob) - self._next_step = max(self._next_step, record.step_idx + 1) - return record - - # -- reading ---------------------------------------------------------- - - def get(self, step_idx: int) -> StepRecord: - """Look up the step record for ``step_idx``.""" - for blob in self._steps: - if int(blob.get("step_idx", -1)) == step_idx: - return StepRecord.from_blob(blob) - raise KeyError(f"step {step_idx} not present in states.json") + state: dict[str, Any], + command: dict | None = None, + result: dict | None = None, + elapsed_s: float | None = None, + extras: dict[str, Any] | None = None, + ) -> Iterator[int]: + if self._pending_step is not None: + raise RuntimeError("a step record is already open") + record = StepRecord( + step_idx=self._next_step, + state=copy.deepcopy(state), + command=copy.deepcopy(command), + result=copy.deepcopy(result), + elapsed_s=elapsed_s, + extras=copy.deepcopy(extras or {}), + ) + self._pending_step = record + try: + yield record.step_idx + except BaseException: + raise + else: + self._steps.append(record) + try: + self._write_manifest() + except Exception: + self._steps.pop() + raise + self._next_step = record.step_idx + 1 + finally: + self._pending_step = None + + def get(self, step: int = -1) -> StepRecord: + resolved_step = self._resolve_read_step(step) + if resolved_step is None: + raise ValueError("step records are not run-level artifacts") + for record in self._steps: + if record.step_idx == resolved_step: + return copy.deepcopy(record) + raise KeyError(f"step {resolved_step} not present in state trace") def records(self) -> list[StepRecord]: - """Return the persisted step records in trace order.""" - return [StepRecord.from_blob(blob) for blob in self._steps] + return copy.deepcopy(self._steps) - def load_image_bytes(self, step_idx: int, stream: str) -> bytes | None: - path = stream_path(self.output_dir, stream, step_idx) - if not path.exists(): - return None - return path.read_bytes() - - def load_depth(self, step_idx: int, stream: str) -> np.ndarray: - path = stream_path(self.output_dir, stream, step_idx) - return np.load(path) - - # -- LLM-facing view (ex-view_driver_state) --------------------------- - - def view(self, step: int | None = None, *, image_slots: dict[str, str] | None = None) -> dict: - """Build the ``view_driver_state`` dict for one step. - - ``image_slots`` maps a ToolResult image slot (``_image_bytes``, - ``_image_cam_bytes``, ``_image_wrist_bytes``) to a stream name whose - saved PNG should be embedded as bytes. The env decides which cameras - map to which slots. - """ - latest = self.latest_step - if latest is None: - return {"error": "no driver state entries; driver not ready"} - nn = latest if step is None else int(step) + # -- LLM-facing view ------------------------------------------------- + + def view( + self, + step: int = -1, + *, + image_slots: dict[str, str] | None = None, + ) -> dict[str, Any]: try: - rec = self.get(nn) - except Exception as e: - return {"error": f"step {nn} not present in driver state trace: {e}"} + record = self.get(step) + except Exception as exc: + return {"error": f"state step not available: {exc}"} + + metadata: dict[str, Any] = {} + metadata_suffix = "_metadata.json" + for name in sorted(record.artifacts): + if not name.endswith(metadata_suffix): + continue + key = name.removesuffix(metadata_suffix) + try: + loaded = self.load(name, step=record.step_idx) + if isinstance(loaded, dict): + loaded = { + field: value + for field, value in loaded.items() + if field not in {"K", "T_base_cam"} + } + metadata[key] = loaded + except Exception as exc: + metadata[key] = {"error": str(exc)} + out: dict[str, Any] = { - "step": nn, - "state": rec.state, - "frames": rec.frames, - "depth": rec.depth, - "camera_meta": { - name: {k: v for k, v in meta.items() if k not in {"K", "T_base_cam"}} - for name, meta in (rec.camera_meta or {}).items() - }, + "step": record.step_idx, + "state": record.state, + "artifacts": sorted(record.artifacts), + "camera_meta": metadata, "log": { - "command": rec.command, - "result": rec.result, - "elapsed_s": rec.elapsed_s, + "command": record.command, + "result": record.result, + "elapsed_s": record.elapsed_s, }, } - if rec.extras: - out["extras"] = rec.extras + if record.extras: + out["extras"] = record.extras if image_slots: - for slot, stream in image_slots.items(): - b = self.load_image_bytes(nn, stream) - if b: - out[slot] = b + for slot, name in image_slots.items(): + if name not in record.artifacts: + continue + try: + out[slot] = self.load_bytes(name, step=record.step_idx) + except FileNotFoundError: + continue return out diff --git a/rpent/utils/sam3_client.py b/rpent/utils/sam3_client.py index f8a80037..22c23977 100644 --- a/rpent/utils/sam3_client.py +++ b/rpent/utils/sam3_client.py @@ -5,7 +5,6 @@ import base64 import io from dataclasses import dataclass -from pathlib import Path from typing import Any import imageio.v2 as imageio @@ -35,7 +34,7 @@ def __init__(self, client: RpcClient, *, timeout_s: float = 120.0) -> None: def segment( self, - image_path: str | Path, + image: bytes | bytearray | memoryview | np.ndarray, *, text_prompt: str | None = None, point: list[int] | None = None, @@ -55,7 +54,12 @@ def segment( if not 0.0 <= float(min_score) <= 1.0: raise ValueError("min_score must be between 0 and 1") - image_bytes = Path(image_path).read_bytes() + if isinstance(image, np.ndarray): + buffer = io.BytesIO() + imageio.imwrite(buffer, image, format="png") + image_bytes = buffer.getvalue() + else: + image_bytes = bytes(image) body: dict[str, Any] = { "image_base64": base64.b64encode(image_bytes).decode("ascii"), "min_score": float(min_score), From 2b90919407e326fa5d4e46ac012130f8546cb3e8 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Wed, 5 Aug 2026 03:30:40 +0000 Subject: [PATCH 03/25] refactor(toolkit): centralize state capture in base Toolkit.execute_tool Signed-off-by: Jiaxing Qiu --- robots/libero/toolkit.py | 64 +++++++++++++--------------- rpent/tools/toolkit.py | 91 +++++++++++++++++++++++++++++++++------- 2 files changed, 106 insertions(+), 49 deletions(-) diff --git a/robots/libero/toolkit.py b/robots/libero/toolkit.py index b6ae17d5..89a5c1da 100644 --- a/robots/libero/toolkit.py +++ b/robots/libero/toolkit.py @@ -5,14 +5,13 @@ """ from __future__ import annotations -import time from functools import partial from typing import Any from robots.libero import tools as libero_tools from rpent.dashboard.events import DashboardEventSink, ToolResultEvent from rpent.tools.state import EnvState -from rpent.tools.toolkit import ToolCancelled, Toolkit +from rpent.tools.toolkit import Toolkit from rpent.utils.logging import get_logger, get_output_dir @@ -52,46 +51,40 @@ def _register_libero_tools(self) -> None: "segment": partial(self._primitives.segment, state=self._state), } for name, handler in inspection_handlers.items(): - self.add_tool(name, specs[name], handler) - # Primitive tools: each goes through _step, which looks up the - # matching primitive method via getattr at call time. + self.add_tool( + name, + specs[name], + handler, + captures_state=False, + ) for name in libero_tools.PRIMITIVE_TOOL_NAMES: - self.add_tool(name, specs[name], partial(self._step, name)) - - def _step(self, name: str, **kwargs) -> dict: - """Run ``self._primitives.(**kwargs)``, dump the new step, and - return the rendered state view + log. - """ - command = {"action": name, **kwargs} - t0 = time.time() - start_frame = self._primitives.recorded_frame_count() - try: - result = getattr(self._primitives, name)(**kwargs) - self.raise_if_cancelled() - except ToolCancelled as exc: - result = { - "error": str(exc), - "code": "tool_cancelled", - "interrupted": True, - } - elapsed = round(time.time() - t0, 2) - - if isinstance(result, dict): - result_dict = result - else: - result_dict = {"value": result} + self.add_tool( + name, + specs[name], + getattr(self._primitives, name), + captures_state=True, + ) + def get_state( + self, + *, + command: dict[str, Any], + result: dict[str, Any], + elapsed_s: float, + ) -> dict[str, Any]: + frame_start = self._action_frame_cursor + self._action_frame_cursor = self._primitives.recorded_frame_count() record = libero_tools.dump_state( self._primitives, self._state, - log={"command": command, "result": result_dict, "elapsed_s": elapsed}, + log={"command": command, "result": result, "elapsed_s": elapsed_s}, ) action_video_name = None if self._dashboard_events.enabled: try: - frames = self._primitives.frame_slice(start_frame) + frames = self._primitives.frame_slice(frame_start) if frames: - candidate = f"action_{name}.mp4" + candidate = f"action_{command['action']}.mp4" if self._state.save( candidate, frames, @@ -106,9 +99,9 @@ def _step(self, name: str, **kwargs) -> dict: e, ) out = libero_tools.view_driver_state(record.step_idx, state=self._state) - out["agent_elapsed_s"] = elapsed - if result_dict.get("interrupted"): - out.update(result_dict) + out["agent_elapsed_s"] = elapsed_s + if result.get("interrupted"): + out.update(result) if action_video_name is not None: out["action_video_artifact"] = action_video_name return out @@ -127,6 +120,7 @@ def init_primitives_clean( ) primitives.reset() primitives.start_recording() + self._action_frame_cursor = primitives.recorded_frame_count() libero_tools.dump_state(primitives, self._state, log=None) self._dashboard_events.emit( ToolResultEvent( diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index a747020f..7502b1f3 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -7,8 +7,10 @@ from __future__ import annotations import base64 +import inspect import json import threading +import time import traceback from collections.abc import Callable from dataclasses import dataclass, field @@ -112,8 +114,10 @@ def __init__( dashboard_events: DashboardEventSink, state: Any = None, ) -> None: - # name -> (spec, handler) - self._tools: dict[str, tuple[dict[str, Any], Callable[..., dict[str, Any]]]] = {} + self._tools: dict[ + str, + tuple[dict[str, Any], Callable[..., Any], bool], + ] = {} self._dashboard_events = dashboard_events self._state = state self._operation_lock = threading.Lock() @@ -128,7 +132,9 @@ def add_tool( self, name: str, spec: dict[str, Any], - handler: Callable[..., dict[str, Any]], + handler: Callable[..., Any], + *, + captures_state: bool, ) -> None: """Register one tool under ``name`` with its schema and handler. @@ -138,8 +144,10 @@ def add_tool( ``description``, ``input_schema``). handler: Callable invoked with the tool's input kwargs; returns a result dict. + captures_state: Whether the tool updates environment state + that must be captured after the handler returns or raises. """ - self._tools[name] = (spec, handler) + self._tools[name] = (spec, handler, captures_state) def _register_common_tools(self) -> None: """Register the file/IO tools shared by every run.""" @@ -147,7 +155,12 @@ def _register_common_tools(self) -> None: for spec in common.TOOLS_SPEC: name = spec["name"] - self.add_tool(name, spec, common.TOOL_HANDLERS[name]) + self.add_tool( + name, + spec, + common.TOOL_HANDLERS[name], + captures_state=False, + ) # ------------------------------------------------------------------ # Planner-facing API @@ -156,39 +169,89 @@ def _register_common_tools(self) -> None: def get_tools_spec(self) -> list[dict[str, Any]]: """Return the tool schemas the LLM sees.""" return substitute( - [spec for spec, _ in self._tools.values()] + [spec for spec, _, _ in self._tools.values()] ) def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: """Dispatch a tool call to its registered handler.""" entry = self._tools.get(name) if entry is None: - return ToolResult(name=name, result={"error": f"unknown tool: {name}"}) - handler = entry[1] + return self._finish_tool(name, {"error": f"unknown tool: {name}"}) + _, handler, captures_state = entry with self._operation_lock: if self._active_operation is not None: - return ToolResult( - name=name, - result={"error": "another tool operation is still active"}, + return self._finish_tool( + name, + {"error": "another tool operation is still active"}, ) operation = _ToolOperation() self._active_operation = operation try: + started = time.perf_counter() + failed = False try: result = handler(**input_dict) except TypeError as e: - result = {"error": f"bad arguments for {name}: {e}", "got": input_dict} + return self._finish_tool( + name, + {"error": f"bad arguments for {name}: {e}", "got": input_dict}, + ) + except ToolCancelled as e: + result = { + "error": str(e), + "code": "tool_cancelled", + "interrupted": True, + } + failed = True except Exception as e: result = {"error": str(e), "traceback": traceback.format_exc()} - self._dashboard_events.emit(ToolResultEvent(name=name, result=result)) - return ToolResult(name=name, result=result) + failed = True + + if captures_state: + elapsed_s = round(time.perf_counter() - started, 2) + result_dict = result if isinstance(result, dict) else {"value": result} + command = {"action": name, **input_dict} + try: + captured = self.get_state( + command=command, + result=result_dict, + elapsed_s=elapsed_s, + ) + except Exception as e: + captured = result_dict + captured["state_capture_error"] = str(e) + captured.setdefault( + "error", f"failed to capture state after {name}: {e}" + ) + captured.setdefault("traceback", traceback.format_exc()) + result = captured + if failed: + result.setdefault("error", result_dict["error"]) + if "traceback" in result_dict: + result.setdefault("traceback", result_dict["traceback"]) + + return self._finish_tool(name, result) finally: with self._operation_lock: self._active_operation = None operation.done_event.set() + def _finish_tool(self, name: str, result: Any) -> ToolResult: + self._dashboard_events.emit(ToolResultEvent(name=name, result=result)) + return ToolResult(name=name, result=result) + + def get_state( + self, + *, + command: dict[str, Any], + result: dict[str, Any], + elapsed_s: float, + ) -> dict[str, Any]: + """Capture and return the observation produced by a stateful tool.""" + raise NotImplementedError + # ------------------------------------------------------------------ # Server lifecycle hooks (overridden by env toolkits) # ------------------------------------------------------------------ From 489a2e4c3c99382203e9705cb62d0540567181cf Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Wed, 5 Aug 2026 09:08:19 +0000 Subject: [PATCH 04/25] refactor(toolkit): mark state-advancing primitives with @updatestate Signed-off-by: Jiaxing Qiu --- .../rst_source/development/add_primitive.rst | 24 +++-- .../rst_source/development/add_robot.rst | 11 ++- .../rst_source/development/add_primitive.rst | 23 +++-- .../rst_source/development/add_robot.rst | 8 +- robots/libero/toolkit.py | 37 ++++---- robots/libero/tools.py | 35 +++++--- rpent/tools/state.py | 87 ++++++++++--------- rpent/tools/toolkit.py | 48 +++++++--- 8 files changed, 160 insertions(+), 113 deletions(-) diff --git a/docs/source-en/rst_source/development/add_primitive.rst b/docs/source-en/rst_source/development/add_primitive.rst index 1148f94c..297851e5 100644 --- a/docs/source-en/rst_source/development/add_primitive.rst +++ b/docs/source-en/rst_source/development/add_primitive.rst @@ -35,7 +35,7 @@ call. They differ only in how the method is implemented. Add a scripted primitive ------------------------ -Adding a scripted primitive usually involves three steps: +Adding a scripted primitive usually involves two steps: 1. **Add a method to the primitives.** Add the method to the current environment's primitives class, such as @@ -43,16 +43,25 @@ Adding a scripted primitive usually involves three steps: the tool-call arguments, performs the work, usually through one or more ``self._env.step(...)`` calls, and returns a small log ``dict``. + Mark the method with :func:`~rpent.tools.toolkit.updatestate` so the + toolkit re-renders state (``get_state``) automatically after it runs: + .. code-block:: python + from rpent.tools.toolkit import updatestate + + @updatestate def open_drawer(self, dx: float = 0.15) -> dict: # Move end-effector back by dx while gripper is closed. for _ in range(N): self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} + Read-only tools (``view_driver_state``, ``back_project``, ``segment``, + ...) are simply left unmarked -- the toolkit skips state capture for them. + 2. **Add the tool schema.** Add an entry to ``TOOLS_SPEC`` in - ``toolkit.py``: + ``robots//tools.py``: .. code-block:: python @@ -67,13 +76,10 @@ Adding a scripted primitive usually involves three steps: }, } -3. **Register the tool in the toolkit.** Route it through the toolkit's - ``_step`` helper so that state is re-rendered after execution: - - .. code-block:: python - - self.add_tool("open_drawer", OPEN_DRAWER_SPEC, - lambda **kw: self._step("open_drawer", **kw)) +Once both exist, the toolkit registers the tool automatically: it iterates +``TOOLS_SPEC`` and binds each spec to the matching primitive-driver method +(e.g. ``getattr(self._primitives, name)``); ``@updatestate`` decides whether +state is captured -- no explicit ``add_tool`` call is needed. After these steps, the ``api``, ``claude_code``, and ``codex`` planners can all call the primitive without any other code changes. diff --git a/docs/source-en/rst_source/development/add_robot.rst b/docs/source-en/rst_source/development/add_robot.rst index 0a2b5497..7f61d588 100644 --- a/docs/source-en/rst_source/development/add_robot.rst +++ b/docs/source-en/rst_source/development/add_robot.rst @@ -223,10 +223,13 @@ plus any module-level functions referenced by the toolkit (e.g. ``view_driver_state``, ``back_project``, ``finish``). **Per-step state dump** — ``dump_state(driver, env_state, log)`` opens -``env_state.record_step(...)`` and receives the allocated step index. Save -large observations through ``env_state.save(..., step=step_idx)``. ``EnvState`` -owns and commits the ``StepRecord`` and adds every successfully saved base name -to its flat ``artifacts`` set automatically. Readers use the canonical artifact +``env_state.record_step(...)`` and receives the allocated step index; the +``StepRecord`` is appended and committed immediately. Save large observations +through ``env_state.save(...)`` — inside a ``record_step`` block the ``step`` +argument may be omitted (it defaults to the new step), pass an explicit +``step=`` to target a different step, and ``step=None`` for run-level +artifacts. ``EnvState`` adds every successfully saved base name to the step's +flat ``artifacts`` set automatically. Readers use the canonical artifact filenames rather than maintaining a parallel observation index. **Toolkit class** — subclass ``rpent.tools.toolkit.Toolkit``: diff --git a/docs/source-zh/rst_source/development/add_primitive.rst b/docs/source-zh/rst_source/development/add_primitive.rst index 2ba6ce24..5d475ea3 100644 --- a/docs/source-zh/rst_source/development/add_primitive.rst +++ b/docs/source-zh/rst_source/development/add_primitive.rst @@ -33,22 +33,31 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 添加一个脚本化原语 ------------------ -添加脚本化原语通常需要以下三个步骤： +添加脚本化原语通常需要以下两个步骤： 1. **在 primitives 中添加方法。** 在当前环境的 primitives 类（如 ``LiberoPrimitives``、``MyRobotPrimitives``）中添加 一个方法。该方法接收工具调用的参数，执行一次或多次 ``self._env.step(...)``，并返回一个简短的日志字典。 + 为该方法加上 :func:`~rpent.tools.toolkit.updatestate` 装饰器， + toolkit 会在其执行后自动重新渲染状态（``get_state``）： + .. code-block:: python + from rpent.tools.toolkit import updatestate + + @updatestate def open_drawer(self, dx: float = 0.15) -> dict: # 保持夹爪闭合，沿 -x 方向后拉 dx 米。 for _ in range(N): self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} -2. **添加工具定义。** 在 ``toolkit.py`` 的 ``TOOLS_SPEC`` 中新增一项： + 只读工具（``view_driver_state``、``back_project``、``segment`` 等） + 无需装饰——toolkit 会跳过它们的状态捕获。 + +2. **添加工具定义。** 在 ``robots//tools.py`` 的 ``TOOLS_SPEC`` 中新增一项： .. code-block:: python @@ -63,13 +72,9 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 }, } -3. **在 toolkit 中注册工具。** 通过 toolkit 的 ``_step`` 辅助函数运行 - 该工具，使其在执行结束后自动重新渲染状态： - - .. code-block:: python - - self.add_tool("open_drawer", OPEN_DRAWER_SPEC, - lambda **kw: self._step("open_drawer", **kw)) +两者就位后，toolkit 会自动注册该工具：它遍历 ``TOOLS_SPEC``，把每个定义 +绑定到对应的 primitive driver 方法（如 ``getattr(self._primitives, name)``）， +由 ``@updatestate`` 决定是否捕获状态——无需显式调用 ``add_tool``。 完成以上步骤后，``api``、``claude_code`` 和 ``codex`` 三种 planner 都可以调用该工具，无需修改其他代码。 diff --git a/docs/source-zh/rst_source/development/add_robot.rst b/docs/source-zh/rst_source/development/add_robot.rst index 0f76750c..394d9e90 100644 --- a/docs/source-zh/rst_source/development/add_robot.rst +++ b/docs/source-zh/rst_source/development/add_robot.rst @@ -211,9 +211,11 @@ Anthropic API 的工具定义格式，包含 ``name``、``description`` 和 **每步状态 dump** —— ``dump_state(driver, env_state, log)`` 通过 ``env_state.record_step(...)`` 创建由 ``EnvState`` 持有的步骤，并取得分配的 -step index。大型观测通过 ``env_state.save(..., step=step_idx)`` 保存。每次保存 -成功后，``EnvState`` 会自动把基础文件名加入该 ``StepRecord`` 的扁平 -``artifacts`` 集合并最终提交记录；读取方直接使用规范化的工件文件名。 +step index；该 ``StepRecord`` 会被立即追加并提交。大型观测通过 +``env_state.save(...)`` 保存——在 ``record_step`` 块内可省略 ``step`` 参数 +（默认指向刚创建的步骤），传显式 ``step=`` 可指定其它步骤，``step=None`` +用于运行级工件。每次保存成功后，``EnvState`` 会自动把基础文件名加入该 +``StepRecord`` 的扁平 ``artifacts`` 集合；读取方直接使用规范化的工件文件名。 **Toolkit 类** 继承 ``rpent.tools.toolkit.Toolkit``： diff --git a/robots/libero/toolkit.py b/robots/libero/toolkit.py index 89a5c1da..95b9253f 100644 --- a/robots/libero/toolkit.py +++ b/robots/libero/toolkit.py @@ -18,10 +18,6 @@ class LiberoToolkit(Toolkit): """Toolkit for the LIBERO environment.""" - # Tool schemas keyed by name (built once from the canonical ordered list - # in libero_tools.TOOLS_SPEC) so each tool registers with its own spec. - _SPECS = {spec["name"]: spec for spec in libero_tools.TOOLS_SPEC} - def __init__( self, *, @@ -37,10 +33,12 @@ def __init__( # Registration # ------------------------------------------------------------------ def _register_libero_tools(self) -> None: - specs = self._SPECS - # Inspection tools do not advance environment state. Most are stateless - # module functions; segment is bound to the primitives-owned SAM3 client. - inspection_handlers = { + # Read-only tools whose handlers aren't primitive methods (they need + # the run's EnvState bound in, or -- like segment -- must stay + # read-only despite being a primitives method). Every other spec binds + # to its primitive-driver method; @updatestate on the method decides + # whether state is captured. + state_handlers = { "view_driver_state": partial( libero_tools.view_driver_state, state=self._state ), @@ -50,20 +48,15 @@ def _register_libero_tools(self) -> None: "back_project": partial(libero_tools.back_project, state=self._state), "segment": partial(self._primitives.segment, state=self._state), } - for name, handler in inspection_handlers.items(): - self.add_tool( - name, - specs[name], - handler, - captures_state=False, - ) - for name in libero_tools.PRIMITIVE_TOOL_NAMES: - self.add_tool( - name, - specs[name], - getattr(self._primitives, name), - captures_state=True, - ) + for spec in libero_tools.TOOLS_SPEC: + name = spec["name"] + if name in state_handlers: + handler = state_handlers[name] + else: + handler = getattr(self._primitives, name, None) + if handler is None: + continue # spec without a backing primitive method + self.add_tool(name, spec, handler) def get_state( self, diff --git a/robots/libero/tools.py b/robots/libero/tools.py index 7f511e8e..ff8cb901 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -8,6 +8,7 @@ from robots.libero.env_client import LiberoEnvClient from rpent.tools.state import EnvState, StepRecord +from rpent.tools.toolkit import updatestate from rpent.utils.logging import get_logger from rpent.utils.sam3_client import Sam3Client from rpent.utils.vla_client import VLAClient @@ -124,6 +125,7 @@ def _vlm_chunk(self, instruction: str): if original_task is not None: self._last_obs["task_descriptions"] = original_task + @updatestate def pi0_pick( self, prompt: str, @@ -199,6 +201,7 @@ def pi0_pick( }, } + @updatestate def pi0_doubled( self, prompt: str, @@ -240,6 +243,7 @@ def pi0_doubled( }, } + @updatestate def move_to( self, xyz, @@ -304,6 +308,7 @@ def move_to( "libero_terminated": self.env.episode_terminated, } + @updatestate def rotate_wrist( self, *, @@ -377,6 +382,7 @@ def _yaw_of(quat_xyzw): "libero_terminated": self.env.episode_terminated, } + @updatestate def rotate_pitch( self, *, @@ -458,6 +464,7 @@ def _pitch_of(quat_xyzw): "libero_terminated": self.env.episode_terminated, } + @updatestate def move_pose( self, xyz, @@ -528,6 +535,7 @@ def _yaw_of(q): "libero_terminated": self.env.episode_terminated, } + @updatestate def release( self, *, @@ -556,6 +564,7 @@ def release( "libero_terminated": self.env.episode_terminated, } + @updatestate def set_gripper( self, *, @@ -725,6 +734,19 @@ def segment( return result +def _is_primitive_action(name: object) -> bool: + """Whether ``name`` is a state-advancing LIBERO primitive. + + A primitive is any ``@updatestate``-marked method on + :class:`LiberoPrimitives`; read-only tools (``view_driver_state``, + ``back_project``, ``segment``, ...) and non-strings read as ``False``. + """ + if not isinstance(name, str): + return False + method = getattr(LiberoPrimitives, name, None) + return method is not None and bool(getattr(method, "_captures_state", False)) + + def write_recipe_from_states(state: EnvState, recipe_tag: str) -> str: """Find a command sequence that gets ``libero_terminated=True``. @@ -736,7 +758,7 @@ def write_recipe_from_states(state: EnvState, recipe_tag: str) -> str: result = record.result if ( command is not None - and command.get("action") in PRIMITIVE_TOOL_NAMES + and _is_primitive_action(command.get("action")) and not (isinstance(result, dict) and result.get("error")) ): command_events.append(((record.step_idx, -1), command)) @@ -1032,17 +1054,6 @@ def _save_observation_artifacts( # Tool schema declarations (Anthropic-shaped canonical schema) # --------------------------------------------------------------------------- -PRIMITIVE_TOOL_NAMES: tuple[str, ...] = ( - "move_to", - "pi0_pick", - "pi0_doubled", - "release", - "set_gripper", - "rotate_wrist", - "rotate_pitch", - "move_pose", -) - TOOLS_SPEC = [ { "name": "view_driver_state", diff --git a/rpent/tools/state.py b/rpent/tools/state.py index a24d15d9..3d43677b 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -89,8 +89,8 @@ def __init__(self, output_dir: Path | str): self._output_dir = Path(output_dir) self._output_dir.mkdir(parents=True, exist_ok=True) self._steps: list[StepRecord] = [] - self._pending_step: StepRecord | None = None self._run_artifacts: set[str] = set() + self._open_count = 0 self._next_step = 0 self.reset() @@ -125,12 +125,11 @@ def _temporary_file(self, destination: Path) -> Path: f".{destination.stem}.tmp{destination.suffix}" ) - def _record_for_write(self, step: int) -> tuple[StepRecord, bool]: - if self._pending_step is not None and self._pending_step.step_idx == step: - return self._pending_step, False + def _record_for(self, step: int) -> StepRecord: + """Return the live step record at ``step`` for in-place updates.""" for record in self._steps: if record.step_idx == step: - return record, True + return record raise KeyError(f"step {step} not present in state trace") # -- manifest -------------------------------------------------------- @@ -155,18 +154,15 @@ def _write_manifest(self) -> None: def reset(self) -> None: """Remove state-owned artifacts and start a fresh trace.""" self._output_dir.mkdir(parents=True, exist_ok=True) - records = list(self._steps) - if self._pending_step is not None: - records.append(self._pending_step) - for record in records: + for record in self._steps: for name in record.artifacts: self._artifact_file(name, record.step_idx).unlink(missing_ok=True) for name in self._run_artifacts: self._artifact_file(name, None).unlink(missing_ok=True) self._manifest_file().unlink(missing_ok=True) self._steps = [] - self._pending_step = None self._run_artifacts = set() + self._open_count = 0 self._next_step = 0 @property @@ -198,17 +194,23 @@ def save( name: str, value: Any, *, - step: int | None, + step: int | None = -1, **options: Any, ) -> str | None: - """Serialize ``value`` according to ``name`` and return its base name.""" + """Serialize ``value`` according to ``name`` and return its base name. + + ``step`` defaults to ``-1`` (the most recently recorded step), so calls + made inside (or right after) a :meth:`record_step` block attach to that + step without an explicit index. Pass an ``int`` to target a specific + step, or ``None`` for a run-level artifact such as an episode video. + """ + step = self._resolve_read_step(step) destination = self._artifact_file(name, step) temporary = self._temporary_file(destination) suffix = destination.suffix.lower() - record: StepRecord | None = None - committed = False - if step is not None: - record, committed = self._record_for_write(step) + record: StepRecord | None = ( + self._record_for(step) if step is not None else None + ) try: if suffix in _IMAGE_SUFFIXES: array = np.asarray(value) @@ -247,11 +249,9 @@ def save( os.replace(temporary, destination) if record is not None: record.artifacts.add(name) - if committed: - self._write_manifest() else: self._run_artifacts.add(name) - self._write_manifest() + self._write_manifest() return name except Exception as exc: logger.warning("failed to save artifact %s: %s", name, exc) @@ -293,19 +293,12 @@ def remove(self, name: str, *, step: int | None) -> bool: destination = self._artifact_file(name, step) if not destination.exists(): return False - record: StepRecord | None = None - committed = False - if step is not None: - record, committed = self._record_for_write(step) + if step is None: + self._run_artifacts.discard(name) + else: + self._record_for(step).artifacts.discard(name) destination.unlink() - if step is None and name in self._run_artifacts: - self._run_artifacts.remove(name) - self._write_manifest() - elif record is not None: - was_recorded = name in record.artifacts - record.artifacts.discard(name) - if was_recorded and committed: - self._write_manifest() + self._write_manifest() return True def list( @@ -358,7 +351,13 @@ def record_step( elapsed_s: float | None = None, extras: dict[str, Any] | None = None, ) -> Iterator[int]: - if self._pending_step is not None: + """Append a new step record and yield its index. + + The step is committed to the trace immediately, so any subsequent + :meth:`save` without an explicit ``step`` attaches to it. If the block + raises, the step and any artifacts written to it are rolled back. + """ + if self._open_count: raise RuntimeError("a step record is already open") record = StepRecord( step_idx=self._next_step, @@ -368,21 +367,27 @@ def record_step( elapsed_s=elapsed_s, extras=copy.deepcopy(extras or {}), ) - self._pending_step = record + self._steps.append(record) + self._next_step = record.step_idx + 1 + self._open_count += 1 + self._write_manifest() try: yield record.step_idx except BaseException: - raise - else: - self._steps.append(record) + for name in list(record.artifacts): + self._artifact_file(name, record.step_idx).unlink(missing_ok=True) + if self._steps and self._steps[-1] is record: + self._steps.pop() + self._next_step = record.step_idx try: self._write_manifest() - except Exception: - self._steps.pop() - raise - self._next_step = record.step_idx + 1 + except Exception as exc: + logger.warning( + "failed to rewrite manifest after step rollback: %s", exc + ) + raise finally: - self._pending_step = None + self._open_count -= 1 def get(self, step: int = -1) -> StepRecord: resolved_step = self._resolve_read_step(step) diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 7502b1f3..22d30871 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -7,7 +7,6 @@ from __future__ import annotations import base64 -import inspect import json import threading import time @@ -30,6 +29,37 @@ class ToolCancelled(Exception): """Raised when an environment reaches a safe cancellation boundary.""" +def updatestate(func): + """Mark a tool handler as one that advances environment state. + + The toolkit captures a fresh observation (:meth:`Toolkit.get_state`) + after a handler carrying this marker runs (or raises). Apply it to + primitive-driver methods that move the robot; read-only tools (file IO, + ``view_driver_state``, ``back_project``, ``segment``, ...) are left + unmarked and skip state capture. + + The marker is read off the underlying function, so registering a bound + method (``getattr(self._driver, name)``) inherits it automatically. + ``functools.partial`` does not delegate attribute access, so a partial + wrapping a marked method is treated as read-only (intentional -- see the + LIBERO ``segment`` tool). + """ + func._captures_state = True + return func + + +def _captures_state(handler: Callable[..., Any]) -> bool: + """Whether ``handler`` was marked with :func:`updatestate`. + + Resolves through ``__func__`` so bound methods (the common case for + primitive-driver tools) report the marker set on their underlying + function. Plain functions, closures, and ``functools.partial`` objects + do not delegate, so undecorated read-only handlers read as ``False``. + """ + target = getattr(handler, "__func__", handler) + return bool(getattr(target, "_captures_state", False)) + + @dataclass class ToolResult: """Result of executing one tool call. @@ -133,8 +163,6 @@ def add_tool( name: str, spec: dict[str, Any], handler: Callable[..., Any], - *, - captures_state: bool, ) -> None: """Register one tool under ``name`` with its schema and handler. @@ -143,11 +171,10 @@ def add_tool( spec: Anthropic-shaped tool schema dict (``name``, ``description``, ``input_schema``). handler: Callable invoked with the tool's input kwargs; returns - a result dict. - captures_state: Whether the tool updates environment state - that must be captured after the handler returns or raises. + a result dict. Decorate state-advancing primitives with + :func:`updatestate`. """ - self._tools[name] = (spec, handler, captures_state) + self._tools[name] = (spec, handler, _captures_state(handler)) def _register_common_tools(self) -> None: """Register the file/IO tools shared by every run.""" @@ -155,12 +182,7 @@ def _register_common_tools(self) -> None: for spec in common.TOOLS_SPEC: name = spec["name"] - self.add_tool( - name, - spec, - common.TOOL_HANDLERS[name], - captures_state=False, - ) + self.add_tool(name, spec, common.TOOL_HANDLERS[name]) # ------------------------------------------------------------------ # Planner-facing API From 65b5f79fce01ba9277659ba2bbf47ed5844a4fea Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Wed, 5 Aug 2026 09:22:14 +0000 Subject: [PATCH 05/25] chore: several renames Signed-off-by: Jiaxing Qiu --- .../rst_source/development/add_primitive.rst | 6 +++--- .../rst_source/development/add_robot.rst | 6 +++--- .../rst_source/development/interfaces.rst | 2 +- docs/source-en/rst_source/quickstart.rst | 2 +- docs/source-en/rst_source/usage/libero.rst | 4 ++-- .../rst_source/development/add_primitive.rst | 6 +++--- .../rst_source/development/add_robot.rst | 6 +++--- .../rst_source/development/interfaces.rst | 2 +- docs/source-zh/rst_source/quickstart.rst | 2 +- docs/source-zh/rst_source/usage/libero.rst | 4 ++-- robots/libero/guides/env_calibration.md | 8 ++++---- robots/libero/guides/pro_hybrid_guide.md | 8 ++++---- robots/libero/guides/strict_hybrid_guide.md | 6 +++--- robots/libero/prompts/system.py | 12 ++++++------ robots/libero/prompts/user.py | 4 ++-- robots/libero/toolkit.py | 12 ++++++------ robots/libero/tools.py | 10 +++++----- rpent/context/prompts/prompt.py | 2 +- rpent/planner/api_loop.py | 4 ++-- rpent/tools/toolkit.py | 16 ++++++++-------- scripts/codex_proxy/litellm_callbacks.py | 2 +- 21 files changed, 62 insertions(+), 62 deletions(-) diff --git a/docs/source-en/rst_source/development/add_primitive.rst b/docs/source-en/rst_source/development/add_primitive.rst index 297851e5..947a38bf 100644 --- a/docs/source-en/rst_source/development/add_primitive.rst +++ b/docs/source-en/rst_source/development/add_primitive.rst @@ -44,7 +44,7 @@ Adding a scripted primitive usually involves two steps: more ``self._env.step(...)`` calls, and returns a small log ``dict``. Mark the method with :func:`~rpent.tools.toolkit.updatestate` so the - toolkit re-renders state (``get_state``) automatically after it runs: + toolkit re-renders state (``get_env_state``) automatically after it runs: .. code-block:: python @@ -57,7 +57,7 @@ Adding a scripted primitive usually involves two steps: self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} - Read-only tools (``view_driver_state``, ``back_project``, ``segment``, + Read-only tools (``view_env_state``, ``back_project``, ``segment``, ...) are simply left unmarked -- the toolkit skips state capture for them. 2. **Add the tool schema.** Add an entry to ``TOOLS_SPEC`` in @@ -175,7 +175,7 @@ Design principles for a new primitive - **Return small dicts.** Tool return values are fed back to the LLM as text. Save larger observations through ``EnvState.save``; ``EnvState`` automatically records each logical base name in its owned - ``StepRecord.artifacts`` set. Expose images through ``view_driver_state`` and + ``StepRecord.artifacts`` set. Expose images through ``view_env_state`` and geometry through environment tools rather than returning raw paths. - **Guardrails belong in env_server**, not in the toolkit. The LLM can and will call any tool with any arguments; workspace bounds diff --git a/docs/source-en/rst_source/development/add_robot.rst b/docs/source-en/rst_source/development/add_robot.rst index 7f61d588..b3578486 100644 --- a/docs/source-en/rst_source/development/add_robot.rst +++ b/docs/source-en/rst_source/development/add_robot.rst @@ -220,7 +220,7 @@ state needed for the current run. It exposes one method per primitive tool **Tool definitions and handlers** — a module-level ``TOOLS_SPEC`` list of Anthropic-style tool definitions (``name``, ``description``, ``input_schema``), plus any module-level functions referenced by the toolkit (e.g. -``view_driver_state``, ``back_project``, ``finish``). +``view_env_state``, ``back_project``, ``finish``). **Per-step state dump** — ``dump_state(driver, env_state, log)`` opens ``env_state.record_step(...)`` and receives the allocated step index; the @@ -238,7 +238,7 @@ filenames rather than maintaining a parallel observation index. helper (named ``init_primitives_clean`` in LIBERO; it calls ``EnvState.reset()``, constructs the primitives, and dumps step 0), - register each tool with ``self.add_tool(name, spec, handler)`` — stateless - readers (``view_driver_state``, ``finish``, …) bind directly to module-level + readers (``view_env_state``, ``finish``, …) bind directly to module-level functions; primitive tools route through ``_step(name, **kwargs)`` which calls ``getattr(self._primitives, name)(**kwargs)`` and re-renders state, - override ``close()`` to save remaining agent-side artifacts through @@ -260,7 +260,7 @@ Conventions worth keeping exposed to all planners. - Server-side return values must be picklable and torch-free. - Each primitive tool dumps a fresh state snapshot after running so the next - ``view_driver_state`` call reflects the post-action world. + ``view_env_state`` call reflects the post-action world. - Treat ``dump_state`` as the source of truth for what the agent sees — any new modality (e.g. tactile, force) goes through it. diff --git a/docs/source-en/rst_source/development/interfaces.rst b/docs/source-en/rst_source/development/interfaces.rst index bf6a6a59..02d865a0 100644 --- a/docs/source-en/rst_source/development/interfaces.rst +++ b/docs/source-en/rst_source/development/interfaces.rst @@ -90,7 +90,7 @@ Subclass ``Toolkit`` in ``robots//toolkit.py`` and register env tools with ends; optional ``_image_bytes`` (etc.) to return camera images. The base class already registers common file tools; call ``super().__init__()`` then -``add_tool`` for env tools. Per-step state and ``view_driver_state`` are in +``add_tool`` for env tools. Per-step state and ``view_env_state`` are in :doc:`add_primitive`. Inter-process communication diff --git a/docs/source-en/rst_source/quickstart.rst b/docs/source-en/rst_source/quickstart.rst index 8fb27a7d..ac49ab68 100644 --- a/docs/source-en/rst_source/quickstart.rst +++ b/docs/source-en/rst_source/quickstart.rst @@ -106,7 +106,7 @@ A successful run: 4. By default, artifacts are saved under ``logs/__t_s/``. They include ``transcript_*.json`` (run record), ``states.json`` (the versioned ``EnvState`` manifest), ``recipe_*.jsonl`` (action sequence), and ``episode.mp4`` (episode video). Step artifact files use flat, zero-padded step prefixes with a minimum width of two digits and are managed internally by ``EnvState``. Inspect the final state through the Dashboard or -``view_driver_state(step=-1)``. Its top-level ``libero_terminated`` value is the +``view_env_state(step=-1)``. Its top-level ``libero_terminated`` value is the benchmark outcome. ``states.json`` is internal ``EnvState`` storage and should not be parsed by callers. You can also open ``episode.mp4`` to review the run. If something goes wrong, inspect the four log files described at the diff --git a/docs/source-en/rst_source/usage/libero.rst b/docs/source-en/rst_source/usage/libero.rst index b2f7682b..7e90c120 100644 --- a/docs/source-en/rst_source/usage/libero.rst +++ b/docs/source-en/rst_source/usage/libero.rst @@ -116,7 +116,7 @@ What runs where transports (HTTP or socket). It returns only the top compressed PNG mask. - **toolkit** (``robots/libero/toolkit.py``) — defines the tools the LLM can call: ``pi0_pick`` (fed to Pi0.5), ``move_to``, - ``rotate_wrist``, ``back_project``, ``view_driver_state``, + ``rotate_wrist``, ``back_project``, ``view_env_state``, ``finish``, … Tools the planner can call @@ -147,7 +147,7 @@ Physical action tools advance the environment and record new state and images. coordinates. - ``segment(prompt=... / point=..., ...)`` — use SAM3 to segment an existing image with a text or point prompt. -- ``view_driver_state(step=-1)`` — read a recorded state and its embedded +- ``view_env_state(step=-1)`` — read a recorded state and its embedded observation images. Step ``0`` is initial; ``-1`` is latest. - ``view_camera_meta(camera=..., step=-1)`` — read camera metadata for a recorded step. Step ``-1`` is latest. diff --git a/docs/source-zh/rst_source/development/add_primitive.rst b/docs/source-zh/rst_source/development/add_primitive.rst index 5d475ea3..59022970 100644 --- a/docs/source-zh/rst_source/development/add_primitive.rst +++ b/docs/source-zh/rst_source/development/add_primitive.rst @@ -41,7 +41,7 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 ``self._env.step(...)``，并返回一个简短的日志字典。 为该方法加上 :func:`~rpent.tools.toolkit.updatestate` 装饰器， - toolkit 会在其执行后自动重新渲染状态（``get_state``）： + toolkit 会在其执行后自动重新渲染状态（``get_env_state``）： .. code-block:: python @@ -54,7 +54,7 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} - 只读工具（``view_driver_state``、``back_project``、``segment`` 等） + 只读工具（``view_env_state``、``back_project``、``segment`` 等） 无需装饰——toolkit 会跳过它们的状态捕获。 2. **添加工具定义。** 在 ``robots//tools.py`` 的 ``TOOLS_SPEC`` 中新增一项： @@ -159,7 +159,7 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 - **工具只返回简短的字典。** 返回值会以文本形式提供给 LLM；图像、深度数据和 其他大型观测应通过 ``EnvState.save`` 保存；``EnvState`` 会把每个逻辑基础 文件名自动加入其持有的 ``StepRecord.artifacts`` 集合。图像通过 - ``view_driver_state`` 提供，几何数据通过环境工具访问，不返回原始路径。 + ``view_env_state`` 提供，几何数据通过环境工具访问，不返回原始路径。 - **安全限制由 ``env_server`` 强制执行。** LLM 可能使用任意参数调用工具， 因此工作空间边界和安全限制不能只依赖 toolkit。 diff --git a/docs/source-zh/rst_source/development/add_robot.rst b/docs/source-zh/rst_source/development/add_robot.rst index 394d9e90..e646a37d 100644 --- a/docs/source-zh/rst_source/development/add_robot.rst +++ b/docs/source-zh/rst_source/development/add_robot.rst @@ -207,7 +207,7 @@ toolkit 模块通常包含四部分： **工具定义和处理函数** 包括模块级的 ``TOOLS_SPEC`` 列表（列表元素采用 Anthropic API 的工具定义格式，包含 ``name``、``description`` 和 ``input_schema``），以及 toolkit 引用的模块级函数，例如 -``view_driver_state``、``back_project`` 和 ``finish``。 +``view_env_state``、``back_project`` 和 ``finish``。 **每步状态 dump** —— ``dump_state(driver, env_state, log)`` 通过 ``env_state.record_step(...)`` 创建由 ``EnvState`` 持有的步骤，并取得分配的 @@ -223,7 +223,7 @@ step index；该 ``StepRecord`` 会被立即追加并提交。大型观测通过 中的方法名为 ``init_primitives_clean``；它会调用 ``EnvState.reset()``、构造 原语并 dump 第 0 步）, - 用 ``self.add_tool(name, spec, handler)`` 注册每个工具。无状态的读取工具 - （如 ``view_driver_state``、``finish``）直接绑定模块级函数；原语工具通过 + （如 ``view_env_state``、``finish``）直接绑定模块级函数；原语工具通过 ``_step(name, **kwargs)`` 调用。``_step`` 使用 ``getattr(self._primitives, name)(**kwargs)`` 调用 driver 方法并重新渲染状态； - 重写 ``close()``，通过 ``EnvState`` 保存 agent 侧剩余工件（例如 @@ -244,7 +244,7 @@ primitives 的 ``__init__``。其中通常包含 每个用 ``self.add_tool(...)`` 注册的工具都会暴露给所有 planner。 - 环境侧的返回值必须可 pickle，且不包含 torch 对象。 - 每个原语工具执行后要 dump 一次新的状态快照, 这样下一次 - ``view_driver_state`` 看到的是动作后的世界。 + ``view_env_state`` 看到的是动作后的世界。 - ``dump_state`` 是 Agent 获取环境状态的唯一数据来源；任何新的模态 （例如触觉、力）都通过它提供。 diff --git a/docs/source-zh/rst_source/development/interfaces.rst b/docs/source-zh/rst_source/development/interfaces.rst index d28c3d34..822b02fa 100644 --- a/docs/source-zh/rst_source/development/interfaces.rst +++ b/docs/source-zh/rst_source/development/interfaces.rst @@ -87,7 +87,7 @@ Planner 需要回传相机图时可设 ``_image_bytes`` 等字段。 基类已注册公共文件工具；子类 ``super().__init__()`` 后追加本环境工具即可。逐步状态与 -``view_driver_state`` 见 :doc:`add_primitive`。 +``view_env_state`` 见 :doc:`add_primitive`。 进程间通信 ---------- diff --git a/docs/source-zh/rst_source/quickstart.rst b/docs/source-zh/rst_source/quickstart.rst index 9d870b55..2b153eb0 100644 --- a/docs/source-zh/rst_source/quickstart.rst +++ b/docs/source-zh/rst_source/quickstart.rst @@ -97,7 +97,7 @@ LIBERO-PRO 仿真资源。下面以 LIBERO-PRO 和 ``claude_code`` planner 3. 启用 Dashboard 后，智能体的输出、相机视图、动作时间线和片段回放也会实时显示在 Dashboard 中。 4. 默认输出目录为 ``logs/__t_s/``，其中包含 ``transcript_*.json``\ （运行记录）、``states.json``\ （带版本号的 ``EnvState`` 清单）、``recipe_*.jsonl``\ （动作序列）和 ``episode.mp4``\ （回合录像）。每步工件文件采用至少两位、零填充的步骤前缀扁平命名，并由 ``EnvState`` 在内部管理。 -通过 Dashboard 或 ``view_driver_state(step=-1)`` 查看最终状态；其顶层 +通过 Dashboard 或 ``view_env_state(step=-1)`` 查看最终状态；其顶层 ``libero_terminated`` 即为基准任务结果。``states.json`` 是 ``EnvState`` 的内部 存储，调用方不应直接解析。也可以打开 ``episode.mp4`` 复核运行过程。 出问题时，参考 :doc:`installation` 页底部提到的四份日志文件。 diff --git a/docs/source-zh/rst_source/usage/libero.rst b/docs/source-zh/rst_source/usage/libero.rst index b66e06ac..940de6da 100644 --- a/docs/source-zh/rst_source/usage/libero.rst +++ b/docs/source-zh/rst_source/usage/libero.rst @@ -112,7 +112,7 @@ LIBERO-PRO 核心套件一览 排名第一的压缩 PNG mask。 - **toolkit（工具集）** （``robots/libero/toolkit.py``）—— 定义 LLM 能调用的工具：``pi0_pick``（交给 Pi0.5）、``move_to``、``rotate_wrist``、 - ``back_project``、``view_driver_state``、``finish``… + ``back_project``、``view_env_state``、``finish``… Planner 能调用的工具 -------------------- @@ -141,7 +141,7 @@ LIBERO 工具分为物理动作工具和只读工具。 - ``back_project(row, col, ...)`` —— 将图像像素反投影到世界坐标。 - ``segment(prompt=... / point=..., ...)`` —— 通过 SAM3 对已有图像进行文本或 点提示分割。 -- ``view_driver_state(step=-1)`` —— 读取已记录的状态和内嵌观测图像；第 0 步为 +- ``view_env_state(step=-1)`` —— 读取已记录的状态和内嵌观测图像；第 0 步为 初始状态，``-1`` 表示最新状态。 - ``view_camera_meta(camera=..., step=-1)`` —— 读取指定步骤的相机元数据； ``-1`` 表示最新状态。 diff --git a/robots/libero/guides/env_calibration.md b/robots/libero/guides/env_calibration.md index 886fd711..c3eb4203 100644 --- a/robots/libero/guides/env_calibration.md +++ b/robots/libero/guides/env_calibration.md @@ -7,7 +7,7 @@ owns the environment server and the `EnvState` lifecycle. Do not issue file-based driver commands, inspect observation storage directly, or read BDDL files for coordinates. -Start with `view_driver_state({"step": 0})`. It returns the initial robot state, +Start with `view_env_state({"step": 0})`. It returns the initial robot state, top-level task language, logical observation references, and embedded camera images. Use `back_project` or `segment` for geometry and `view_camera_meta` for calibration. Step `-1` selects the latest record. @@ -21,7 +21,7 @@ Measured 2026-05-20 on `libero_10_with_mug` t0 (LIVING_ROOM frame) and t8 Each task scene uses one of the table fixtures below, which sets the entire world-frame z origin. The OSC workspace and all pick/place altitudes shift accordingly. **Check `state.robot0_eef_pos[2]` in the result of -`view_driver_state({"step": 0})` and branch on it.** Do not read BDDL files for +`view_env_state({"step": 0})` and branch on it.** Do not read BDDL files for this; use runtime state and visual evidence. | Fixture | eef home z | Table top z | Used by tasks | @@ -150,7 +150,7 @@ limit 1.15). My libero_10 t0 used z=0.95 for travel — safe and consistent. Each calibration motion returns its state, command result, elapsed time, and embedded observation. Use that tool result immediately. To revisit a recorded -step, call `view_driver_state({"step": N})`; use `-1` for the latest step. +step, call `view_env_state({"step": N})`; use `-1` for the latest step. The internal `states.json` file is a versioned manifest owned by `EnvState`, not a list for manual indexing. Calibration analysis should use structured tool @@ -166,5 +166,5 @@ move_to({"xyz": [-0.20, 0.10, 0.65], "gripper": -1, "tol": 0.008, "step_clip": 0.010, "max_steps": 80}) # Inspect the returned result for final_eef_pos and final_dist_m. -# Call view_driver_state({"step": -1}) only when the latest state is needed again. +# Call view_env_state({"step": -1}) only when the latest state is needed again. ``` diff --git a/robots/libero/guides/pro_hybrid_guide.md b/robots/libero/guides/pro_hybrid_guide.md index 7da40264..7b3054cb 100644 --- a/robots/libero/guides/pro_hybrid_guide.md +++ b/robots/libero/guides/pro_hybrid_guide.md @@ -16,7 +16,7 @@ Call: {"step": 0} ``` -through `view_driver_state`. From the returned tool result: +through `view_env_state`. From the returned tool result: - read top-level `task_language` verbatim; - inspect `state.robot0_eef_pos` to identify the scene frame; @@ -64,7 +64,7 @@ The initial end-effector height distinguishes the principal scene frames: | approximately 0.68 m | living-room table | plates, baskets, pudding | | approximately 1.17 m | kitchen table | stove, cabinet, drawer, microwave | -Use `view_driver_state({"step": 0})["state"]["robot0_eef_pos"][2]` as the +Use `view_env_state({"step": 0})["state"]["robot0_eef_pos"][2]` as the measurement. Then use the matching safe-height guidance from [env_calibration.md](./env_calibration.md). @@ -117,7 +117,7 @@ for coordinates. Long-horizon Pro tasks often move or occlude objects during earlier steps. Before each new pick or placement: -1. call `view_driver_state({"step": -1})` if the previous primitive result is +1. call `view_env_state({"step": -1})` if the previous primitive result is no longer in context; 2. inspect the newest embedded images; 3. re-localize any entity that may have moved; @@ -186,7 +186,7 @@ traceability, not manual file access. ## Quick Checklist - Read strict guide and relevant memory. -- Call `view_driver_state({"step": 0})`. +- Call `view_env_state({"step": 0})`. - Select the scene frame from initial EEF z. - Read top-level `task_language`. - Build the complete perception table. diff --git a/robots/libero/guides/strict_hybrid_guide.md b/robots/libero/guides/strict_hybrid_guide.md index 3518a0a7..417b380e 100644 --- a/robots/libero/guides/strict_hybrid_guide.md +++ b/robots/libero/guides/strict_hybrid_guide.md @@ -22,7 +22,7 @@ Every motion primitive appends a `StepRecord`. The record contains: Artifact storage is private to `EnvState`. Do not construct filenames or read observation files manually. Use the structured tools: -- `view_driver_state`: state, artifact names, log, and embedded images; +- `view_env_state`: state, artifact names, log, and embedded images; - `view_camera_meta`: calibration data for one camera and step; - `back_project`: matching image pixel or region to world coordinates; - `segment`: text- or point-prompted mask plus projected `world_xyz`. @@ -35,7 +35,7 @@ API and should not be indexed directly. ## Camera Roles -`view_driver_state` embeds the best available images for the selected step: +`view_env_state` embeds the best available images for the selected step: - **policy image**: the Pi0-oriented agentview input; - **agentview image**: fixed global view, high resolution when available; @@ -53,7 +53,7 @@ specified in the call. The tool selects the matching world map internally. Before the first manipulation primitive: -1. Call `view_driver_state({"step": 0})`. +1. Call `view_env_state({"step": 0})`. 2. Read the returned top-level `task_language` verbatim. 3. Identify every movable target, destination, support, and relation landmark named by the task. diff --git a/robots/libero/prompts/system.py b/robots/libero/prompts/system.py index 70591e30..d93b8df8 100644 --- a/robots/libero/prompts/system.py +++ b/robots/libero/prompts/system.py @@ -141,7 +141,7 @@ - Call the real structured tools exposed by the runtime. - Use bare tool names in this prompt: `move_to`, `pi0_pick`, `release`, `set_gripper`, `rotate_wrist`, `rotate_pitch`, `move_pose`, `pi0_doubled`, - `view_driver_state`, `view_camera_meta`, `back_project`, `segment`, + `view_env_state`, `view_camera_meta`, `back_project`, `segment`, `read_text_file`, `write_text_file`, `list_dir`, `finish`. - Under some runtimes these same tools may appear namespaced; call the actual tool name shown in your tool list, preserving the same arguments and semantics. @@ -151,7 +151,7 @@ world-map, and metadata artifacts, but storage paths are internal to the runtime. Do not construct or read artifact paths manually. -Use `view_driver_state` to retrieve a state. It embeds the policy image and the +Use `view_env_state` to retrieve a state. It embeds the policy image and the best available agentview and wrist images, preferring high resolution. Use `view_camera_meta`, `back_project`, and `segment` to consume metadata, depth, and world maps. These tools guarantee that the selected camera, resolution, @@ -164,7 +164,7 @@ supersedes any reset/retry wording in the Rules below).""" RULES = """Rule 0 — USE IMAGES. After every primitive tool call, inspect the returned state - and embedded images. If you need a state again, call `view_driver_state`. + and embedded images. If you need a state again, call `view_env_state`. Use agentview for global layout and wrist for close-range geometry. The image is your spatial-reasoning input; the JSON state only gives proprioception + object names. @@ -194,7 +194,7 @@ `pi0_pick.success` (eef-lift + gripper-closure heuristic) is a HINT, not proof — always confirm with the wrist cam before carrying. -Rule 2 — Inspect THEN act. Call `view_driver_state({"step": 0})`, inspect the +Rule 2 — Inspect THEN act. Call `view_env_state({"step": 0})`, inspect the returned high-resolution images, and inspect the relevant memory/guides BEFORE your first primitive. **Your task is the returned `task_language`; read it and obey it verbatim.** This is the authoritative instruction (the BDDL @@ -399,7 +399,7 @@ with older/oracle assumptions; do NOT copy coordinates and do NOT replay stale command lists. Re-derive every coordinate from THIS scene. """, - """INSPECT INITIAL STATE: call `view_driver_state({"step": 0})`; inspect + """INSPECT INITIAL STATE: call `view_env_state({"step": 0})`; inspect `task_language`, object_names, eef pose, the returned agentview and wrist images, and call `view_camera_meta` if needed. Identify ALL target objects, destination surfaces, and relation landmarks named by task_language. @@ -488,7 +488,7 @@ OUTPUT_DISCIPLINE = """- Brief reasoning before each tool call (1-2 sentences): observation → decision. - Don't re-read files already in this session. -- Don't call `view_driver_state` immediately after a primitive tool already +- Don't call `view_env_state` immediately after a primitive tool already returned the new state. - Save the audit BEFORE calling `finish`. - Stop immediately after writing the audit and calling `finish`. Do not chat further.""" diff --git a/robots/libero/prompts/user.py b/robots/libero/prompts/user.py index 63d171ac..a71b2444 100644 --- a/robots/libero/prompts/user.py +++ b/robots/libero/prompts/user.py @@ -11,10 +11,10 @@ MODE = """Inspect the embedded high-resolution images returned by -view_driver_state, then use back_project or segment to localize objects before +view_env_state, then use back_project or segment to localize objects before motion.""" BEGIN = """Read MEMORY.md and the guides, then call -`view_driver_state({"step": 0})` and inspect its embedded images. Localize every +`view_env_state({"step": 0})` and inspect its embedded images. Localize every task-relevant entity before planning and execution.""" diff --git a/robots/libero/toolkit.py b/robots/libero/toolkit.py index 95b9253f..6b94cc25 100644 --- a/robots/libero/toolkit.py +++ b/robots/libero/toolkit.py @@ -39,8 +39,8 @@ def _register_libero_tools(self) -> None: # to its primitive-driver method; @updatestate on the method decides # whether state is captured. state_handlers = { - "view_driver_state": partial( - libero_tools.view_driver_state, state=self._state + "view_env_state": partial( + libero_tools.view_env_state, state=self._state ), "view_camera_meta": partial( libero_tools.view_camera_meta, state=self._state @@ -58,7 +58,7 @@ def _register_libero_tools(self) -> None: continue # spec without a backing primitive method self.add_tool(name, spec, handler) - def get_state( + def get_env_state( self, *, command: dict[str, Any], @@ -91,7 +91,7 @@ def get_state( record.step_idx, e, ) - out = libero_tools.view_driver_state(record.step_idx, state=self._state) + out = libero_tools.view_env_state(record.step_idx, state=self._state) out["agent_elapsed_s"] = elapsed_s if result.get("interrupted"): out.update(result) @@ -117,8 +117,8 @@ def init_primitives_clean( libero_tools.dump_state(primitives, self._state, log=None) self._dashboard_events.emit( ToolResultEvent( - name="view_driver_state", - result=libero_tools.view_driver_state(0, state=self._state), + name="view_env_state", + result=libero_tools.view_env_state(0, state=self._state), ) ) diff --git a/robots/libero/tools.py b/robots/libero/tools.py index ff8cb901..a143a898 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -738,13 +738,13 @@ def _is_primitive_action(name: object) -> bool: """Whether ``name`` is a state-advancing LIBERO primitive. A primitive is any ``@updatestate``-marked method on - :class:`LiberoPrimitives`; read-only tools (``view_driver_state``, + :class:`LiberoPrimitives`; read-only tools (``view_env_state``, ``back_project``, ``segment``, ...) and non-strings read as ``False``. """ if not isinstance(name, str): return False method = getattr(LiberoPrimitives, name, None) - return method is not None and bool(getattr(method, "_captures_state", False)) + return method is not None and bool(getattr(method, "_updates_state", False)) def write_recipe_from_states(state: EnvState, recipe_tag: str) -> str: @@ -1056,7 +1056,7 @@ def _save_observation_artifacts( TOOLS_SPEC = [ { - "name": "view_driver_state", + "name": "view_env_state", "description": ( "Read one recorded state and its observation artifacts. Step -1 " "selects the latest entry. Embeds policy, agentview, and wrist " @@ -1330,7 +1330,7 @@ def _save_observation_artifacts( "selected camera's precomputed world map. Row 0 = top of image, " "col 0 = left. Returns world_xyz in meters.\n\n" "USE THIS to find where an object is in the world — look at " - "the embedded high-resolution image returned by view_driver_state " + "the embedded high-resolution image returned by view_env_state " "to pick a pixel on the target object, then call back_project. " "The default resolution is high (1024x1024). Pass " "resolution='low' only for pixels from the embedded/standard " @@ -1401,7 +1401,7 @@ def _save_observation_artifacts( ] -def view_driver_state(step: int = -1, *, state: EnvState) -> dict: +def view_env_state(step: int = -1, *, state: EnvState) -> dict: try: record = state.get(step) except Exception as exc: diff --git a/rpent/context/prompts/prompt.py b/rpent/context/prompts/prompt.py index b3c9e47b..43e6dad0 100644 --- a/rpent/context/prompts/prompt.py +++ b/rpent/context/prompts/prompt.py @@ -5,7 +5,7 @@ OUTPUT = BulletList([ "Brief reasoning before each tool call (1-2 sentences): observation -> decision.", - "Don't re-read files already in this session. Don't view_driver_state right after a primitive tool already returned the state.", + "Don't re-read files already in this session. Don't view_env_state right after a primitive tool already returned the state.", "Numerical coords in 3 decimals are enough.", "Save artifacts BEFORE calling finish. Stop immediately after writing the audit; do not chat further.", ]) diff --git a/rpent/planner/api_loop.py b/rpent/planner/api_loop.py index 71890c94..f649fd15 100644 --- a/rpent/planner/api_loop.py +++ b/rpent/planner/api_loop.py @@ -709,7 +709,7 @@ def _build_tools(toolkit: Toolkit, *, no_images: bool = False) -> list[Tool]: def read_image(path: str) -> ToolReturn: """Read an explicitly provided local image file as visual input. - Environment observations are already embedded by ``view_driver_state``; + Environment observations are already embedded by ``view_env_state``; this helper is only for other user-selected local files. """ return ToolReturn( @@ -722,7 +722,7 @@ def read_image_text_only(path: str) -> str: """``read_image`` stub for ``--no-images``: acknowledge, send no bytes.""" return ( f"{path} exists, but image input is disabled (--no-images, text-only " - "model). Reason from textual state instead: view_driver_state, " + "model). Reason from textual state instead: view_env_state, " "back_project, and the numeric fields in tool results." ) diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 22d30871..9c424020 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -32,10 +32,10 @@ class ToolCancelled(Exception): def updatestate(func): """Mark a tool handler as one that advances environment state. - The toolkit captures a fresh observation (:meth:`Toolkit.get_state`) + The toolkit captures a fresh observation (:meth:`Toolkit.get_env_state`) after a handler carrying this marker runs (or raises). Apply it to primitive-driver methods that move the robot; read-only tools (file IO, - ``view_driver_state``, ``back_project``, ``segment``, ...) are left + ``view_env_state``, ``back_project``, ``segment``, ...) are left unmarked and skip state capture. The marker is read off the underlying function, so registering a bound @@ -44,11 +44,11 @@ def updatestate(func): wrapping a marked method is treated as read-only (intentional -- see the LIBERO ``segment`` tool). """ - func._captures_state = True + func._updates_state = True return func -def _captures_state(handler: Callable[..., Any]) -> bool: +def _updates_state(handler: Callable[..., Any]) -> bool: """Whether ``handler`` was marked with :func:`updatestate`. Resolves through ``__func__`` so bound methods (the common case for @@ -57,7 +57,7 @@ def _captures_state(handler: Callable[..., Any]) -> bool: do not delegate, so undecorated read-only handlers read as ``False``. """ target = getattr(handler, "__func__", handler) - return bool(getattr(target, "_captures_state", False)) + return bool(getattr(target, "_updates_state", False)) @dataclass @@ -174,7 +174,7 @@ def add_tool( a result dict. Decorate state-advancing primitives with :func:`updatestate`. """ - self._tools[name] = (spec, handler, _captures_state(handler)) + self._tools[name] = (spec, handler, _updates_state(handler)) def _register_common_tools(self) -> None: """Register the file/IO tools shared by every run.""" @@ -236,7 +236,7 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: result_dict = result if isinstance(result, dict) else {"value": result} command = {"action": name, **input_dict} try: - captured = self.get_state( + captured = self.get_env_state( command=command, result=result_dict, elapsed_s=elapsed_s, @@ -264,7 +264,7 @@ def _finish_tool(self, name: str, result: Any) -> ToolResult: self._dashboard_events.emit(ToolResultEvent(name=name, result=result)) return ToolResult(name=name, result=result) - def get_state( + def get_env_state( self, *, command: dict[str, Any], diff --git a/scripts/codex_proxy/litellm_callbacks.py b/scripts/codex_proxy/litellm_callbacks.py index 23aca5f2..5d54e05b 100644 --- a/scripts/codex_proxy/litellm_callbacks.py +++ b/scripts/codex_proxy/litellm_callbacks.py @@ -92,7 +92,7 @@ def _extract_mcp_namespace_map(data: dict) -> dict[str, str]: Example:: {"back_project": "mcp__rpent", - "view_driver_state": "mcp__rpent"} + "view_env_state": "mcp__rpent"} """ tools = data.get("tools") if not isinstance(tools, list): From 9bbb395737271e37ca143cba7010861c473722f2 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Thu, 6 Aug 2026 03:12:30 +0000 Subject: [PATCH 06/25] refactor(toolkit): publish dashboard steps as StepRecords, introduce _FRAME_ARTIFACTS Signed-off-by: Jiaxing Qiu --- robots/libero/toolkit.py | 25 ++++++------- robots/libero/tools.py | 4 +-- rpent/cli/dashboard.py | 1 - rpent/dashboard/events.py | 10 ++++++ rpent/dashboard/state.py | 74 ++++++++++++++++++++++++++++++++++----- rpent/tools/state.py | 4 +++ rpent/tools/toolkit.py | 23 ++++++++++-- 7 files changed, 112 insertions(+), 29 deletions(-) diff --git a/robots/libero/toolkit.py b/robots/libero/toolkit.py index 6b94cc25..23a315cb 100644 --- a/robots/libero/toolkit.py +++ b/robots/libero/toolkit.py @@ -9,7 +9,7 @@ from typing import Any from robots.libero import tools as libero_tools -from rpent.dashboard.events import DashboardEventSink, ToolResultEvent +from rpent.dashboard.events import DashboardEventSink from rpent.tools.state import EnvState from rpent.tools.toolkit import Toolkit from rpent.utils.logging import get_logger, get_output_dir @@ -18,6 +18,11 @@ class LiberoToolkit(Toolkit): """Toolkit for the LIBERO environment.""" + _FRAME_ARTIFACTS = { + "camera": "agentview.png", + "wrist": "wrist.png", + } + def __init__( self, *, @@ -72,19 +77,17 @@ def get_env_state( self._state, log={"command": command, "result": result, "elapsed_s": elapsed_s}, ) - action_video_name = None if self._dashboard_events.enabled: try: frames = self._primitives.frame_slice(frame_start) if frames: candidate = f"action_{command['action']}.mp4" - if self._state.save( + self._state.save( candidate, frames, step=record.step_idx, fps=20, - ): - action_video_name = candidate + ) except Exception as e: get_logger("libero_toolkit").warning( "failed to save action clip for step %s: %s", @@ -95,8 +98,6 @@ def get_env_state( out["agent_elapsed_s"] = elapsed_s if result.get("interrupted"): out.update(result) - if action_video_name is not None: - out["action_video_artifact"] = action_video_name return out def init_primitives_clean( @@ -114,15 +115,9 @@ def init_primitives_clean( primitives.reset() primitives.start_recording() self._action_frame_cursor = primitives.recorded_frame_count() - libero_tools.dump_state(primitives, self._state, log=None) - self._dashboard_events.emit( - ToolResultEvent( - name="view_env_state", - result=libero_tools.view_env_state(0, state=self._state), - ) - ) - + record = libero_tools.dump_state(primitives, self._state, log=None) self._primitives = primitives + self._publish_step(record) def close(self) -> None: """Flush the agent-side video buffer through ``EnvState``.""" diff --git a/robots/libero/tools.py b/robots/libero/tools.py index a143a898..2f66eb76 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -844,7 +844,7 @@ def dump_state( result=log.get("result"), elapsed_s=log.get("elapsed_s"), extras={ - "libero_terminated": primitives.env.episode_terminated, + "terminated": primitives.env.episode_terminated, "episode_truncated": primitives.env.episode_truncated, "task_language": primitives.env.get_task_language(), }, @@ -1415,7 +1415,7 @@ def view_env_state(step: int = -1, *, state: EnvState) -> dict: "artifacts": sorted(record.artifacts), } out["task_language"] = extras.get("task_language") - out["libero_terminated"] = extras.get("libero_terminated") + out["libero_terminated"] = extras.get("terminated") out["episode_truncated"] = extras.get("episode_truncated") out["log"] = { "command": record.command, diff --git a/rpent/cli/dashboard.py b/rpent/cli/dashboard.py index c63e5c49..9a5f324c 100644 --- a/rpent/cli/dashboard.py +++ b/rpent/cli/dashboard.py @@ -155,7 +155,6 @@ def _run_dashboard_task( toolkit = get_toolkit( args.env_name, primitives_kwargs=primitives_kwargs, - video_path=str(output_dir / "episode.mp4"), dashboard_events=state, ) planner = build_planner( diff --git a/rpent/dashboard/events.py b/rpent/dashboard/events.py index 5784624d..0c6cbfae 100644 --- a/rpent/dashboard/events.py +++ b/rpent/dashboard/events.py @@ -39,6 +39,15 @@ class ToolResultEvent: result: Any +@dataclass(frozen=True, slots=True) +class StepRecordEvent: + """Publish one recorded environment step and its artifact context.""" + + record: Any + env_state: Any + frame_artifacts: dict[str, str] + + @dataclass(frozen=True, slots=True) class RunStartedEvent: """Mark startup complete and the agent run active.""" @@ -49,6 +58,7 @@ class RunStartedEvent: | UsageEvent | RuntimeStatusEvent | ToolResultEvent + | StepRecordEvent | RunStartedEvent ) diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py index d4075cfa..ce2bb920 100644 --- a/rpent/dashboard/state.py +++ b/rpent/dashboard/state.py @@ -7,12 +7,13 @@ import uuid from dataclasses import dataclass, replace from pathlib import Path -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from rpent.dashboard.events import ( DashboardEvent, RunStartedEvent, RuntimeStatusEvent, + StepRecordEvent, ToolResultEvent, TranscriptEvent, UsageEvent, @@ -26,6 +27,9 @@ UnknownDashboardMessageError, ) +if TYPE_CHECKING: + from rpent.tools.state import EnvState, StepRecord + RUNTIME_STATUSES = {"pending", "starting", "ready", "failed"} TERMINAL_RUN_STATES = {"succeeded", "failed", "cancelled"} _PLANNER_ACTIVITIES = {"starting", "idle", "busy", "ended"} @@ -121,6 +125,8 @@ def __init__( self._timeline: list[dict[str, Any]] = [] self._frames: dict[str, bytes] = {} self._frame_idx = -1 + self.env_state: EnvState | None = None + self.frame_artifacts: dict[str, str] = {} self._accepting_input = False self._planner_activity: PlannerActivity = "starting" self._interrupt_requested = False @@ -307,6 +313,8 @@ def _begin_task_locked( self._timeline = [] self._frames = {} self._frame_idx = -1 + self.env_state = None + self.frame_artifacts = {} self._accepting_input = False self._planner_activity = "starting" self._interrupt_requested = False @@ -538,6 +546,11 @@ def emit(self, event: DashboardEvent) -> None: if isinstance(event, ToolResultEvent): self._apply_tool_result(event) return + if isinstance(event, StepRecordEvent): + self.env_state = event.env_state + self.frame_artifacts = dict(event.frame_artifacts) + self.on_step(event.record) + return if isinstance(event, RunStartedEvent): self._start() return @@ -632,6 +645,46 @@ def _action_video_from_result( return None return path if path.exists() else None + def on_step(self, record: StepRecord) -> None: + """Project one recorded environment step into frames and timeline.""" + self._update_step_frames(record) + command = record.command + if not isinstance(command, dict) or not command.get("action"): + return + terminated = bool(record.extras.get("terminated")) + action_video = next( + (name for name in sorted(record.artifacts) if name.endswith(".mp4")), + None, + ) + item = { + "step": record.step_idx, + "action": str(command.get("action")), + "args": {key: value for key, value in command.items() if key != "action"}, + "result": record.result, + "elapsed_s": record.elapsed_s, + "terminated": terminated, + "action_video_artifact": action_video, + "has_action_video": action_video is not None, + } + with self._lock: + self._timeline.append(item) + self._terminated = self._terminated or terminated + + def _update_step_frames(self, record: StepRecord) -> None: + """Load dashboard frame bytes from the step's canonical artifacts.""" + env_state = self.env_state + if env_state is None: + return + frames: dict[str, bytes] = {} + for kind, artifact in self.frame_artifacts.items(): + if kind not in self._frame_names or artifact not in record.artifacts: + continue + try: + frames[kind] = env_state.load_bytes(artifact, step=record.step_idx) + except FileNotFoundError: + continue + self._update_frames(step=record.step_idx, frames=frames) + def _resolve_output_path(self, value: Any) -> Path: path = Path(value) if path.is_absolute() or path.is_relative_to(self.output_dir): @@ -783,19 +836,24 @@ def frame(self, kind: str) -> bytes | None: return self._frames.get(kind) def action_video(self, step: int) -> bytes | None: + env_state = self.env_state with self._lock: + artifact = None + raw_path = None for item in self._timeline: if int(item.get("step", -1)) != int(step): continue artifact = item.get("action_video_artifact") - if artifact: - video_path = self.output_dir / f"{int(step):02d}_{artifact}" - return video_path.read_bytes() if video_path.exists() else None raw_path = item.get("action_video_path") - if not raw_path: - return None - video_path = Path(raw_path) - return video_path.read_bytes() if video_path.exists() else None + break + if artifact and env_state is not None: + try: + return env_state.load_bytes(artifact, step=int(step)) + except FileNotFoundError: + return None + if raw_path: + video_path = Path(raw_path) + return video_path.read_bytes() if video_path.exists() else None return None def video(self) -> bytes | None: diff --git a/rpent/tools/state.py b/rpent/tools/state.py index 3d43677b..a059cb77 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -175,6 +175,10 @@ def latest_step(self) -> int | None: return None return self._steps[-1].step_idx + def latest_record(self) -> StepRecord | None: + """Return the most recently recorded step (live reference, no copy).""" + return self._steps[-1] if self._steps else None + def _resolve_read_step(self, step: int | None) -> int | None: if step is None: return None diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 9c424020..76882f81 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -13,11 +13,14 @@ import traceback from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar -from rpent.dashboard.events import DashboardEventSink, ToolResultEvent +from rpent.dashboard.events import DashboardEventSink, StepRecordEvent from rpent.utils.templates import substitute +if TYPE_CHECKING: + from rpent.tools.state import StepRecord + @dataclass(slots=True) class _ToolOperation: @@ -235,6 +238,7 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: elapsed_s = round(time.perf_counter() - started, 2) result_dict = result if isinstance(result, dict) else {"value": result} command = {"action": name, **input_dict} + record: StepRecord | None = None try: captured = self.get_env_state( command=command, @@ -248,11 +252,15 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: "error", f"failed to capture state after {name}: {e}" ) captured.setdefault("traceback", traceback.format_exc()) + else: + record = self._state.latest_record() result = captured if failed: result.setdefault("error", result_dict["error"]) if "traceback" in result_dict: result.setdefault("traceback", result_dict["traceback"]) + if record is not None: + self._publish_step(record) return self._finish_tool(name, result) finally: @@ -260,8 +268,17 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: self._active_operation = None operation.done_event.set() + def _publish_step(self, record: StepRecord) -> None: + """Publish one recorded environment step to the dashboard sink.""" + self._dashboard_events.emit( + StepRecordEvent( + record=record, + env_state=self._state, + frame_artifacts=dict(getattr(type(self), "_FRAME_ARTIFACTS", {})), + ) + ) + def _finish_tool(self, name: str, result: Any) -> ToolResult: - self._dashboard_events.emit(ToolResultEvent(name=name, result=result)) return ToolResult(name=name, result=result) def get_env_state( From 1d354211f5905b83709e06960f98866453685d77 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Thu, 6 Aug 2026 08:10:21 +0000 Subject: [PATCH 07/25] chore: remove robot-specific common tools Signed-off-by: Jiaxing Qiu --- rpent/tools/common.py | 88 ------------------------------------------- 1 file changed, 88 deletions(-) diff --git a/rpent/tools/common.py b/rpent/tools/common.py index 54e02ce9..fd959566 100644 --- a/rpent/tools/common.py +++ b/rpent/tools/common.py @@ -3,16 +3,10 @@ import os from pathlib import Path -from typing import Any - -import numpy as np from rpent.utils.config import get_repo_root from rpent.utils.logging import get_output_dir -_BACKPROJECT_RADIUS = 6 -_DEPTH_BAND_M = 0.02 - TOOLS_SPEC: list[dict] = [ { "name": "read_text_file", @@ -140,88 +134,6 @@ def finish(status: str, summary: str) -> dict: return {"_finish": True, "status": status, "summary": summary} -def backproject_points(K, rows, cols, depths) -> np.ndarray: - """Back-project pixel coords + depths to camera-frame XYZ, shape (N, 3).""" - K = np.asarray(K, dtype=np.float64) - rows = np.asarray(rows, dtype=np.float64) - cols = np.asarray(cols, dtype=np.float64) - depths = np.asarray(depths, dtype=np.float64) - fx, fy = K[0, 0], K[1, 1] - cx, cy = K[0, 2], K[1, 2] - return np.stack( - [(cols - cx) * depths / fx, (rows - cy) * depths / fy, depths], axis=1 - ) - - -def robust_surface_centroid( - depth: np.ndarray, - K, - T_base_cam, - row: int, - col: int, - *, - radius: int = _BACKPROJECT_RADIUS, - band: float = _DEPTH_BAND_M, -) -> dict: - """Back-project a pixel neighbourhood to a robust 3D point. - - Back-projects every valid pixel in a ``(2*radius+1)`` window, keeps those on - the dominant surface (depth within ``band`` of the window median, rejecting - background / table / dropouts), and returns the median point + diagnostics. - Returns world ``xyz`` when ``T_base_cam`` is given, otherwise camera-frame - ``xyz_cam``. - """ - row, col = int(row), int(col) - radius = max(0, int(radius)) - height, width = depth.shape[:2] - if not (0 <= row < height and 0 <= col < width): - return { - "error": ( - f"pixel ({row},{col}) out of bounds; image is {height}x{width}" - ) - } - row_start, row_end = max(0, row - radius), min(height, row + radius + 1) - col_start, col_end = max(0, col - radius), min(width, col + radius + 1) - rows, cols = np.mgrid[row_start:row_end, col_start:col_end] - depths = depth[row_start:row_end, col_start:col_end].reshape(-1).astype( - np.float64 - ) - rows = rows.reshape(-1).astype(np.float64) - cols = cols.reshape(-1).astype(np.float64) - valid = np.isfinite(depths) & (depths > 0) - if not np.any(valid): - return {"error": f"no valid depth near ({row},{col}); pick another pixel"} - depths, rows, cols = depths[valid], rows[valid], cols[valid] - median_depth = float(np.median(depths)) - surface = np.abs(depths - median_depth) <= band - depths, rows, cols = depths[surface], rows[surface], cols[surface] - if depths.size == 0: - return {"error": f"no dominant surface depth near ({row},{col})"} - - camera_points = backproject_points(K, rows, cols, depths) - camera_point = np.median(camera_points, axis=0) - out: dict[str, Any] = { - "pixel": [row, col], - "radius": radius, - "n_points": int(camera_points.shape[0]), - "depth_m": round(median_depth, 4), - "xyz_cam": [round(float(value), 4) for value in camera_point], - } - if T_base_cam is not None: - transform = np.asarray(T_base_cam, dtype=np.float64) - base_points = camera_points @ transform[:3, :3].T + transform[:3, 3] - base_point = np.median(base_points, axis=0) - out["xyz"] = [round(float(value), 4) for value in base_point] - out["xy_spread_m"] = round( - float(np.hypot(*base_points[:, :2].std(axis=0))), 4 - ) - else: - out["xy_spread_m"] = round( - float(np.hypot(*camera_points[:, :2].std(axis=0))), 4 - ) - return out - - TOOL_HANDLERS: dict = { "read_text_file": read_text_file, "write_text_file": write_text_file, From ce13adbb9a424ef6c81548f09e20fcd33ed5ece3 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Thu, 6 Aug 2026 08:31:12 +0000 Subject: [PATCH 08/25] docs: nits Signed-off-by: Jiaxing Qiu --- .../rst_source/development/add_primitive.rst | 11 ++++++----- docs/source-en/rst_source/development/add_robot.rst | 2 +- .../rst_source/development/add_primitive.rst | 7 ++++--- docs/source-zh/rst_source/development/add_robot.rst | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/source-en/rst_source/development/add_primitive.rst b/docs/source-en/rst_source/development/add_primitive.rst index 947a38bf..a48f5fba 100644 --- a/docs/source-en/rst_source/development/add_primitive.rst +++ b/docs/source-en/rst_source/development/add_primitive.rst @@ -43,8 +43,9 @@ Adding a scripted primitive usually involves two steps: the tool-call arguments, performs the work, usually through one or more ``self._env.step(...)`` calls, and returns a small log ``dict``. - Mark the method with :func:`~rpent.tools.toolkit.updatestate` so the - toolkit re-renders state (``get_env_state``) automatically after it runs: + You should mark all methods that change environment state with + :func:`~rpent.tools.toolkit.updatestate`, so the toolkit re-renders + state (``get_env_state``) automatically after it runs: .. code-block:: python @@ -57,8 +58,9 @@ Adding a scripted primitive usually involves two steps: self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} + Failing to do so will result in the next tool call seeing stale state information. Read-only tools (``view_env_state``, ``back_project``, ``segment``, - ...) are simply left unmarked -- the toolkit skips state capture for them. + ...) can be left unmarked -- the toolkit skips state capture for them. 2. **Add the tool schema.** Add an entry to ``TOOLS_SPEC`` in ``robots//tools.py``: @@ -78,8 +80,7 @@ Adding a scripted primitive usually involves two steps: Once both exist, the toolkit registers the tool automatically: it iterates ``TOOLS_SPEC`` and binds each spec to the matching primitive-driver method -(e.g. ``getattr(self._primitives, name)``); ``@updatestate`` decides whether -state is captured -- no explicit ``add_tool`` call is needed. +(e.g. ``getattr(self._primitives, name)``). After these steps, the ``api``, ``claude_code``, and ``codex`` planners can all call the primitive without any other code changes. diff --git a/docs/source-en/rst_source/development/add_robot.rst b/docs/source-en/rst_source/development/add_robot.rst index b3578486..d6e56ecd 100644 --- a/docs/source-en/rst_source/development/add_robot.rst +++ b/docs/source-en/rst_source/development/add_robot.rst @@ -234,7 +234,7 @@ filenames rather than maintaining a parallel observation index. **Toolkit class** — subclass ``rpent.tools.toolkit.Toolkit``: -- build the primitive driver in ``__init__`` through a custom initialization +- build the primitives in ``__init__`` through a custom initialization helper (named ``init_primitives_clean`` in LIBERO; it calls ``EnvState.reset()``, constructs the primitives, and dumps step 0), - register each tool with ``self.add_tool(name, spec, handler)`` — stateless diff --git a/docs/source-zh/rst_source/development/add_primitive.rst b/docs/source-zh/rst_source/development/add_primitive.rst index 59022970..2effeb52 100644 --- a/docs/source-zh/rst_source/development/add_primitive.rst +++ b/docs/source-zh/rst_source/development/add_primitive.rst @@ -40,7 +40,8 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 一个方法。该方法接收工具调用的参数，执行一次或多次 ``self._env.step(...)``，并返回一个简短的日志字典。 - 为该方法加上 :func:`~rpent.tools.toolkit.updatestate` 装饰器， + 您需要为所有可能改变环境状态的方法加上 + :func:`~rpent.tools.toolkit.updatestate` 装饰器， toolkit 会在其执行后自动重新渲染状态（``get_env_state``）： .. code-block:: python @@ -54,6 +55,7 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} + 若不这样做，之后的工具调用可能将看到过时的环境状态信息。 只读工具（``view_env_state``、``back_project``、``segment`` 等） 无需装饰——toolkit 会跳过它们的状态捕获。 @@ -73,8 +75,7 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 } 两者就位后，toolkit 会自动注册该工具：它遍历 ``TOOLS_SPEC``，把每个定义 -绑定到对应的 primitive driver 方法（如 ``getattr(self._primitives, name)``）， -由 ``@updatestate`` 决定是否捕获状态——无需显式调用 ``add_tool``。 +绑定到对应的 primitive 方法（如 ``getattr(self._primitives, name)``）。 完成以上步骤后，``api``、``claude_code`` 和 ``codex`` 三种 planner 都可以调用该工具，无需修改其他代码。 diff --git a/docs/source-zh/rst_source/development/add_robot.rst b/docs/source-zh/rst_source/development/add_robot.rst index e646a37d..14eddb67 100644 --- a/docs/source-zh/rst_source/development/add_robot.rst +++ b/docs/source-zh/rst_source/development/add_robot.rst @@ -219,7 +219,7 @@ step index；该 ``StepRecord`` 会被立即追加并提交。大型观测通过 **Toolkit 类** 继承 ``rpent.tools.toolkit.Toolkit``： -- 在 ``__init__`` 中通过自定义的初始化辅助方法构建 primitive driver（LIBERO +- 在 ``__init__`` 中通过自定义的初始化辅助方法构建 primitives（LIBERO 中的方法名为 ``init_primitives_clean``；它会调用 ``EnvState.reset()``、构造 原语并 dump 第 0 步）, - 用 ``self.add_tool(name, spec, handler)`` 注册每个工具。无状态的读取工具 From 6851f03bebc75bf740034f09f73abb8cc428b5ad Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Thu, 6 Aug 2026 09:10:58 +0000 Subject: [PATCH 09/25] docs: precise changeset Signed-off-by: Jiaxing Qiu --- robots/libero/guides/env_calibration.md | 37 +- robots/libero/guides/pro_hybrid_guide.md | 621 ++++++++++++---- robots/libero/guides/strict_hybrid_guide.md | 775 +++++++++++++++----- robots/libero/prompts/system.py | 81 +- robots/libero/prompts/user.py | 9 +- 5 files changed, 1114 insertions(+), 409 deletions(-) diff --git a/robots/libero/guides/env_calibration.md b/robots/libero/guides/env_calibration.md index c3eb4203..8b3a6240 100644 --- a/robots/libero/guides/env_calibration.md +++ b/robots/libero/guides/env_calibration.md @@ -2,15 +2,12 @@ ## Current LIBERO MCP Runtime Contract -Use this file as a calibration reference for structured-tool runs. The runner -owns the environment server and the `EnvState` lifecycle. Do not issue -file-based driver commands, inspect observation storage directly, or read BDDL -files for coordinates. - -Start with `view_env_state({"step": 0})`. It returns the initial robot state, -top-level task language, logical observation references, and embedded camera -images. Use `back_project` or `segment` for geometry and `view_camera_meta` for -calibration. Step `-1` selects the latest record. +Use this file as a calibration reference only. For current MCP-based runs, use +structured MCP tools, do not issue file-based protocol commands, and do not +manually manage `env_server.py`. Do not read BDDL files or hidden task +definition files to infer coordinates. Start with +`view_env_state({"step": 0})`; localize objects from `agentview_high.png` or +`wrist_high.png` through `back_project` or `segment`. Measured 2026-05-20 on `libero_10_with_mug` t0 (LIVING_ROOM frame) and t8 (KITCHEN frame). All probes use `move_to` with `gripper=-1` and tight @@ -133,7 +130,7 @@ limit 1.15). My libero_10 t0 used z=0.95 for travel — safe and consistent. 4. **`set_gripper` after a stalled `move_to` is unsafe.** The previous t0 attempt closed the gripper above the bottle (eef stalled high) then opened it; this returned ok but the env was effectively desynced. - Treat any `move_to` with `final_dist_m > 0.02` as a failure and + Treat any `move_to` with `log.result.final_dist_m > 0.02` as a failure and recover (back to safe altitude, re-plan) before proceeding. 5. **Drop height matters for basket tasks.** Release at eef z=0.58 in LIVING_ROOM frame caused basket displacement Δ≈4–5 cm; z=0.53 keeps @@ -146,25 +143,23 @@ limit 1.15). My libero_10 t0 used z=0.95 for travel — safe and consistent. `move_to` + `set_gripper` (last resort; unreliable for objects <6 cm — see `resources/libero/memory/feedback_scripted_pick_limits.md`). -## Calibration Records - -Each calibration motion returns its state, command result, elapsed time, and -embedded observation. Use that tool result immediately. To revisit a recorded -step, call `view_env_state({"step": N})`; use `-1` for the latest step. +## Calibration records -The internal `states.json` file is a versioned manifest owned by `EnvState`, not -a list for manual indexing. Calibration analysis should use structured tool -results rather than parsing storage files. +Each calibration motion returns its state and `log` immediately. To revisit a +recorded step, call `view_env_state({"step": N})`; use `-1` for the latest. +The internal `states.json` file is a versioned `EnvState` manifest, not an +agent-facing list for manual indexing. ## Reproducer -Use the runner-managed environment and call structured tools: +Legacy calibration notes below describe the old file-protocol flow. In the +current MCP runtime, use the runner-managed environment and call structured MCP +tools instead. ```bash # For each z in 0.65 .. 0.42, call the MCP tool: move_to({"xyz": [-0.20, 0.10, 0.65], "gripper": -1, "tol": 0.008, "step_clip": 0.010, "max_steps": 80}) -# Inspect the returned result for final_eef_pos and final_dist_m. -# Call view_env_state({"step": -1}) only when the latest state is needed again. +# Inspect the returned log for final_eef_pos and final_dist_m. ``` diff --git a/robots/libero/guides/pro_hybrid_guide.md b/robots/libero/guides/pro_hybrid_guide.md index 7b3054cb..13e36228 100644 --- a/robots/libero/guides/pro_hybrid_guide.md +++ b/robots/libero/guides/pro_hybrid_guide.md @@ -1,197 +1,504 @@ -# LIBERO-Pro Hybrid Perception Guide - -This guide extends [strict_hybrid_guide.md](./strict_hybrid_guide.md) for the -LIBERO-Pro evaluation tracks. Read the strict guide first; its runtime contract, -localization discipline, and single-attempt rules apply unchanged. - -LIBERO-Pro perturbs object placement, fixture placement, spatial relations, and -task language. The agent must solve the observed scene rather than replaying a -seed-specific command sequence. - -## Start Of Run - -Call: - -```json -{"step": 0} +# LIBERO-Pro Hybrid (Pi0.5 + LLM-in-the-loop) — Perception-Isolated Guide + +You are picking up the **LIBERO-Pro** evaluation track in **perception-isolated** +mode — the only mode in this repository (the legacy oracle-state mode is not +included here). + +> **Pi0.5 only does the grasp (`pi0_pick`). The LLM (you) handles every motion +> (`move_to`), every release, sequencing, retries — and you do not get GT object +> coordinates. You localize objects yourself from the depth + camera calibration +> the runtime dumps each step.** + +This document layers on the base playbook. Read it first: + +- [`strict_hybrid_guide.md`](./strict_hybrid_guide.md) — the perception protocol: + back-projection localization, the perception artifacts + (`agentview_high.png`, `agentview_world_high.npy`, and + `agentview_metadata.json`), + the primitive vocabulary, the Rules (0/1/2/4/5), and the `strict_perception` + audit format. **This is the source of truth for *how you localize*.** +- **This file** — LIBERO-Pro–specific setup, the four perturbation axes, the Pi0 + fullshot baseline, the frame split, and how perception isolation changes the + P2-swap story. + +> Your task comes from the initial `view_env_state` result's `task_language`; +> the BDDL is FORBIDDEN. +> Read the authoritative instruction (the BDDL `:language` tag, coord-free) that +> the runtime injects and obey it verbatim. Do **not** scrape the BDDL or import +> the benchmark: that is error-prone (wrong task-map index → wrong task) and a +> perception-isolation breach (the `:init` block holds the GT coordinates this +> mode withholds). For swap fixtures, localize them **visually** (§3.5), never +> from `:init`. You never hand-roll projection math — `back_project` applies +> `K⁻¹` + the cam→world extrinsic and returns the surface `world_xyz` under a +> pixel; just use it. + +Whenever this guide says "see the perception protocol" or "the localization +snippet", it means `strict_hybrid_guide.md`. **Every rule there applies here**; +this guide only *adds* PRO constraints and tooling. Runner contract: call the +structured MCP tools (the runner owns the env server — do not start/stop it, and +issue no file-based commands); it is a single-episode run — no `reset`/`exit`, +recover in place or write an honest failure audit and `finish`. + +## 0. What's different from oracle PRO mode (read this first) + +| | oracle mode (not in this repo) | **perception (this guide)** | +|---|---|---| +| how launched | oracle-state run | `rpent/cli/main.py --libero-type pro` (or `LIBERO_TYPE=pro`); perception artifacts always dumped, coords withheld | +| returned `state` objects | full `objects:{name:[x,y,z]}` | **`object_names:[…]` only — NO coords** | +| how you learn the task | env prompt / scrape BDDL | **initial `task_language` returned by `view_env_state`** (authoritative `:language`, coord-free) — never read the BDDL | +| extra obs artifacts | agentview RGB only | **logical agentview/wrist image, depth, world-map, and metadata keys, including `agentview_high.png` and `wrist_high.png`** — geometry consumed through `back_project` | +| cameras | agentview only | **agentview (fixed, ~1m → ±8-13cm) + eye-in-hand wrist (moves with gripper, ±1-2cm when <20cm to target)** — see §3.6 | +| how P2 swap is solved | read swapped coords from oracle state | **localize the swapped objects by `back_project`** | +| how a swapped *fixture* site is found | read the swap BDDL `:init` block | **localize the fixture visually** (see §3.5) | +| run budget | short | **larger** — perception localization + manipulation is slower (raise `--max-turns` / `--planner-timeout-s`) | +| audit `regime` | `strict` | `strict_perception` | +| which image you pick pixels in | agentview RGB | `agentview_high.png` (or `agentview.png` at low resolution) for **pixel-picking**; never use `agentview_policy.png` for back-projection | + +**The single most important conceptual shift.** In oracle PRO mode the headline +is "the hybrid beats Pi0 on P2 because it reads the *swapped* coordinates straight +out of oracle state, while Pi0 is prompt-/memory-blind." In **perception** +mode there are no coordinates to read — so the hybrid's P2 win now comes from +**seeing where the object is and back-projecting it**. This is a *stronger* claim +(no oracle state at all), and it is the whole point of running PRO in perception +mode: P1 (Task) is still won by reading the *language*; P2 (Position) is now won by +*perception*, not by an oracle. + +## 1. Why LIBERO-Pro + +LIBERO-Pro +([paper](https://arxiv.org/pdf/2510.03827), [repo](https://github.com/Zxy-MLlab/LIBERO-PRO)) +perturbs each base task along five axes; all end-to-end VLAs (OpenVLA / Pi0 / +Pi0.5 / UniVLA) collapse on the two strongest: + +| Axis | Suffix | Paper column | What changes | Headline result | +|---|---|---|---|---| +| **Task** | `_task` | **P1** | Instruction + goal predicate inverted | All VLAs ≈ 0.0 | +| **Position** | `_swap` | **P2** | Object **and fixture** initial positions swapped | All VLAs 0.0–0.4 | +| Semantic | `_lan` | — | Instruction paraphrased; goal unchanged | VLAs handle (memorize visual) | +| Object | `_object` | — | Object appearance / colour / scale | VLA visual policy stressed | +| Environment | `_environment` | — | Table / scene swapped | Visual policy stressed | + +The agentic hybrid wins on **P1 and P2** by routing the language channel and the +*perceived* spatial-state channel through the LLM. Object and Environment +perturbations enter through the Pi0 vision channel and the hybrid inherits the +VLA's weakness there — declare that upfront, don't oversell. **In perception +mode, the Object/Environment axes also stress your own localization** (a +recolored or rescaled object is harder to pixel-pick), so lean on the +multi-pixel-median tip from the perception protocol. + +## 2. Setup (do these once on a fresh checkout) + +Run the idempotent installer from the repo root: + +```bash +bash scripts/install_libero_pro_plus.sh ``` -through `view_env_state`. From the returned tool result: - -- read top-level `task_language` verbatim; -- inspect `state.robot0_eef_pos` to identify the scene frame; -- inspect object names only as a scene inventory, never as coordinates; -- inspect the embedded agentview image for semantic identity and relations; -- inspect the wrist image only when it contains useful close geometry. - -Step `-1` means the latest record. Do not inspect or index `states.json` -directly; it is an internal manifest. - -## Perturbation Axes +It does all four steps below: the liberopro editable install, applies the +benchmark-registration patch, syncs the authoritative HF dataset snapshot, and +verifies with the `get_benchmark(...).get_task(0).language` check. The perception +observables (depth + both cameras + hi-res) are unconditional once the runner +launches with `--libero-type pro`; there is nothing perception-specific to +install beyond the PRO setup itself. -### Object perturbation +### 2.1. LIBERO-PRO repo -Objects move relative to their seed positions. Re-localize every target and -destination from the current images. Seed recipes remain useful for primitive -ordering, prompt wording, safe heights, and known failure modes, but never for -coordinates. +Cloned at `${LIBERO_PRO_PATH:-/path/to/LIBERO-PRO}/` from +`https://github.com/RLinf/LIBERO-PRO.git` and installed editable into the openpi +venv: -### Spatial perturbation - -Relations such as left/right, on-top-of, or next-to may select a different -instance than in the reference scene. Identify the candidate satisfying the -current visual relation, then localize it with `back_project` or `segment`. - -### Goal perturbation - -The destination or requested interaction changes. Re-read `task_language` and -classify the destination semantically. Do not infer the goal from the suite, -task index, object list, or a sibling result file. - -### Task-language perturbation - -The authoritative instruction is the current top-level `task_language`. Do not -read BDDL files. They combine language with hidden initialization data and -therefore violate perception isolation. - -## Scene Frame Selection - -The initial end-effector height distinguishes the principal scene frames: - -| Initial EEF z | Frame | Typical fixtures | -|---|---|---| -| approximately 0.26 m | object/low-table | grocery and basket tasks | -| approximately 0.68 m | living-room table | plates, baskets, pudding | -| approximately 1.17 m | kitchen table | stove, cabinet, drawer, microwave | +```bash +python -m pip show liberopro +# Name: liberopro Version: 0.1.0 Location: ${LIBERO_PRO_PATH:-/path/to/LIBERO-PRO} +``` -Use `view_env_state({"step": 0})["state"]["robot0_eef_pos"][2]` as the -measurement. Then use the matching safe-height guidance from -[env_calibration.md](./env_calibration.md). +### 2.2. Apply the benchmark-registration patch -## Mandatory Perception Table +The upstream `__init__.py` does **not** expose the 16 perturbation suites through +`get_benchmark()`. Our patch +[`scripts/liberopro_register_perturbations.patch`](../../../../scripts/liberopro_register_perturbations.patch) +adds them and overrides `Task.language` to read each BDDL's actual `:language` +tag (so the perturbed instruction reaches Pi0 / hybrid). -Before manipulating, create one row for every task-relevant entity: +```bash +cd ${LIBERO_PRO_PATH:-/path/to/LIBERO-PRO} +git apply /scripts/liberopro_register_perturbations.patch +``` -| field | meaning | -|---|---| -| role | target, destination, support, fixture, or relation landmark | -| semantic evidence | visual properties and relation establishing identity | -| agentview pixels | several interior pixels or a verified segment mask | -| agentview xyz | robust projection from the global camera | -| wrist refinement | accepted, rejected, basket-confirmed, or unnecessary | -| final xyz | coordinate used for planning | -| uncertainty | duplicates, occlusion, rim bias, label ambiguity, etc. | +If already applied (likely), `git status -s` shows clean. If you reinstall +liberopro, re-apply. -Do not begin manipulation while a required row lacks a defensible identity or -coordinate. +### 2.3. Huggingface dataset (authoritative) -## Swapped Objects And Fixtures +The LIBERO-PRO git repo ships **incomplete / broken** init files for several +perturbation suites (e.g. `libero_spatial_swap` has 0 BDDLs; some +`libero_spatial_task` `.pruned_init` files are 0 bytes). Treat the git repo as +unreliable for perturbation data. The full, correct set lives on Huggingface +([`zhouxueyang/LIBERO-Pro`](https://huggingface.co/datasets/zhouxueyang/LIBERO-Pro)), +persisted locally at: -For swap-style perturbations: +``` +${LIBEROPRO_DATASET_PATH:-/path/to/liberopro_hf}/ +├── bddl_files/ 16 perturbation suites, 10 BDDLs each +└── init_files/ 16 perturbation suites, 10 init files each +``` -1. Identify both swapped entities in the embedded agentview image. -2. Classify each by appearance and current relation, not expected seed layout. -3. Localize each independently. -4. Verify the chosen target still satisfies `task_language`. -5. Re-localize after any contact that could move either entity. +Covers `{libero_spatial, libero_object, libero_goal, libero_10} × {swap, task, +lan, object}`. The installer syncs this into the liberopro install (overwriting +the broken upstream files); if the persistent copy is gone, re-download with: -The runtime withholds privileged coordinates, so there is no coordinate field -to fall back on. The current images and geometry tools are the source of truth. +```bash +python -c " +from huggingface_hub import snapshot_download +snapshot_download(repo_id='zhouxueyang/LIBERO-Pro', repo_type='dataset', + local_dir='${LIBEROPRO_DATASET_PATH:-/path/to/liberopro_hf}', + allow_patterns=['bddl_files/**','init_files/**'])" +``` -## Destination Classification +### 2.4. Verify -Pro scenes frequently contain look-alike surfaces. Before placement, explicitly -classify all plausible destinations: +```bash +LIBERO_TYPE=pro python -c " +import liberopro.liberopro.benchmark as bench +for n in ['libero_spatial_task','libero_spatial_swap','libero_spatial_lan']: + b = bench.get_benchmark(n)(); t = b.get_task(0) + print(f'{n} t0: {t.language!r} trials={len(b.get_task_init_states(0))}')" +``` -- plate versus stove burner; -- cabinet top versus drawer opening; -- basket interior versus rim; -- microwave cavity versus door or surrounding counter; -- movable lid versus fixed fixture surface. +Expected: +``` +libero_spatial_task t0: 'Pick the akita black bowl not between the plate and the ramekin and place it on the plate' trials=50 +libero_spatial_swap t0: 'Pick the akita black bowl between the plate and the ramekin and place it on the plate' trials=50 +libero_spatial_lan t0: 'lift the black bowl between the plate and ramekin and set it on the plate' trials=50 +``` -Only after semantic classification should you call `back_project` or `segment` -for coordinates. +## 3. PRO-specific environment gotchas -## Mid-Carry Re-Localization +Everything in `strict_hybrid_guide.md` applies. The following are **additional** +PRO constraints (most live in detail at [`env_calibration.md`](./env_calibration.md)). -Long-horizon Pro tasks often move or occlude objects during earlier steps. -Before each new pick or placement: +### 3.1. Three scene frames, picked per-task -1. call `view_env_state({"step": -1})` if the previous primitive result is - no longer in context; -2. inspect the newest embedded images; -3. re-localize any entity that may have moved; -4. update the working perception table; -5. verify the remaining primitive order still matches `task_language`. +PRO scenes use one of three table fixtures; the eef home z differs by up to +~0.9 m. -Never assume the initial coordinate remains valid after contact, release, or a -fixture interaction. +| Fixture | eef home z | Table top z | xy reachable | Where | +|---|---|---|---|---| +| `living_room_table` | ≈ 0.68 | ≈ 0.43 | `(x∈±0.30, y∈±0.30)` | basket / plate / pudding | +| `kitchen_table` | ≈ 1.17 | ≈ 0.90 | `(x∈±0.30, y∈±0.30)` | stove / cabinet / drawer / microwave | +| `object` (low table) | ≈ 0.26 | ≈ 0.0 | `(x∈±0.30, y∈±0.30)` | `libero_object` grocery-into-basket | -## Contact Tasks +**Mandatory check at session start: read `state.robot0_eef_pos[2]` from +`view_env_state({"step": 0})`.** ≈ 0.68 → LIVING_ROOM; ≈ 1.17 → KITCHEN; +≈ 0.26 → OBJECT. Pick `pre_pos_z` / `carry_z` / `release_z` accordingly (per-item +OBJECT-frame altitudes are in +`resources/libero/memory/project_libero_object_pro_done.md`). Sending a +wrong-frame z (e.g. KITCHEN coordinates while the env is in LIVING_ROOM frame) +crashes the env worker (EOFError, silent state loss). -Use `pi0_doubled` for learned contact behavior such as turning a knob or -opening/closing a drawer. Its success flag mirrors the benchmark termination -predicate, so an intermediate contact can be useful even when success is false. -Inspect state and image evidence after every contact attempt. +> **Perception note.** This proprioceptive z is *not* an object coordinate — it's +> the robot's own pose, which the returned `state` still gives you in perception mode. +> Reading it to pick the frame is fine. It also doubles as a free depth sanity +> check: your back-projected table z should sit ≈0.25 m below eef home z (≈0.43 in +> LIVING_ROOM, ≈0.90 in KITCHEN). If a back-projection lands far from that, you +> picked a wrong pixel. -Use short scripted alignment motions around contact skills. Avoid long blind -pushes, which can destabilize MuJoCo or move the end effector into an invalid IK -branch. +For `libero_spatial_*` you are always in **KITCHEN frame**. Standard heights: -## Planning Across Multiple Objects +``` +pre_pos_z = 1.05 # ~7 cm above objects at z≈0.97 +carry_z = 1.10 # safe traversal, well under upper limit 1.15 +release_z = 1.01 # ~7 cm above plate top at z≈0.90 +``` -For multi-object tasks: +These are *altitude* knobs (robot-frame), not object positions — you still derive +the object's **xy** by `back_project` every time. + +### 3.2. xy single-step ±0.30 cap + +OSC flips IK branches if you command `|x|>0.30` or `|y|>0.30` in a single +`move_to` (the eef lands in the wrong half-space and corrupts the run). **Never +command beyond ±0.30 in a single move** — split into carry-z waypoints. (Detail +in `env_calibration.md`.) + +### 3.3. Slow long-distance carry for swap variants + +P2 swap can move a large object 15 cm across the table. `step_clip=0.025` lets it +slip in the gripper and the object ends up centimetres off target. Mitigation +(proven on `_swap` t0, carries over directly): + +- `carry_z = 1.15` (higher than usual 1.10) +- `step_clip = 0.020` (slower) +- **Re-localize mid-travel** instead of trusting a cached `object_xyz - eef_xyz` + offset: inspect `agentview_high.png` mid-carry and call `back_project` on the + carried object's pixel; if the + perceived offset drifted >5 mm from post-pick, re-pre-position and re-`pi0_pick`. + +### 3.4. Task language is from BDDL, not filename + +After the patch, `get_task(i).language` returns the perturbed `:language` tag, and +the env passes it to Pi0 as the prompt (surfaced as top-level `task_language` +in each returned state view). For `_task` and `_lan` this is the perturbed +instruction. **Don't override it** — falsifying the VLA's prompt-blindness is the +point. You read the same language to decide *which* object to localize and place. + +### 3.5. ⚠ Swap moves FIXTURES too — and you must localize them visually + +This is the biggest perception-mode trap, and it is **specific to `_swap`**. The +P2 perturbation does not only swap loose objects; for `libero_goal_swap` it swaps +entire **fixtures** (stove ↔ cabinet ↔ wine_rack), so a goal predicate like +`On(bowl, flat_stove_1_cook_region)` now points at wherever the *stove* was +relocated to, and there are no coordinates in the returned `state`. See +`resources/libero/memory/feedback_swap_perturbs_fixtures.md`. + +In **oracle** mode the documented fix is to read the swap BDDL `:init` block and +recompute the fixture site's world coordinates. **That is forbidden here** — the +`:init` block is ground-truth geometry. In **perception** mode you instead: + +1. Identify the target fixture by name from the (perturbed) task language and + `state.object_names`. +2. **Localize the fixture's predicate site visually** in + `agentview_high.png`: pick pixels on the stove's cook region / + cabinet top surface / rack top shelf, then `back_project` (sample 3–5 pixels, + median the xy — fixtures are large and the surface you want is the *placement* + surface, not the nearest edge). +3. Carry target xy = perceived site xy; descend to the perceived site z + a small + clearance, `release`, then **retreat with gripper open** — the predicate often + fires *during* the settle, not at the release step itself. + +So the swap-fixture problem becomes "find the fixture in the image" rather than +"read where the BDDL put it." Loose-object swaps (e.g. `libero_spatial_swap`, +where only bowls/plates move) are simpler: just localize each object by +`back_project` as usual; their new positions fall straight out of the depth. + +### 3.6. Two cameras — agentview = IDENTITY, wrist = GEOMETRY + +Do **not** restate the two-camera protocol here — the strict guide's **First-step +perception protocol** is the source of truth: `agentview.png` / `agentview_high.png` is the +semantic identity authority (decides *which* object/surface satisfies the task +language + relation); `wrist.png` / `wrist_high.png` refines geometry for the *same* candidate +(accept only within ~3–5 cm of the agentview anchor, never average, basket/cavity +excepted). Both maps share one world frame, read via +`back_project({"camera":"wrist"})`; the optional `segment` honours both cameras +(`{"camera":"agentview"|"wrist"}`, `min_score` default 0.2, or `point:[row,col]`). + +**PRO implication:** `_swap` (and `_task`) can *invert* target or destination +semantics, so **never reuse base-task or recipe coordinates** — the object +satisfying the relation may now sit where the base task never put it. Let +agentview decide *what*; the near-vertical wrist is BAD at identity (locks onto +look-alikes) and only refines *where*. + +### 3.6c. Mandatory pre-task perception pass + +Also owned by the strict guide's First-step perception protocol: before ANY +pick/place, build the localization table (one row per task-relevant entity) and +pass the FINAL READY CHECK. **PRO implication:** in single-attempt mode a +wrong-target first grab is unrecoverable, and under `_swap` the "right" target is +exactly the one you'd get wrong by habit — so localizing all entities up front is +cheap insurance, not optional. + +### 3.6b. Hi-res perception channel (1024×1024) + +Also owned by the strict guide's Hi-res perception channel section (1024×1024 +pairs each step; prefer the hi image for identification; `back_project` defaults +`resolution="high"`; never mix pixel grids). **PRO implication:** hi-res fixes +*which* object you point at — decisive for the recolored/rescaled Object axis and +for telling swapped same-shape groceries apart — but does NOT change metric +accuracy or replace the wrist coarse→fine refinement. **Identity caveat:** SAM3 +gives two same-shaped, different-brand objects the SAME category (it labelled the +tomato-sauce and alphabet-soup cans identically) — use its mask only as a category +candidate, then READ the label yourself in the hi-res crop to assign identity; +never let SAM3's category settle a brand choice. + +## 4. The four-cell experiment per (base task, seed) + +For each base task you claim coverage on, generate four runs — every run is a +perception cell: + +| Suite | Variant | Perception-mode expectation | +|---|---|---| +| `libero_spatial` | base sanity | Pi0 and hybrid both pass | +| `libero_spatial_task` | **P1 Task** | Pi0 ✗ (picks base target); hybrid ✓ (LLM flips target from instruction, then localizes it) | +| `libero_spatial_swap` | **P2 Position** | Pi0 mixed; hybrid ✓ — **by localizing the swapped object/fixture from depth, not from state** | +| `libero_spatial_lan` | Semantic | Both pass (paraphrase invariant) | -- process objects in an order that minimizes collision and occlusion; -- preserve already-correct placements; -- route later carries around placed objects rather than over them; -- re-confirm each source and destination immediately before use; -- verify intermediate relations visually rather than assuming they held. +Replace `spatial` with `object`, `goal`, or `10` for the other base suites. -When order is specified by the task, obey it even if another order appears -easier. +### 4.1. Hybrid run — the MCP runner -## Reference Results And Memory +Launch a cell with the CLI; the runner owns `env_server.py`, exposes the +structured tools, and runs single-attempt: -Reference results and memory entries provide reusable strategy: +```bash +python rpent/cli/main.py --env libero --suite --task --seed \ + --libero-type pro --planner claude_code --model claude-opus-4-8 -- prompt ladders; -- manipulation ordering; -- safe approach and carry heights; -- fixture-specific contact patterns; -- known failure and recovery modes. +# e.g. --suite libero_spatial_task 0 (P1) · libero_spatial_swap 0 (P2) · +# libero_goal_swap 2 (P2 fixture swap) · libero_10_task 5 (long horizon) +``` -They do not provide valid coordinates for the current run. Re-derive all -positions through perception. +`--libero-type pro` may be given as `LIBERO_TYPE=pro` instead. -## Outcome And Audit +Audit + recipe land in the run's `output_dir` (`output_dir` and `recipe_tag` +arrive in your first message): -The current benchmark outcome is top-level `libero_terminated` in the latest -environment tool result. It is not stored inside `state`. +``` +{output_dir}/{recipe_tag}.json <- you write this (write_text_file) +{output_dir}/recipe_{recipe_tag}.jsonl <- exported automatically by the runner +``` -The audit should record: +Do NOT write into `resources/libero/results_*_pert/` — that tree is a **read-only +seed-0 reference corpus**, not a write target. + +### 4.2. Environment server is runner-owned + +There is no manual server to launch and no REPL to drive: the MCP runner starts, +manages, and tears down `env_server.py` for you and blocks each tool call until +the next state record is dumped. Do not start, stop, or background it, and +do not poll for readiness. + +### 4.3. Pi0 fullshot baseline + +The baseline is Pi0.5 driving the task end-to-end with the runtime's own +(perturbed) `task_language`. There is **no standalone baseline CLI in this repo**; +the numbers to compare against are the recorded full-shot results (the team's +`SUCCESS_RATES` table). **Do not invent a `pi0_baseline.py` path.** + +Pi0 never sees object coords in either mode, so there is no perception variant of +the baseline. Expected behavior: + +- **P1 (task):** Pi0 "succeeds at the wrong task" — it picks the *base*-task + target object, places it on the plate, and `libero_terminated=False` because the + goal predicate names a different object. This is exactly the gap the hybrid + closes. +- **P2 (position):** Pi0 picks / places at the *base* (un-swapped) location. + +### 4.4. Audit JSON — PRO + perception fields + +Start from the `strict_perception` audit schema in the perception protocol, and +add the PRO fields: + +```jsonc +{ + "suite": "libero_spatial_swap", + "task_id": 0, + "seed": 0, + "regime": "strict_perception", + "perturbation_type": "swap (P2 Position perturbation)", + "perturbed_task_language": "", + "perturbation_semantics": "", + "expected_baseline_behavior": "", + "strategy_notes": "HOW you localized — which pixel(s) in agentview_high.png, back-projected world xyz; for swap, how you found the relocated object/fixture", + "pick_result": { /* the pi0_pick step's result */ }, + "final_state": { /* latest view_env_state result's `state` field */ }, + "libero_terminated": true +} +``` -- exact perturbed `task_language`; -- perturbation type when known; -- semantic identification evidence; -- agentview and accepted wrist localization results; -- primitive sequence and recovery decisions; -- memory files consulted; -- final state and `libero_terminated`. +`strategy_notes` **must** describe the localization (pixel → `back_project` → +world xyz). For swap cells, explicitly note that the relocated object/fixture was +found by perception, not by reading coords. If unrecoverable after honest +exploration, write `libero_terminated: false` with what you tried and which step +failed — never warp (teleport primitives are deleted; see Rule 4 in the +perception protocol). Write the audit with `write_text_file` to +`{output_dir}/{recipe_tag}.json`, then call `finish`. + +## 5. Perception-protocol rules — PRO clarifications + +- **Rule 0 (use images for reasoning).** Even more critical under PRO: P2 swap can + move a large object/fixture clear across the table, and you have *no* + coordinates to fall back on. `agentview_high.png` plus `back_project` are your + spatial truth — inspect the embedded image and describe the scene before deciding + targets. +- **Rule 1 (no `pi0_end_to_end`).** Pi0 does the grasp via `pi0_pick`; the LLM + scripts every motion + release. Under PRO this is doubly important — handing + back to Pi0 means handing back to the prompt-blind / memorized-place habit you + are trying to falsify. +- **Rule 2 (single-episode current run).** This is a one-shot eval: do not call + `reset` or `exit`. Recover *within* the episode when safe (re-localize, + re-pre-position, re-`pi0_pick`, walk the prompt ladder, + `rotate_pitch`/`move_pose`); otherwise write an honest stuck/failure audit and + `finish`. `_swap` typically needs an in-episode retry — document it. +- **Rule 4 (no teleport).** `set_object_pose`, `articulate_to`, `js_move_to`, + `carry_object` are deleted. A goal past OSC reach with no physical approach → + honest `libero_terminated:false`. +- **Rule 5 (assume solvable).** A localization that moves the gripper into thin + air means you picked a wrong pixel (a reflection, a rim, a decoy object under + the Object perturbation), not that the cell is unreachable. Re-look, re-pick, + re-`back_project` before concluding failure. + +## 6. Existing corpus -Recipe export is runtime-managed from recorded primitives and successful -segmentation events. Artifact identifiers returned by tools are for audit and -traceability, not manual file access. +``` +robots/libero/guides/ +├── pro_hybrid_guide.md <- this file +├── strict_hybrid_guide.md <- perception protocol + Rules (source of truth) +└── env_calibration.md <- OSC frame bounds + safe altitudes +scripts/ +└── liberopro_register_perturbations.patch +resources/libero/memory/ <- MEMORY.md index + feedback_*/project_* notes +resources/libero/results_spatial_pert/ <- read-only seed-0 reference corpus +resources/libero/results_{object,goal,10}_pert/ <- same, other suites (seed-0) +``` -## Quick Checklist +Before you start, **read the auto-memory**: `resources/libero/memory/MEMORY.md` +(one-line hooks, auto-injected via CLAUDE.md). For perception PRO cells always +open `feedback_no_teleport_rule.md` and — for any `_swap` cell — +`feedback_swap_perturbs_fixtures.md` (what swaps, and why you re-find the +relocated fixture visually). For bowl→plate spatial tasks also read +`feedback_bowl_eef_y_offset.md`; for cluttered picks, `feedback_pi0_pick_full_prompt.md`; +after two failed retries, `feedback_failure_forensics.md`. The +`resources/libero/results_*_pert/` recipes are **inputs** (technique priors) — +consult them for prompt ladders, staging, and target zones, but never reuse their +coordinates (re-derive every xyz from THIS scene) and never write there. + +## 7. What to do next (priority order) + +1. **Extend spatial to all 10 tasks at seed 0**, four perception cells each + (base / `_task` / `_swap` / `_lan`). For hybrid runs, use the seed-0 reference + recipes in `resources/libero/results_spatial_pert/` as *technique* starting + points — the pick step is usually identical; the place target changes for + `_swap`, the target object changes for `_task`. Never reuse their coordinates. +2. **Scale to seeds beyond 0** (50 trials per task). Recipes must re-localize per + scene; a perception recipe is *data-flow* (perceive → plan → act), never + hard-coded xyz. +3. **Replicate on `libero_object`, `libero_goal`, `libero_10`.** Frame split + applies (read the initial returned `state.robot0_eef_pos[2]`). For + `libero_goal_swap`, apply §3.5 — localize the **swapped fixture** visually. +4. **Aggregate into a main table** `(suite × perturbation × {Pi0, hybrid})`. The + headline number is the conditional: of the seeds Pi0 fails on, what fraction + does the *perception* hybrid solve? Because there is no oracle state anywhere in + the hybrid's reasoning here, this is the strongest single statistic the agentic + decomposition can claim. + +## 8. Quick reference for a brand-new session + +```bash +# 1. Sanity-check the liberopro patch (perturbed language must show) +LIBERO_TYPE=pro python -c \ + "import liberopro.liberopro.benchmark as b; print(b.get_benchmark('libero_spatial_task')().get_task(0).language)" +# -> must read 'Pick the akita black bowl not between ...' (the perturbed text) + +# 2. Read the auto-memory: resources/libero/memory/MEMORY.md + +# 3. Launch a perception cell (runner owns env_server; single-attempt) +python rpent/cli/main.py --env libero --suite libero_spatial_swap --task --seed 0 \ + --libero-type pro --planner claude_code --model claude-opus-4-8 +``` -- Read strict guide and relevant memory. -- Call `view_env_state({"step": 0})`. -- Select the scene frame from initial EEF z. -- Read top-level `task_language`. -- Build the complete perception table. -- Re-derive all coordinates for this scene. -- Use agentview for identity and wrist for consistent refinement. -- Re-localize after contacts and placements. -- Check top-level `libero_terminated` after each relevant action. -- Write an honest audit and call `finish` without resetting. +Then, inside the run: + +4. `view_env_state({"step": 0})` → read `state.robot0_eef_pos[2]` to pick the + frame (§3.1). Inspect `agentview_high.png` and call + `view_camera_meta`. Localize the target (and, for `_swap`, the relocated + object/fixture) with `back_project` — run the mandatory pre-task perception pass + (§3.6c). Plan, then execute one structured tool at a time. +5. `write_text_file` the audit to `{output_dir}/{recipe_tag}.json` + (`regime: strict_perception`) before `finish`; the recipe + `{output_dir}/recipe_{recipe_tag}.jsonl` is exported automatically by the runner. + +When in doubt about *how to localize* or a primitive, the source of truth is +[`strict_hybrid_guide.md`](./strict_hybrid_guide.md); about *PRO setup / +perturbation semantics*, see +[`scripts/install_libero_pro_plus.sh`](../../../../scripts/install_libero_pro_plus.sh) +and §2. diff --git a/robots/libero/guides/strict_hybrid_guide.md b/robots/libero/guides/strict_hybrid_guide.md index 417b380e..786f85a4 100644 --- a/robots/libero/guides/strict_hybrid_guide.md +++ b/robots/libero/guides/strict_hybrid_guide.md @@ -1,206 +1,603 @@ -# Strict Hybrid LLM + Pi0.5 Perception Guide +# Strict Hybrid LLM + Pi0.5 — Perception-Isolated Guide + +You are taking over a hybrid LIBERO experiment in **perception-isolated** mode +— the only mode in this repository (the legacy oracle-state mode, where the +state JSON carried GT object coordinates, is not included here). + +> **Pi0.5 only does the grasp (`pi0_pick`). The LLM (you) handles every motion +> (`move_to`), every release, sequencing, retries — and you do not get GT +> object coordinates. You localize objects yourself from the depth + camera +> calibration the toolkit dumps each step.** + +## What's different from legacy oracle mode (read this first) + +| | legacy oracle mode (not in this repo) | **perception (this guide)** | +|---|---|---| +| how object coords are withheld | (none — full GT coords in `state`) | **the returned `state` carries `object_names` only, no coordinates** | +| returned `state` objects | full `objects:{name:[x,y,z]}` | **`object_names:[…]` only — NO coords** | +| how you learn the task | env prompt / BDDL | **top-level `task_language` returned by `view_env_state`** (authoritative `:language`, coord-free) — never scrape the BDDL | +| extra obs artifacts | policy image only | **logical agentview/wrist image, depth, world-map, and metadata keys, including `agentview_high.png` and `wrist_high.png`** | +| cameras | agentview only | **agentview (fixed, ~1m → ±8–13 cm) + eye-in-hand wrist (moves with gripper, ±1–2 cm when <20 cm to target)** — coarse→fine, see below | +| how you get an object's xyz | read `state["objects"][name]` | **pick the object pixel, then `back_project({"row":ROW,"col":COL,"step":NN})` (K⁻¹+extrinsic already done); refine with the wrist map up close** | +| how you confirm the grasp | GT object-lift oracle | **grasp-only, no oracle — you judge the grasp from gripper width + wrist cam (Rule 1 / 1b)** | +| how you pick which of two identical objects | read their distinct coords | **by SPATIAL RELATION from `task_language` (elevation / left-right), never by `_1`/`_2` name** | +| cell budget | 600 (short suites) | **1200** — perceptual localization + manipulation is slower | +| audit `regime` | `strict` | `strict_perception` | + +How localization works (the core of this mode): + +The toolkit already back-projects EVERY pixel for you through +`agentview_world.npy` / `agentview_world_high.npy` or +`wrist_world.npy` / `wrist_world_high.npy` — all in the SAME world frame. +These are logical artifact names scoped by step, not paths to construct. +**You do NOT write back-projection math; you pick a pixel and call +`back_project`, which selects the matching map for you.** + +1. Inspect `agentview_high.png` — agentview RGB **in the calibration frame** + (vertical-flip of the raw buffer). This is the image you pick pixels in. + `agentview_policy.png` is the Pi0 frame; **do not pick pixels there**. +2. Find the target object visually → pixel `(row, col)`. +3. Read its world xyz directly: `back_project({"row":ROW,"col":COL,"step":NN})`. + Sample 3–5 pixels on the object's top surface and median the returned xy + (robust to one mis-picked rim/edge pixel). That's the object's surface point + in world frame. + +### Hi-res perception channel (ON BY DEFAULT, 1024×1024) + +The toolkit records, every step, a 1024×1024 pair per camera IN ADDITION to the +256 artifacts: `agentview_high.png` + `agentview_world_high.npy` and +`wrist_high.png` + `wrist_world_high.npy`. +**PREFER the hi pair for looking and identification** — a far object spans 4× +more pixels, and package text/label art is actually legible (e.g. "Cream +Cheese" vs "BUTTER" boxes are readable at 1024 and indistinguishable smears +at 256). If the hi files are absent, everything below works unchanged at 256. + +- A hi-res pixel `(row,col)` indexes ONLY the hi-res world map (`back_project` + default `resolution:"high"`; float16, same world frame). Never index a 1024 + pixel into the 256 map or vice versa — pass `resolution:"low"` only for a + pixel taken from a 256 image (convert by dividing/multiplying by 4 if needed). +- The `segment` command automatically uses the hi-res frame when present + (its `centroid_pixel`/`box` are then in 1024 coords). +- Metric accuracy of mask-median localization is the SAME at both resolutions + (the residual ~2 cm error is the surface-vs-center offset, not pixel size) — + the hi channel is for **identification**, not for replacing the wrist-cam + fine-localization protocol. +- If a high-resolution artifact is absent, everything below works unchanged + with `agentview.png` or `wrist.png` and `resolution:"low"`. + +### Two cameras — agentview = IDENTITY, wrist = GEOMETRY + +**Roles are NOT symmetric (this is the core discipline, from the 80-task +localization sweep):** +- **agentview** (`agentview_high.png` + `agentview_world_high.npy`, ~1 m away): + the **semantic IDENTITY authority** — decides WHICH object/surface satisfies + the task language + spatial relation. At 1024 a far object spans enough pixels + to read labels/shape. Its metric precision is only ±8–13 cm, but identity is + its job, not millimetres. +- **wrist** (`wrist_high.png` + `wrist_world_high.npy`, + <20 cm → ±1–2 cm): a **GEOMETRY refinement** camera for the SAME candidate + agentview already chose. It is near-vertical and **bad at identity** — it + cannot read side labels or tell duplicate/similar items apart, and in failed + probes it locked onto a look-alike hundreds of pixels away. **NEVER let the + wrist freely re-identify a non-basket target.** + +Protocol (non-basket objects/surfaces): +1. **Identity (agentview)** — in `agentview_high.png` choose the target by RGB / + label / shape / global spatial relation; sample 3–8 pixels on it and median + `back_project` → the **identity anchor** (rough xy ok). +2. **Approach** — `move_to` ~15–20 cm directly above that anchor xy (toolkit + re-renders both cams after every primitive). +3. **Geometry refine (wrist)** — pick the SAME candidate's pixel in + `wrist_high.png`, `back_project` it with `camera:"wrist"`. **Accept + this corrected xy ONLY if it is within ~3–5 cm of the agentview anchor**; if + it jumps >5 cm, REJECT it (it hit a look-alike/background) and keep the + agentview xyz. The wrist may sharpen coordinates but may NOT override the + agentview semantic choice. Never average the two. +- **Basket / cavity special case:** for `basket`/cavity the wrist MAY also + confirm/refine the true interior centre (basket failures are rim/edge bias, + not semantic confusion). + +### Mandatory pre-task perception pass — localize EVERYTHING, THEN act + +Before ANY pick/place, build a localization table in your reasoning — one row +per task-relevant entity (every movable target, every destination/support/ +fixture, every relation landmark named by `task_language`), each with: +`name_or_role · agentview_evidence (why this candidate) · agentview_pixels · +agentview_xyz (median back_project on hi-res) · wrist_refine +accepted|rejected|basket_confirmed · final_xyz · uncertainty`. Do the +**FINAL READY CHECK** (every entity has a final xyz; non-basket wrist +refinements are spatially consistent with agentview; basket points are +interior-centred) — only then start manipulating. Rationale: in single-attempt +mode a **wrong-target first grab is unrecoverable** (it tips/displaces both the +grabbed object and the target zone), so the cheap insurance is to identify all +entities up front instead of recovering later. + +> The underlying math (already done for you): `cam = [(col−cx)·z/fx, +> (row−cy)·z/fy, z]` with `z=depth[row,col]`, then `P_world = +> extrinsic_cam2world @ [cam,1]`. You must invert `K` BEFORE the extrinsic — the +> old `E @ [col·z, row·z, z, 1]` recipe (no `K⁻¹`) is **wrong** (metres off). +> The agentview and wrist world-map artifacts bake in the correct +> `K⁻¹`+extrinsic, and +> `back_project` reads them for you — that is precisely why `back_project` +> exists; you never write this math yourself. Forward world→pixel was verified +> 5/5 vs GT at design time (plate Δ=6 mm). The wrist map may be all-table/null +> early (the wrist only sees gripper + table until you move it over a target). + +## Before you start: READ THE AUTO-MEMORY + +Operating wisdom lives in the in-repo memory: -This guide describes the perception-isolated LIBERO runtime. The agent receives -robot proprioception, object names, task language, and camera observations. It -does not receive privileged object coordinates. - -Pi0.5 performs grasping through `pi0_pick`. The LLM owns semantic perception, -localization, scripted motion, release, verification, and recovery within the -current episode. - -## Runtime Contract - -Every motion primitive appends a `StepRecord`. The record contains: - -- `step_idx`: sequential motion-step number; -- `state`: robot proprioception and coordinate-free scene information; -- `artifacts`: sorted logical base names for all files captured at the step; -- `command`, `result`, and `elapsed_s`; -- `extras`: LIBERO outcome fields such as `task_language`, - `libero_terminated`, and `episode_truncated`. - -Artifact storage is private to `EnvState`. Do not construct filenames or read -observation files manually. Use the structured tools: - -- `view_env_state`: state, artifact names, log, and embedded images; -- `view_camera_meta`: calibration data for one camera and step; -- `back_project`: matching image pixel or region to world coordinates; -- `segment`: text- or point-prompted mask plus projected `world_xyz`. - -Step `0` is the initial observation. Step `-1` means the latest observation. -When a tool omits `step`, its schema default is `-1`. - -`states.json` is an internal versioned manifest. It is not an agent-facing state -API and should not be indexed directly. - -## Camera Roles - -`view_env_state` embeds the best available images for the selected step: - -- **policy image**: the Pi0-oriented agentview input; -- **agentview image**: fixed global view, high resolution when available; -- **wrist image**: eye-in-hand view, high resolution when available. - -Use agentview for semantic identity and global relationships. Use wrist for -close-range geometry after the target identity has already been anchored in -agentview. The wrist view must not freely replace the semantic decision when -similar objects are nearby. - -Pixels passed to `back_project` must come from the same camera and resolution -specified in the call. The tool selects the matching world map internally. - -## Initial Perception Pass - -Before the first manipulation primitive: - -1. Call `view_env_state({"step": 0})`. -2. Read the returned top-level `task_language` verbatim. -3. Identify every movable target, destination, support, and relation landmark - named by the task. -4. Inspect the embedded agentview image and classify candidates by color, - shape, label, container type, and spatial relation. -5. Localize each chosen candidate with several interior pixels and - `back_project`, or use `segment` when a stable text/point prompt exists. -6. Record a working table with semantic evidence, sampled pixels, projected - coordinates, uncertainty, and the selected final coordinate. - -Do not start manipulation until every required target and destination has a -defensible identity and coordinate estimate. - -## Semantic Identity Before Geometry - -Depth and world coordinates cannot distinguish visually similar surfaces. A -plate, stove burner, pot lid, and cabinet top can all be flat circular or planar -regions at similar heights. - -Classify the destination in RGB before localizing it: - -- **plate**: ceramic disc with a clean rim, often white or ringed; -- **stove region**: darker fixture surface, coil, grate, or burner pattern; -- **basket**: open container whose interior center differs from the rim; -- **cabinet or drawer**: fixture geometry associated with the task noun. - -When duplicate objects exist, choose by the relation in `task_language`, not by -internal object suffixes. - -## Coarse-to-Fine Localization - -For each non-basket object: - -1. Select the semantic candidate in agentview. -2. Project three to eight interior pixels and take a robust median. -3. Move approximately 15-20 cm above the agentview anchor. -4. Inspect the wrist image and refine the same physical candidate. -5. Accept the wrist estimate only when it remains within roughly 3-5 cm of the - agentview anchor. Otherwise reject it and retain the global estimate. - -For baskets and cavities, use agentview for global identity and wrist for the -interior center. Avoid rim pixels. `back_project` region mode can summarize a -bounded pixel window with optional `z_min` and `z_max` filtering. - -## Segmentation - -`segment` does not move the robot. Supply exactly one of: - -```json -{"prompt": "the black bowl on the stove", "camera": "agentview", "step": -1} ``` - -```json -{"point": [420, 615], "camera": "agentview", "step": -1} +resources/libero/memory/MEMORY.md ``` -The result includes the mask score, projected `world_xyz`, logical artifact -identifiers for audit, and an embedded overlay image when available. Inspect -the overlay before trusting the projection. +Scan the ~30 one-line hooks. For perception cells **always** open: +- `feedback_no_teleport_rule.md` — the deleted primitives. +- `feedback_redo_cell_timeout_1200.md` — why the cell budget is long here. +(The grasp is oracle-free too: there is no GT-lift oracle — you judge the grasp +from gripper width + the wrist cam, see Rule 1 / 1b.) -Use plain visual phrases. Remove benchmark or brand names that the segmenter -cannot ground. If segmentation fails, choose pixels manually and call -`back_project`. +For bowl→plate spatial tasks always also read `feedback_bowl_eef_y_offset.md` +(bowl-eef y-offset 4.5 cm: place at `eef_y = plate_y + 0.045`, not `plate_y`). -## Grasping +## Rule 1 — `pi0_pick` is grasp-only (no oracle) -Pre-position above the localized object before calling Pi0.5: +`pi0_end_to_end` is FORBIDDEN. `pi0_pick` is for the grasp only; you script every +`move_to` and every `release`. Use: -```json -{ - "prompt": "pick up the short red-label can", +``` +pi0_pick({ + "prompt": "", "max_chunks": 20, "lift_thresh": 0.05, "gripper_closed_thresh": 0.06 -} +}) ``` -Treat `pi0_pick.success` as a hint. Verify the grasp from: - -- end-effector lift; -- nonzero gripper opening consistent with holding an object; -- wrist or agentview evidence that the correct object moved with the gripper; -- an empty or changed source location. - -If the grasp failed, recover in the same episode by re-localizing, -re-positioning, and improving the visual prompt. Do not reset in single-attempt -evaluation mode. - -## Scripted Motion - -Use short, staged `move_to` commands. Do not issue a single horizontal move -larger than approximately 0.30 m. Long commands can switch IK branches and move -the end effector into the wrong half-space. - -After every motion: - -1. Inspect the returned `result` and final distance. -2. Inspect the new state and embedded images. -3. Re-localize anything that may have moved. -4. Continue only when the observed state matches the plan. +`pi0_pick` takes **no object-tracking / oracle argument** — it is grasp-only +and reads NO GT object pose. Passing a name would do nothing even if you tried. +You judge "did I grab the target?" yourself (Rule 1b). NEVER let Pi0 finish the +place — YOU do every `move_to` and the `release`. + +## Rule 1b — JUDGE THE GRASP from perception, NOT from a name + +After a pick, decide "did I grab the target?" from two coord-free signals: + +- **Gripper** (`state.robot0_gripper_qpos` from the latest returned state + record): fingers closed but NOT fully shut (~0.01–0.05 gap) ⇒ holding an + object; fully closed (~0.0) ⇒ grasped air. +- **Wrist cam**: after the lift, inspect `wrist_high.png` — the target + should be raised into the gripper and the spot it came from now empty (its + surface z jumps up by your lift distance). If needed, `back_project` wrist + pixels to confirm the surface z jumped. + +`pi0_pick`'s returned `success` (an eef-lift + gripper-closure heuristic, pure +proprioception) is a HINT, not proof — confirm with the wrist cam before +carrying. The only authoritative TASK-success signal is +top-level `libero_terminated` (the benchmark predicate), which is name-independent. + +## Rule 4 — NO TELEPORT primitives (physics-only) + +`set_object_pose`, `articulate_to`, `js_move_to`, `carry_object` are **deleted +from the codebase**. They are not callable and are not in the tool list. If a +goal is past OSC reach and no physical approach works, write an honest +`libero_terminated:false` audit — never warp. + +## Rule 0 — Use images for reasoning, not just JSON state + +After every primitive tool call, inspect the returned state and embedded images +(the tool returns them — no separate read needed). Inspect +`agentview_high.png` (calibration frame — the one you pick pixels in) +and, when close to a target, `wrist_high.png`. Even more +important here than in oracle mode: this is *the* signal you use to find +objects. + +`agentview_high.png` (and its 256 counterpart `agentview.png`) is the +calibration-frame RGB — the same scene as `agentview_policy.png`, oriented so +that pixel coordinates align with `agentview_metadata.json`. Pick object pixels +from these; the returned JSON `state` gives only proprioception + object names. + +## Rule 2 — SINGLE EPISODE, NO RESET + +This is a **one-shot** evaluation: you get **exactly ONE episode**. Do NOT reset +and do NOT restart the episode. You MAY recover *within* this one episode +(re-localize — objects may have moved; re-pre-position; re-`pi0_pick` a missed +grasp; walk the next rung of the Pi0 prompt ladder in Rule 3; re-firm the grip; +`rotate_pitch`/`move_pose`) — that is all one continuous attempt. But the +instant you would want to start over, **STOP instead and write the audit** +(success or an honest `libero_terminated:false`), then call `finish`. Never +warp; never reset. + +## Rule 5 — Assume every task is physically solvable + +Same as oracle mode. A localization that "looks right" but moves the gripper +into thin air usually means your pixel was on a wrong surface (e.g. picked the +bowl's reflection on the table). Re-look at `agentview_high.png`, pick a +different pixel firmly on the target's top, re-`back_project`. Don't conclude +"unreachable" until you've validated localization. + +## Rule 6 — Your task is `task_language`; the BDDL is FORBIDDEN + +Each state view carries a top-level **`task_language`** field — the +authoritative task instruction (the BDDL's `:language` tag, which contains +**no** coordinates). **Read it and obey it verbatim.** Do not infer the task +from object names, from sibling recipes, or by guessing a `task_map` index — +that produced *wrong-task* runs (an agent solving a task it was never assigned). + +> ⚠️ **Never read the BDDL files, import the benchmark, or query env object +> poses.** The BDDL is the one place the task language *and* the `:init` +> ground-truth coordinates live together — reading it to get the language also +> leaks the coordinates this mode exists to withhold (a perception-isolation +> breach; observed on several swap cells in earlier runs). You already have the +> task from `task_language`; you get object **positions** ONLY by depth +> back-projection. The runtime strips coords from `state` but does **not** +> sandbox the BDDL — that discipline is on you. + +## Rule 7 — Ground the target by its spatial RELATION, not its name + +When the task names a relation ("the bowl **ON THE COOKIES BOX**", "the mug +**LEFT OF** the plate"), the target is whichever object SATISFIES that relation +in the scene — find it by perception. Identical objects (`akita_black_bowl_1` vs +`_2`) carry NO perceptual difference in their names, so the name can't choose +for you; the RELATION disambiguates: + +- "on the cookies box" ⇒ the bowl that is **elevated** (~0.03–0.06 m above the + table, on the box) — distinguish it by its higher world-z from `back_project` + vs the table-level bowl. +- "left/right/front/back of X" ⇒ compare back-projected world xy to X's xy. + +Pick the target purely from where things ARE. (You never need to know which +`_N` name the target is — no primitive in this mode asks for one.) + +## Mental model + +1. **The runner (`rpent/cli/main.py`) owns a long-lived env server** — Pi0.5 + a + single-env LIBERO sim. It launches and manages the server; you do NOT start, + stop, or restart it. +2. **You call one structured MCP tool per step.** The tool BLOCKS until the + toolkit runs that one primitive and dumps the new step, then RETURNS the new + state + log + embedded images. There is no file bus and no polling — the + tool's return value IS your signal. Each dumped step records logical artifact + names such as `agentview_high.png`, `agentview_world_high.npy`, + `wrist_high.png`, and `wrist_world_high.npy` in that step's `artifacts` list. +3. **You read the returned state, localize via `back_project`, decide the next + move, and call the next tool.** + +## Launch a session + +The runner (`rpent/cli/main.py`) launches and owns the env server (Pi0.5 + single-env +sim) — do not start/stop it. You call MCP tools; begin by reading step 0 via +`view_env_state({"step": 0})`. + +## The perception artifacts you read each step + +| logical artifact or field | what's in it | +|---|---| +| `view_env_state` result | `step`, top-level `task_language`, `libero_terminated`, `episode_truncated`, `state.{robot0_eef_pos, robot0_eef_quat, robot0_gripper_qpos, object_names}`, `artifacts`, and nested `log.{command,result,elapsed_s}`. **No object coordinates.** Step `-1` selects latest. | +| `agentview_policy.png` | RGB in Pi0 frame. *Do not pick pixels here for back-projection.* | +| `agentview.png` | 256×256 agentview RGB in **calibration frame**. Pick object pixels HERE only with `back_project(..., resolution:"low")`. | +| `agentview_high.png` | **HI-RES 1024×1024** agentview RGB, calibration frame. **PREFER this for looking / identification**; `back_project` defaults to `resolution:"high"`. | +| `agentview_depth.npy` | 256×256 agentview metric depth (m), aligned with `agentview.png`. Consume through `back_project`. | +| `agentview_world.npy` / `agentview_world_high.npy` | Precomputed agentview world xyz per low/high-resolution pixel. Prefer `back_project`; do not open them manually. | +| `wrist.png` / `wrist_high.png` | **Wrist (eye-in-hand) RGB**, calibration frame. Moves with the gripper. | +| `wrist_depth.npy` | Wrist metric depth (m), aligned with `wrist.png`. | +| `wrist_world.npy` / `wrist_world_high.npy` | Precomputed wrist world xyz per low/high-resolution pixel in the SAME world frame as agentview. May be all-table until you move over the target. | +| `wrist_metadata.json` | Wrist intrinsics + extrinsic **for THAT step only**. Read via `view_camera_meta({"camera":"wrist","step":NN})`. | +| `agentview_metadata.json` | Agentview intrinsics, cam-to-world extrinsic, depth range, and projection notes. Read via `view_camera_meta({"camera":"agentview","step":NN})`. | +| `segment_XX.json` | Per-step segment record named by the returned `segment_artifact`; includes mode, camera, source step, score, box, centroid, and `world_xyz`. | +| `segment_overlay_XX.png` | Per-step overlay named by `overlay_artifact` and embedded by a successful `segment` result for visual confirmation. | + +The command, result, and `elapsed_s` are returned under `log`; there is no +separate per-step log file to read. + +## Localization with `back_project` (coarse agentview → fine wrist) + +You index the precomputed map through `back_project` — no K⁻¹ math. Coarse +(agentview) then, once the gripper is parked over the target, fine (wrist): -Use `rotate_wrist`, `rotate_pitch`, or `move_pose` when orientation matters. -Use `pi0_doubled` for short learned contact interactions such as knobs, -buttons, doors, or drawers. Alternate it with short, capped scripted alignment -motions; never use one long blind push. - -## Placement - -Carry objects at a collision-safe height. Reconfirm the destination before -release, especially when plate/burner or rim/interior confusion is possible. - -For open containers: - -- localize the interior rather than the nearest rim; -- place above the interior center; -- lower enough to avoid a high-energy drop; -- inspect the post-release observation and outcome flag. - -If the task predicate does not fire, reclassify the destination before assuming -the grasp failed. Wrong-surface placement is a common cause. - -## Completion And Audit - -The current outcome is the top-level `libero_terminated` value returned by the -latest environment tool. Do not look for it inside `state`. - -On completion, write the requested audit with: - -- suite, task, seed, and evaluation regime; -- exact memory files consulted; -- semantic identification and localization strategy; -- grasp and placement evidence; -- final state and `libero_terminated`; -- honest failure details when the task remains incomplete. +``` +# COARSE: agentview hi-res (default resolution:"high") — choose which object / rough xy +back_project({"row": ROW, "col": COL, "step": NN}) -Then call `finish`. Recipe generation is handled by the runtime from recorded -primitive and successful segmentation events. +# FINE: wrist — after move_to ~15-20cm above the target, pick its pixel in +# wrist_high.png and back_project it (±1-2cm; refines the SAME candidate). +back_project({"row": ROW, "col": COL, "step": NN, "camera": "wrist"}) +``` -## Final Checklist +(Pass `"resolution":"low"` only when `ROW,COL` came from a 256 image.) + +Tips: + +- Sample 3–5 pixels on the object (centre + a couple of edge pixels) and + median the back-projected xy — robust to a single mis-picked pixel. Avoid + pixels on the thin rim/edge or the gap to the table (those index a + background/edge depth and give a world point metres away). +- The returned point is the **visible surface** under your chosen pixel. + For a flat object (plate, basket) that surface ≈ the place target. For a + bowl/bottle, the surface is the top of the object; the rim's xy is what + you want for `release`, not the grasp's eef_y (apply + `feedback_bowl_eef_y_offset` — bowl `eef_y_target = perceived_plate_y + 0.045`). +- For the **table z** (when you need a known floor to compare against), + `back_project` a pixel on bare table near the object. + +## The command vocabulary + +Call one structured tool per step. The full primitive set (only the *control +signals* are listed here) — every one blocks until the step is dumped and +returns the new state, log, artifact names, and embedded images: + +```jsonc +// === physics-only primitives (the entire allowed set) ===================== + +// Scripted EEF servo. action_scale=0.05 is the env's units; step_clip caps +// per-step Δxyz (m) BEFORE division by action_scale — smaller = slower. +move_to({"xyz": [x, y, z], "gripper": -1, + "tol": 0.012, "step_clip": 0.025, "max_steps": 80, + "action_scale": 0.05, "target_yaw": null}) // gripper: -1 open, +1 close + +// Pi0.5 closed-loop pick. Grasp-only — no oracle/tracking arg (Rule 1). +// You judge the grasp from gripper width + wrist cam (Rule 1b). +pi0_pick({"prompt": "pick up the X", "max_chunks": 20, + "lift_thresh": 0.05, "gripper_closed_thresh": 0.06}) + +// Pi0.5 for a contact skill (knob turn, drawer/door open-close — rare here). +// success mirrors libero_terminated only; inspect image/state for intermediates. +pi0_doubled({"prompt": "turn off the stove", "max_chunks": 20}) + +// Open gripper to place. Triggers libero termination if the On/In predicate met. +release({"max_steps": 20}) + +// Hold pose + drive gripper. Use to firm a grip mid-carry. +set_gripper({"gripper": 1, "steps": 5}) // -1 open, +1 close + +// Wrist yaw (world-z). Provide target_yaw (absolute) OR delta_yaw (relative). +rotate_wrist({"target_yaw": 0.0, "gripper": 1, + "max_steps": 40, "tol": 0.02, "step_clip": 0.10}) + +// Tilt eef pitch (axis-angle X). Cavity entry / micro-aiming. target_pitch OR +// delta_pitch. Use before threading a narrow opening whose face normal is ±y. +rotate_pitch({"target_pitch": 0.9, "gripper": 1, + "max_steps": 40, "tol": 0.02, "step_clip": 0.10}) + +// Co-vary xyz + pitch + yaw — threads cabinet-front IK singularity that move_to +// walls at. gripper defaults to -1 (OPEN) — pass gripper:1 while holding. +move_pose({"xyz": [x, y, z], "target_pitch": 0.0, "target_yaw": 0.0, + "gripper": 1, "step_clip": 0.02, "pitch_step": 0.08, "yaw_step": 0.08, + "tol": 0.012, "ori_tol": 0.05, "max_steps": 150}) + +// SAM3-grounded localization — does NOT move the robot and does NOT +// replace manual back-projection. Segments the most-recent dumped image for the +// prompt, back-projects the mask through the matching world map, and writes +// segment_XX.json {score, box, centroid_pixel, world_xyz (robust MEDIAN over +// the whole mask), n_pixels} + segment_overlay_XX.png. Read world_xyz +// directly instead of eyeballing a pixel. camera "wrist" uses the wrist world +// map for fine refinement (move the eef over the target first). Pass +// "point":[row,col] for a point prompt instead of text; prompt and point are +// mutually exclusive. min_score default 0.2. +// RPent manages the SAM3 service. If one call fails or finds nothing, the result is +// an {"error":...,"fallback":...} dict — fall back to picking a pixel in +// agentview_high.png and calling back_project. +segment({"prompt": "the black bowl on the stove", "camera": "agentview", + "point": null, "min_score": 0.2}) + +// === FORBIDDEN — DELETED FROM THE CODE — DO NOT EMIT ============ +// set_object_pose, articulate_to, js_move_to, carry_object +``` -- Initial state read with `step: 0`. -- Task language copied from the returned tool result. -- All targets and destinations semantically identified in agentview. -- Coordinates obtained through `back_project` or `segment`. -- Wrist refinements checked against the agentview anchor. -- Grasp confirmed visually and proprioceptively. -- Long motion split into safe waypoints. -- Destination reclassified immediately before release. -- Latest result checked for top-level `libero_terminated`. -- Audit written before `finish`. +### How to use `segment` (SAM3) — practical tips + +`segment` is the fastest way to localize: one call gives you a `world_xyz` +that is a robust MEDIAN over the whole object mask (hundreds of pixels), which +beats eyeballing 3–5 pixels. Workflow: + +1. Inspect `agentview_high.png`, decide the target by the task's spatial RELATION. +2. Call `segment` with a **plain visual phrase + the relation**, then read the + returned `world_xyz` and embedded overlay image to confirm the mask + is on the right object before you move. +3. **Two camera views — YOUR choice via the `"camera"` field** (`segment` works + on either; default `"agentview"`): + - `"camera":"agentview"` — the fixed ~1 m cam (±8–13 cm). Use it to choose + WHICH object (global layout / spatial relation) and get a rough xy. + - `"camera":"wrist"` — the eye-in-hand cam (±1–2 cm), reads the wrist world + map in the SAME world frame. Use it to PRECISELY localize once the gripper + is parked over the target. ⚠ It is null / all-table until you `move_to` + ~15–20 cm over the target (the wrist only sees the gripper+table before + that), so segment agentview first, approach, THEN segment wrist. + Typical coarse→fine: agentview select → `move_to` above → wrist refine → grasp. + You decide when the wrist refinement is worth it (small / closely-spaced / + identical objects benefit most; a big isolated plate may not need it). + +**Prompt phrasing (important — SAM3 is sensitive):** +- ✅ Use plain colour + shape + spatial relation: `"the black bowl on the stove"`, + `"the white plate"`, `"the bowl on the cookies box"`. +- ❌ Do NOT use the object's internal/brand NAME from `object_names`/BDDL: + `"the akita black bowl"` scores ~0.03 (no detection) because SAM3 can't ground + "akita"; the same object as `"the black bowl on the stove"` scores ~0.76. Strip + proper nouns (`akita`, `glazed_rim_porcelain_…`) — say what it *looks like*. +- For two identical objects, the relation in the prompt (`…on the stove`, + `…left of the plate`) usually steers SAM3 to the right instance; verify via the + overlay and the world-z (an object *on* a fixture has a higher z than a + table-level twin). The two-camera relation protocol still applies when SAM3 + can't disambiguate from text alone. +- If `segment` returns `{"error":...}` (low score / service down), walk the + prompt (drop the brand word, simplify, add the relation) or fall back to + manual pixel → `back_project`. + +## The strict-hybrid recipe (perception variant) + +A typical bowl→plate cell looks like: + +1. `view_env_state({"step": 0})`; inspect `task_language`, + `agentview_high.png`, and call `view_camera_meta` if needed. +2. Identify the target by the task's spatial RELATION, not its name (Rule 7). +3. Localize it — pick its pixel in `agentview_high.png`, + `back_project({"row":ROW,"col":COL,"step":0})` (coarse). Refine with the + wrist cam after pre-positioning (fine). +4. **Pre-position** ~15–20 cm above it: `move_to([obj_x, obj_y, carry_z])`, + gripper open. Re-`back_project` a wrist pixel here and correct the xy before + grasping. +5. `pi0_pick` with the right prompt (start "pick up the {object}", escalate per + the Pi0 ladder — see below). Confirm the grasp via gripper + wrist (Rule 1b). +6. `set_gripper({"gripper":1,"steps":5})` to firm the grip. +7. Localize the placement region (basket / plate / drawer slot) the same way. +8. `move_to([place_x, place_y, carry_z])` to traverse at constant height. +9. Optionally descend `move_to([place_x, place_y, place_z])`. +10. `release` — predicate (`On`/`In`) checks → `libero_terminated=True` if hit. +11. Light retreat (`move_to` upward) so the next step's image is clean. + +> **Predicate fire timing.** Most LIBERO `On(X, Y)` predicates fire on +> `release` if `X` is above `Y`'s region. `In(X, container)` needs `X` to +> have actually entered the container's volume before release. If a release +> fires `term=False` but the object is on top of the right region, descend +> 1–2 cm more and re-`release`. + +## Pi0 prompt ladder (Rule 3 — Pi0 IS the delivery service) + +Try in order; each rung uses a slightly more specific prompt or a re-pre-pos. + +1. `"pick up the {object}"` — generic sub-instruction. +2. The full `task_language` verbatim (e.g. `"Pick the akita black bowl on the + cookies box and place it on the plate"`). +3. Add a spatial qualifier (`"…on the wooden cabinet"`, `"…next to the + basket"`). +4. Re-pre-position 5 cm lower or shifted, then retry rung 2 or 3. + +Empirically Pi0 sometimes needs the **full prompt with the spatial +qualifier** for elevated picks (stove, cabinet-top, drawer). See +`feedback_pi0_pick_full_prompt.md` and the prompt-ladder note in MEMORY. + +## Key hyperparameters + +- Single-step `xyz` within ±0.30 m of current eef or OSC flips IK. Split + long traversals into 2–3 carry-z waypoints. +- `lift_thresh`: 0.05 (flat/stable) / 0.08 (slippery tall bottles). +- `step_clip`: 0.025 (empty / box) / 0.015 (cans) / 0.012 (tall bottles). +- Frame z (from `state.robot0_eef_pos[2]` at step 00): + ≈ 0.68 → LIVING_ROOM, ≈ 1.17 → KITCHEN, ≈ 0.26 → OBJECT. +- BOWL: `eef_y_target = perceived_plate_y + 0.045` (bowl-eef y-offset). +- TALL BOTTLES: carry at `z=0.30`, release without descending. +- Approach high-then-vertical; recover by re-`pi0_pick`, not by hovering. + +## Reading state + +After every primitive tool call, the return value already carries the new +`state`, nested `log.{command,result,elapsed_s}`, artifact names, and embedded +images — +you do **not** need a separate read. When you need an older step, call +`view_env_state({"step": NN})` (use `step: -1` for the latest): + +1. Check `log.result.success`, `log.result.final_dist_m`, + `log.result.peak_lift_m`, etc. +2. Inspect the embedded `agentview_high.png` image → visual confirmation; pick + pixels for any new localization. +3. Check `state.robot0_eef_pos`, `state.robot0_gripper_qpos`, and top-level + `libero_terminated` in the returned view. + +You **do not** open the depth maps yourself — feed a pixel to `back_project`. +Don't call `view_env_state` immediately after a primitive that already +returned the new state. + +## Common failure modes + +- **Pick missed (gripper closed empty).** `log.result.peak_lift_m` < + `lift_thresh`, `log.result.min_gripper_opening` ≈ 0. Re-pre-position 1–2 cm + lower or shifted; retry + Pi0 with next prompt-ladder rung. +- **Object slipped mid-carry.** `release` returns top-level + `libero_terminated=false` and the object + is no longer where you `release`d it. `release`, re-pre-pos above it, + `pi0_pick` again, traverse again. +- **OSC stuck.** `move_to` returns `log.result.final_dist_m > 0.05` at + `log.result.max_steps` and + same xy twice → try `rotate_pitch`, split into more waypoints, or + `move_pose` (co-varying) for the cabinet-front singularity. +- **Placement off because localization was wrong.** The release puts the + object on bare table instead of on the plate. Re-inspect `agentview_high.png`, + pick a different pixel on the target region (sample 3 pixels on the plate's + flat top, median the back-projected xy), redo. Tip: if depth at your chosen + pixel is much closer than the table z, you picked the camera's near edge / + an object rim — pick again. + +## Verifying strict compliance + +Before saving the audit, confirm your command history is physics-only. The +teleport primitives are not even in the tool list, but audit it anyway: inspect +each recorded step with `view_env_state` and confirm every `log.command.action` +is one of the +allowed physics primitives (`move_to`, `pi0_pick`, `pi0_doubled`, `release`, +`set_gripper`, `rotate_wrist`, `rotate_pitch`, `move_pose`) — no +`set_object_pose` / `articulate_to` / `js_move_to` / `carry_object` appears. + +## Persisting successful runs as audit JSONs + +When top-level `libero_terminated == true`: + +a. The working command recipe (`{output_dir}/recipe_{recipe_tag}.jsonl`) is + **auto-exported by the runner** from non-error primitive commands in the + recorded state trace plus successful `segment_XX.json` artifacts, merged in + execution order — you do NOT + hand-write it. +b. Write a minimal audit JSON with `write_text_file` to + `{output_dir}/{recipe_tag}.json` with at least: `suite`, `task_id`, `seed`, + `regime: "strict_perception"`, `strategy_notes` (mention HOW you localized — + which pixel, depth, back-projected world xyz), `pick_result` (the `result` + from your `pi0_pick`), `final_state` (the latest returned `state` field), + `libero_terminated: true`. +c. Call `finish({"status":"success","summary":"…"})`. + +If unrecoverable after honest exploration in this one episode, write +`{output_dir}/{recipe_tag}.json` with `libero_terminated: false` + +`strategy_notes` describing what you tried, the back-projected xyz you used, and +which step failed. Then call `finish` (NO reset, NO second attempt). + +> The `regime: "strict_perception"` tag distinguishes these audits from the +> oracle-state `strict` regime in mixed-mode datasets. + +## Iteration heuristics + +- After 2 failed retries on the same step, **stop tuning numerics** and + inspect `agentview_high.png` at pick / pre-release / + post-release. The visual disagreement is usually the bug (you picked the + wrong pixel, or Pi0 grabbed the decoy). This is the lesson of + `feedback_failure_forensics.md` — applies even more strongly here. +- If a release reports `term=False` but the object is visibly on the target + region, descend 1–2 cm more and `release` again; predicate often needs + contact, not just hover. +- Once you use `back_project`, trust it: if a localization "feels off", it's + because you picked the wrong pixel — not because the depth / calibration is + wrong. (`back_project` inverts `K` before the extrinsic; the raw + `E @ [col·z, row·z, z, 1]` form skips it and is wrong — which is why you use + `back_project` and never hand-roll the math.) + +## What "strict_perception" means concretely + +- **No GT object coordinates anywhere in your reasoning.** The `state` you + read has none; the only legitimate sources of object xyz are the camera + images + depth + the precomputed agentview and wrist world maps (via + `back_project`). +- **No teleport primitives.** The four are deleted. +- **Pi0 only does the grasp.** You script every motion + release. +- **Fully oracle-free, including the grasp.** There is no GT-lift oracle — + `pi0_pick` reads NO GT object pose and takes no tracking argument. You judge + the grasp from gripper width + the wrist cam, and TASK success from + top-level `libero_terminated` (the benchmark predicate). +- **Single attempt.** One episode, no reset (Rule 2). +- The expected audit `regime` is `strict_perception`. + +## Reference cases + +The seed-0 sweep results live under `resources/libero/results_*_pert/` +(`results_10_pert`, `results_object_pert`, `results_spatial_pert`, +`results_goal_pert` — PRO swap+task, t0–t9). Each solved cell has an audit JSON ++ a `recipe_{tag}.jsonl` command sequence. The consistent winning pattern: +localize → pre-pos → `pi0_pick` → `set_gripper` → move → `release` in 6–12 +commands. + +When you write a new audit, browse a sibling cell's `recipe_{tag}.jsonl` as a +*technique* template — but never paste its xyz; re-derive every position via +`back_project` from THIS scene's depth. + +Begin by reading `resources/libero/memory/MEMORY.md`, then call +`view_env_state({"step": 0})` and inspect `agentview_high.png` +(+ metadata via `view_camera_meta`); localize the target object via +`back_project`, then plan and execute. diff --git a/robots/libero/prompts/system.py b/robots/libero/prompts/system.py index d93b8df8..f0eba22d 100644 --- a/robots/libero/prompts/system.py +++ b/robots/libero/prompts/system.py @@ -10,7 +10,8 @@ > persistence / up to N attempts" instruction anywhere below).** This is a > ONE-SHOT evaluation: you get **exactly ONE episode**. You MUST NOT call > `reset`, and you must not restart the episode. Plan carefully, then execute -> your single best manipulation sequence toward `libero_terminated == true`. +> your single best manipulation sequence toward top-level +> `libero_terminated == true`. > You MAY recover *within* this one episode (re-pre-position, re-`pi0_pick` a > missed grasp, walk the Pi0 prompt ladder, `rotate_pitch`/`move_pose`) — that is > all one continuous attempt — but the instant you would want to reset/start over, @@ -70,7 +71,7 @@ weak at reading side labels or distinguishing similar grocery items (ketchup/BBQ/tomato sauce, soup cans, cream cheese/butter). Do NOT let the wrist freely re-identify a non-basket target; it often locks onto a look-alike. - Instead: choose the target from the high-resolution agentview image, compute its + Instead: choose the target from `agentview_high.png`, compute its agentview xyz, move over that candidate, project/track that SAME candidate in wrist, and refine only its surface/center coordinates. SAM3 scores ~0.02-0.06 on brand nouns ("alphabet soup", "tomato sauce") — prompt by colour+shape ("the short @@ -146,28 +147,33 @@ - Under some runtimes these same tools may appear namespaced; call the actual tool name shown in your tool list, preserving the same arguments and semantics. -The driver records a state and an `observation` dictionary for every motion -step. Observation entries name the available policy, agentview, wrist, depth, -world-map, and metadata artifacts, but storage paths are internal to the -runtime. Do not construct or read artifact paths manually. +Each state record exposes `step`, top-level `task_language`, +`libero_terminated`, `episode_truncated`, coord-free `state`, an `artifacts` +list of logical base names, and a `log`. Storage paths are internal; do not +construct or parse them. -Use `view_env_state` to retrieve a state. It embeds the policy image and the -best available agentview and wrist images, preferring high resolution. Use -`view_camera_meta`, `back_project`, and `segment` to consume metadata, depth, -and world maps. These tools guarantee that the selected camera, resolution, -and step use matching artifacts. +Canonical observation keys include `agentview_policy.png`, `agentview.png`, +`agentview_high.png`, `agentview_depth.npy`, `agentview_world.npy`, +`agentview_world_high.npy`, `agentview_metadata.json`, `wrist.png`, +`wrist_high.png`, `wrist_depth.npy`, `wrist_world.npy`, +`wrist_world_high.npy`, and `wrist_metadata.json`. -Step `0` is the initial state. Step `-1` selects the latest state.""" +Use `view_env_state` to retrieve a record. It embeds the policy image and the +best available agentview and wrist images, preferring the matching `*_high.png` +artifact. Use `view_camera_meta`, `back_project`, and `segment` to consume +metadata and world maps without opening artifacts directly. Step `0` is the +initial state; step `-1` selects the latest state.""" GOAL = """YOUR GOAL: produce top-level `libero_terminated == true` in ONE episode. ⛔ NO `reset`, NO retry (SINGLE-ATTEMPT MODE — see the override at the very top; it supersedes any reset/retry wording in the Rules below).""" RULES = """Rule 0 — USE IMAGES. After every primitive tool call, inspect the returned state - and embedded images. If you need a state again, call `view_env_state`. - Use agentview for global layout and wrist for close-range geometry. The image - is your spatial-reasoning input; the JSON state only gives proprioception + - object names. + and embedded images. If you need a state again, call `view_env_state`. Inspect + `agentview_high.png` (the calibration-frame image used for pixel selection) + and, when close to a target, `wrist_high.png`. The image is + your spatial-reasoning input; the returned `state` field only gives + proprioception + object names. Rule 1 — Pi0 is ONLY for the grasp. Use: pi0_pick({ @@ -185,19 +191,19 @@ Rule 1b — JUDGE THE GRASP from perception, NOT from a name. After a pick, decide "did I grab the target?" from two coord-free signals: • GRIPPER (proprioception): `state.robot0_gripper_qpos` from the latest - state record — fingers closed but NOT fully shut (~0.01–0.05 gap) + state record — fingers closed but NOT fully shut (~0.01–0.05 gap) ⇒ holding an object; fully closed (~0.0) ⇒ grasped air. - • WRIST CAM: inspect the returned wrist image after lifting. The target should + • WRIST CAM: inspect the embedded `wrist_high.png` image after lifting. The target should now be raised into the gripper, and the spot it came from should be EMPTY. Compare before/after wrist or agentview evidence; if needed, use `back_project` on wrist pixels to confirm the target surface z jumped up. `pi0_pick.success` (eef-lift + gripper-closure heuristic) is a HINT, not proof — always confirm with the wrist cam before carrying. -Rule 2 — Inspect THEN act. Call `view_env_state({"step": 0})`, inspect the - returned high-resolution images, and inspect the relevant memory/guides - BEFORE your first primitive. **Your task is the returned `task_language`; - read it and obey it verbatim.** This is the authoritative instruction (the BDDL +Rule 2 — Inspect THEN act. Call `view_env_state({"step": 0})`, inspect + `agentview_high.png` and the relevant memory/guides BEFORE your first + primitive. **Your task is the returned `task_language`; read it and obey it + verbatim.** This is the authoritative instruction (the BDDL `:language` tag). Do NOT infer the task from object names, from sibling recipes, or by guessing a task_map index — those caused wrong-task runs in the past. @@ -227,7 +233,7 @@ plate, a stove burner/cook-region, a wooden-cabinet top, and a pot lid all read as "flat disc at table height" in back-projected coordinates. They are only separable in the RGB. So before you carry-and-release onto a surface, look at - returned agentview image (and the wrist image once close) and + `agentview_high.png` (and `wrist_high.png` once close) and NAME each candidate surface: • PLATE ⇒ ceramic disc, usually white, with a clean raised rim (often colored concentric rings). This is the place target for "place it on the plate". @@ -262,7 +268,8 @@ LOCALIZATION = """This is the core of perception-isolated mode. To find where an object is: -1. Look at the returned agentview image (high resolution when available) and find the target +1. Look at `agentview_high.png` (1024x1024 — PREFER THIS; fall back to + `agentview.png` only if the high-resolution artifact is absent) and find the target object's pixel (row, col). (row = vertical/y from top, col = horizontal/x from left.) 2. Call `back_project` on that pixel: @@ -285,7 +292,8 @@ metres away. Pick pixels firmly on the object's top surface.) ALWAYS apply the manipulation offsets from memory to the PERCEIVED position -(e.g. BOWL: eef_y = plate_y + 0.045). Verify visually in agentview after moving.""" +(e.g. BOWL: eef_y = plate_y + 0.045). Verify visually in `agentview_high.png` +after moving.""" PERCEPTION_ALGORITHM = """This is the default perception algorithm for EVERY cell (from the 80-task localization sweep: `agentview_identity_wrist_geometry_except_basket`). @@ -304,11 +312,11 @@ ALGORITHM (run this BEFORE manipulating): -1. From the initial `task_language` + returned agentview image + +1. From the initial `task_language` + `agentview_high.png` + object_names, infer the task-relevant TARGETS and DESTINATIONS (language only; never BDDL/poses). -2. GLOBAL SEMANTIC PASS (agentview hi-res): in the returned agentview image choose each +2. GLOBAL SEMANTIC PASS (agentview hi-res): in `agentview_high.png` choose each target/destination candidate by RGB, label/shape, and global spatial relation. For duplicates (two bowls/plates/mugs) pick by RELATION (on stove, on cookie box, left/right/front/back), not `_1/_2`. For sauce/can/box groceries use the @@ -317,7 +325,7 @@ burner vs cabinet/drawer vs basket) semantically in RGB here. 3. COARSE XYZ (agentview): pick 3-8 pixels firmly on the chosen candidate in - agentview image, call `back_project` on the SAME pixels, take the median. + `agentview_high.png`, call `back_project` on the SAME pixels, take the median. Avoid edges/holes/shadows/table-gaps. This median is the IDENTITY ANCHOR for that entity. @@ -400,8 +408,8 @@ command lists. Re-derive every coordinate from THIS scene. """, """INSPECT INITIAL STATE: call `view_env_state({"step": 0})`; inspect - `task_language`, object_names, eef pose, the returned agentview and wrist images, - and call `view_camera_meta` if needed. Identify ALL target + `task_language`, object_names, eef pose, `agentview_high.png`, + `wrist_high.png` if useful, and call `view_camera_meta` if needed. Identify ALL target objects, destination surfaces, and relation landmarks named by task_language. """, """RUN THE MANDATORY PRE-TASK PERCEPTION PASS (FIRST-STEP ALGORITHM above) — @@ -419,9 +427,9 @@ pi0_pick({"prompt": "...", "max_chunks": 20, ...}) release({}) -Each primitive tool blocks until the next state record is dumped and returns the -new state view, log, and embedded images. Inspect them, use `back_project` as -needed, decide, and repeat. +Each primitive tool blocks until the next state record is dumped and returns +the new state view, log, and embedded images. Inspect `agentview_high.png` and +`wrist_high.png` as needed, call `back_project` for geometry, decide, and repeat. """, """ALLOWED PRIMITIVES (physics-only; full schemas in the tool list/guides): `move_to`, `pi0_pick`, `pi0_doubled`, `release`, `set_gripper`, @@ -444,9 +452,8 @@ a pixel, call `segment({"prompt":"the black bowl on the cookies box", "camera":"agentview"})`. It runs SAM3 on the current image, back-projects the mask via the matching world map, and returns a robust median `world_xyz` plus -an embedded overlay image for visual confirmation. The logical -`segment_artifact` and `overlay_artifact` names are audit references, not paths -to open manually. Use `camera":"wrist"` (after parking the eef ~15–20 cm over the +logical `segment_artifact` and `overlay_artifact` names. Inspect the embedded +overlay image to confirm the right object. Use `camera":"wrist"` (after parking the eef ~15–20 cm over the target) for ±1–2 cm refinement, or `"point":[row,col]` for a point prompt. Text `prompt` and `point` are mutually exclusive; provide exactly one. ⚠ PROMPT PHRASING (SAM3 is sensitive): use a plain colour+shape+RELATION phrase, @@ -468,7 +475,7 @@ unrecoverable within this one episode, do NOT reset — write an honest stuck-audit (`libero_terminated:false`) and call `finish`. Never warp. """, - """WHEN `libero_terminated == true` in the latest tool result: + """WHEN top-level `libero_terminated == true` in the latest tool result: a. Write audit `{{output_dir}}/{{recipe_tag}}.json` with: suite, task_id, seed, regime:"strict_perception", strategy_notes (incl. how you localized), pick_result, final_state (latest state's `state`), diff --git a/robots/libero/prompts/user.py b/robots/libero/prompts/user.py index a71b2444..a4437e9d 100644 --- a/robots/libero/prompts/user.py +++ b/robots/libero/prompts/user.py @@ -10,11 +10,10 @@ - recipe: {{output_dir}}/recipe_{{recipe_tag}}.jsonl""" -MODE = """Inspect the embedded high-resolution images returned by -view_env_state, then use back_project or segment to localize objects before -motion.""" +MODE = """Inspect `agentview_high.png` returned by `view_env_state`, then use +`back_project` or `segment` to localize objects before motion.""" BEGIN = """Read MEMORY.md and the guides, then call -`view_env_state({"step": 0})` and inspect its embedded images. Localize every -task-relevant entity before planning and execution.""" +`view_env_state({"step": 0})` and inspect `agentview_high.png`. Localize the +target, then plan and execute.""" From c9287950dc7e823ba4c084fbead86817897e19d7 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Thu, 6 Aug 2026 09:18:05 +0000 Subject: [PATCH 10/25] refactor(dashboard): stream video artifacts from files Signed-off-by: Jiaxing Qiu --- rpent/dashboard/server.py | 15 +++++++-------- rpent/dashboard/state.py | 12 +++++------- rpent/tools/state.py | 5 +++++ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/rpent/dashboard/server.py b/rpent/dashboard/server.py index 060f8603..d6a3bd0f 100644 --- a/rpent/dashboard/server.py +++ b/rpent/dashboard/server.py @@ -257,11 +257,10 @@ def api_frame( @app.get("/api/run/video") def api_video(run: str) -> Response: live = self._resolve(run) - video = live.video() if live else None - if video is None: + if live is None or not live.has_video(): return Response(status_code=404) - return Response( - video, + return FileResponse( + live.video_path, media_type="video/mp4", headers={"Cache-Control": "no-store, max-age=0"}, ) @@ -269,11 +268,11 @@ def api_video(run: str) -> Response: @app.get("/api/run/action-video") def api_action_video(run: str, step: int) -> Response: live = self._resolve(run) - video = live.action_video(step) if live else None - if video is None: + path = live.action_video_path(step) if live else None + if path is None: return Response(status_code=404) - return Response( - video, + return FileResponse( + path, media_type="video/mp4", headers={"Cache-Control": "no-store, max-age=0"}, ) diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py index ce2bb920..0a339ef6 100644 --- a/rpent/dashboard/state.py +++ b/rpent/dashboard/state.py @@ -835,7 +835,7 @@ def frame(self, kind: str) -> bytes | None: with self._lock: return self._frames.get(kind) - def action_video(self, step: int) -> bytes | None: + def action_video_path(self, step: int) -> Path | None: env_state = self.env_state with self._lock: artifact = None @@ -848,17 +848,15 @@ def action_video(self, step: int) -> bytes | None: break if artifact and env_state is not None: try: - return env_state.load_bytes(artifact, step=int(step)) - except FileNotFoundError: + path = env_state.artifact_path(artifact, step=int(step)) + except (LookupError, ValueError): return None + return path if path.exists() else None if raw_path: video_path = Path(raw_path) - return video_path.read_bytes() if video_path.exists() else None + return video_path if video_path.exists() else None return None - def video(self) -> bytes | None: - return self.video_path.read_bytes() if self.video_path.exists() else None - def has_video(self) -> bool: with self._lock: return self._task_state in TERMINAL_RUN_STATES and self.video_path.exists() diff --git a/rpent/tools/state.py b/rpent/tools/state.py index a059cb77..a0d4baad 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -286,6 +286,11 @@ def load_bytes(self, name: str, *, step: int | None = -1) -> bytes: resolved_step = self._resolve_read_step(step) return self._artifact_file(name, resolved_step).read_bytes() + def artifact_path(self, name: str, *, step: int | None = -1) -> Path: + """Return the canonical filesystem path for an artifact.""" + resolved_step = self._resolve_read_step(step) + return self._artifact_file(name, resolved_step) + def exists(self, name: str, *, step: int | None = -1) -> bool: try: resolved_step = self._resolve_read_step(step) From f2ff6414e861b7758d5a1f79171114f07ff39c16 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Thu, 6 Aug 2026 09:25:12 +0000 Subject: [PATCH 11/25] fix(libero): report segment artifact persistence failures Signed-off-by: Jiaxing Qiu --- robots/libero/tools.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/robots/libero/tools.py b/robots/libero/tools.py index 2f66eb76..08baff58 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -708,7 +708,7 @@ def segment( if not data.found: segment_blob["error"] = data.reason or "SAM3 found no mask" segment_blob.update(world_result) - state.save( + saved_segment = state.save( segment_name, segment_blob, step=nn, @@ -719,13 +719,21 @@ def segment( "step": nn, "camera": camera, "image_artifact": image_name, - "segment_artifact": segment_name, "score": segment_blob["score"], "box": segment_blob["box"], "world_xyz": segment_blob["world_xyz"], "world_error": segment_blob.get("world_error"), } - if "error" in segment_blob: + if saved_segment is None: + result["error"] = f"failed to persist segment artifact {segment_name}" + result["code"] = "segment_artifact_save_failed" + result["attempted_segment_artifact"] = segment_name + if "error" in segment_blob: + result["segmentation_error"] = segment_blob["error"] + result["fallback"] = "Use manual visual localization and back_project." + else: + result["segment_artifact"] = saved_segment + if saved_segment is not None and "error" in segment_blob: result["error"] = segment_blob["error"] result["fallback"] = "Use manual visual localization and back_project." if saved_overlay is not None: From 2f0faf7ff362429e69faae8657935a885a88d62c Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 03:28:45 +0000 Subject: [PATCH 12/25] main: generalize env argparse Signed-off-by: Jiaxing Qiu --- rpent/cli/main.py | 15 +++++++++++---- rpent/envs/__init__.py | 3 ++- rpent/envs/base.py | 12 ++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/rpent/cli/main.py b/rpent/cli/main.py index a22bbc30..92b083f7 100644 --- a/rpent/cli/main.py +++ b/rpent/cli/main.py @@ -37,7 +37,7 @@ NullDashboardEventSink, RunStartedEvent, ) -from rpent.envs import get_env_spec, get_toolkit +from rpent.envs import enumerate_envs, get_env_spec, get_toolkit from rpent.planner.base import build_planner from rpent.utils.logging import get_logger, init_output_dir from rpent.utils.resources import ensure_resources @@ -82,12 +82,19 @@ def _serialize_messages(messages: list[dict]) -> list[dict]: def _build_argparser() -> argparse.ArgumentParser: + known_envs = enumerate_envs() + known_envs_text = ", ".join(known_envs) if known_envs else "none" ap = argparse.ArgumentParser( - description="Standalone hybrid LLM-in-the-loop physical agent", + description="RPent: Agentic Infrastructure for the Physical World", ) - ap.add_argument("--env", dest="env_name", required=True, choices=["libero"], - help="Environment backend: libero.") + ap.add_argument( + "--env", + dest="env_name", + required=True, + choices=known_envs, + help=f"Environment backend. Known environments: {known_envs_text}.", + ) # models ap.add_argument("--planner", default="api", diff --git a/rpent/envs/__init__.py b/rpent/envs/__init__.py index f2720d34..7bfc9f0c 100644 --- a/rpent/envs/__init__.py +++ b/rpent/envs/__init__.py @@ -1,13 +1,14 @@ """Environment-specific RPent extensions.""" +from rpent.envs.base import enumerate_envs, get_env_spec, get_toolkit from rpent.envs.env_spec import EnvSpec, RunConfig from rpent.envs.prompt_bundle import PromptBundle -from rpent.envs.base import get_env_spec, get_toolkit __all__ = [ "EnvSpec", "PromptBundle", "RunConfig", + "enumerate_envs", "get_env_spec", "get_toolkit", ] diff --git a/rpent/envs/base.py b/rpent/envs/base.py index dfe5758d..9c8e4b91 100644 --- a/rpent/envs/base.py +++ b/rpent/envs/base.py @@ -10,6 +10,7 @@ from __future__ import annotations import importlib +import pkgutil import sys from typing import Any @@ -36,6 +37,17 @@ def _resolve_env(name: str) -> Any: raise ValueError(f"unknown env: {env_name!r}") from e +def enumerate_envs() -> tuple[str, ...]: + """Return the names of importable environment packages under ``robots``.""" + import robots + + return tuple(sorted( + module.name + for module in pkgutil.iter_modules(robots.__path__) + if module.ispkg and not module.name.startswith("_") + )) + + def get_env_spec(name: str) -> EnvSpec: return _resolve_env(name).get_env_spec() From 74e972a62839eb487fadca8c80fa2542bc8cf7cb Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 03:29:22 +0000 Subject: [PATCH 13/25] chore: inline ToolResult construction Signed-off-by: Jiaxing Qiu --- rpent/tools/toolkit.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 76882f81..6aacd4b7 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -201,14 +201,14 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: """Dispatch a tool call to its registered handler.""" entry = self._tools.get(name) if entry is None: - return self._finish_tool(name, {"error": f"unknown tool: {name}"}) + return ToolResult(name=name, result={"error": f"unknown tool: {name}"}) _, handler, captures_state = entry with self._operation_lock: if self._active_operation is not None: - return self._finish_tool( - name, - {"error": "another tool operation is still active"}, + return ToolResult( + name=name, + result={"error": "another tool operation is still active"}, ) operation = _ToolOperation() self._active_operation = operation @@ -219,9 +219,12 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: try: result = handler(**input_dict) except TypeError as e: - return self._finish_tool( - name, - {"error": f"bad arguments for {name}: {e}", "got": input_dict}, + return ToolResult( + name=name, + result={ + "error": f"bad arguments for {name}: {e}", + "got": input_dict, + }, ) except ToolCancelled as e: result = { @@ -262,7 +265,7 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: if record is not None: self._publish_step(record) - return self._finish_tool(name, result) + return ToolResult(name=name, result=result) finally: with self._operation_lock: self._active_operation = None @@ -278,9 +281,6 @@ def _publish_step(self, record: StepRecord) -> None: ) ) - def _finish_tool(self, name: str, result: Any) -> ToolResult: - return ToolResult(name=name, result=result) - def get_env_state( self, *, From 3cd2d7887c9e5536b274a95894ebb31fbcf9b3e2 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 03:48:53 +0000 Subject: [PATCH 14/25] chore: remove manifest version Signed-off-by: Jiaxing Qiu --- rpent/tools/state.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rpent/tools/state.py b/rpent/tools/state.py index a0d4baad..c50842f5 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -19,7 +19,6 @@ logger = get_logger("env_state") _MANIFEST_NAME = "states.json" -_MANIFEST_VERSION = 2 _IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg"} _TEXT_SUFFIXES = {".txt", ".md"} _SUPPORTED_SUFFIXES = _IMAGE_SUFFIXES | { @@ -138,7 +137,6 @@ def _write_manifest(self) -> None: destination = self._manifest_file() temporary = self._temporary_file(destination) manifest = { - "version": _MANIFEST_VERSION, "run_artifacts": sorted(self._run_artifacts), "steps": [record.to_blob() for record in self._steps], } From 0ab14f1274d190c58a802780ece063ff7960feb2 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 03:54:38 +0000 Subject: [PATCH 15/25] fix: @readonly instead of @updatestate Signed-off-by: Jiaxing Qiu --- .../rst_source/development/add_primitive.rst | 14 +++--- .../rst_source/development/add_primitive.rst | 12 ++--- robots/libero/toolkit.py | 8 ++-- robots/libero/tools.py | 21 ++++----- rpent/tools/common.py | 5 +++ rpent/tools/toolkit.py | 44 +++++++------------ 6 files changed, 42 insertions(+), 62 deletions(-) diff --git a/docs/source-en/rst_source/development/add_primitive.rst b/docs/source-en/rst_source/development/add_primitive.rst index a48f5fba..bb52a8b2 100644 --- a/docs/source-en/rst_source/development/add_primitive.rst +++ b/docs/source-en/rst_source/development/add_primitive.rst @@ -43,24 +43,20 @@ Adding a scripted primitive usually involves two steps: the tool-call arguments, performs the work, usually through one or more ``self._env.step(...)`` calls, and returns a small log ``dict``. - You should mark all methods that change environment state with - :func:`~rpent.tools.toolkit.updatestate`, so the toolkit re-renders - state (``get_env_state``) automatically after it runs: + Primitive methods capture and re-render state (``get_env_state``) + automatically after they run: .. code-block:: python - from rpent.tools.toolkit import updatestate - - @updatestate def open_drawer(self, dx: float = 0.15) -> dict: # Move end-effector back by dx while gripper is closed. for _ in range(N): self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} - Failing to do so will result in the next tool call seeing stale state information. - Read-only tools (``view_env_state``, ``back_project``, ``segment``, - ...) can be left unmarked -- the toolkit skips state capture for them. + You can mark read-only tools (``view_env_state``, ``back_project``, ``segment``, + ...) with :func:`~rpent.tools.toolkit.readonly` so the toolkit skips state + capture for them, improving performance. 2. **Add the tool schema.** Add an entry to ``TOOLS_SPEC`` in ``robots//tools.py``: diff --git a/docs/source-zh/rst_source/development/add_primitive.rst b/docs/source-zh/rst_source/development/add_primitive.rst index 2effeb52..61288934 100644 --- a/docs/source-zh/rst_source/development/add_primitive.rst +++ b/docs/source-zh/rst_source/development/add_primitive.rst @@ -40,24 +40,20 @@ primitives 方法，以及调用完成后的状态快照。区别仅在于方法 一个方法。该方法接收工具调用的参数，执行一次或多次 ``self._env.step(...)``，并返回一个简短的日志字典。 - 您需要为所有可能改变环境状态的方法加上 - :func:`~rpent.tools.toolkit.updatestate` 装饰器， - toolkit 会在其执行后自动重新渲染状态（``get_env_state``）： + primitive 方法执行后默认会自动捕获并重新渲染状态 + （``get_env_state``）： .. code-block:: python - from rpent.tools.toolkit import updatestate - - @updatestate def open_drawer(self, dx: float = 0.15) -> dict: # 保持夹爪闭合，沿 -x 方向后拉 dx 米。 for _ in range(N): self._env.step(build_open_drawer_chunk(dx)) return {"ok": True, "dx": dx} - 若不这样做，之后的工具调用可能将看到过时的环境状态信息。 只读工具（``view_env_state``、``back_project``、``segment`` 等） - 无需装饰——toolkit 会跳过它们的状态捕获。 + 可以使用 :func:`~rpent.tools.toolkit.readonly` 标记，toolkit 会跳过 + 它们的状态捕获，提升性能。 2. **添加工具定义。** 在 ``robots//tools.py`` 的 ``TOOLS_SPEC`` 中新增一项： diff --git a/robots/libero/toolkit.py b/robots/libero/toolkit.py index 23a315cb..a9e22dc2 100644 --- a/robots/libero/toolkit.py +++ b/robots/libero/toolkit.py @@ -38,11 +38,9 @@ def __init__( # Registration # ------------------------------------------------------------------ def _register_libero_tools(self) -> None: - # Read-only tools whose handlers aren't primitive methods (they need - # the run's EnvState bound in, or -- like segment -- must stay - # read-only despite being a primitives method). Every other spec binds - # to its primitive-driver method; @updatestate on the method decides - # whether state is captured. + # These read-only handlers need the run's EnvState bound in. Every + # other spec binds to a primitive-driver method and captures state by + # default unless that method is explicitly marked @readonly. state_handlers = { "view_env_state": partial( libero_tools.view_env_state, state=self._state diff --git a/robots/libero/tools.py b/robots/libero/tools.py index 08baff58..254c9342 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -8,7 +8,7 @@ from robots.libero.env_client import LiberoEnvClient from rpent.tools.state import EnvState, StepRecord -from rpent.tools.toolkit import updatestate +from rpent.tools.toolkit import readonly from rpent.utils.logging import get_logger from rpent.utils.sam3_client import Sam3Client from rpent.utils.vla_client import VLAClient @@ -125,7 +125,6 @@ def _vlm_chunk(self, instruction: str): if original_task is not None: self._last_obs["task_descriptions"] = original_task - @updatestate def pi0_pick( self, prompt: str, @@ -201,7 +200,6 @@ def pi0_pick( }, } - @updatestate def pi0_doubled( self, prompt: str, @@ -243,7 +241,6 @@ def pi0_doubled( }, } - @updatestate def move_to( self, xyz, @@ -308,7 +305,6 @@ def move_to( "libero_terminated": self.env.episode_terminated, } - @updatestate def rotate_wrist( self, *, @@ -382,7 +378,6 @@ def _yaw_of(quat_xyzw): "libero_terminated": self.env.episode_terminated, } - @updatestate def rotate_pitch( self, *, @@ -464,7 +459,6 @@ def _pitch_of(quat_xyzw): "libero_terminated": self.env.episode_terminated, } - @updatestate def move_pose( self, xyz, @@ -535,7 +529,6 @@ def _yaw_of(q): "libero_terminated": self.env.episode_terminated, } - @updatestate def release( self, *, @@ -564,7 +557,6 @@ def release( "libero_terminated": self.env.episode_terminated, } - @updatestate def set_gripper( self, *, @@ -589,6 +581,7 @@ def set_gripper( # ---- introspection helpers (for LLM-in-the-loop) ---- + @readonly def segment( self, prompt: str = "", @@ -745,14 +738,13 @@ def segment( def _is_primitive_action(name: object) -> bool: """Whether ``name`` is a state-advancing LIBERO primitive. - A primitive is any ``@updatestate``-marked method on - :class:`LiberoPrimitives`; read-only tools (``view_env_state``, - ``back_project``, ``segment``, ...) and non-strings read as ``False``. + A primitive is any non-read-only method on :class:`LiberoPrimitives`; + read-only tools and non-strings read as ``False``. """ if not isinstance(name, str): return False method = getattr(LiberoPrimitives, name, None) - return method is not None and bool(getattr(method, "_updates_state", False)) + return method is not None and not bool(getattr(method, "_readonly", False)) def write_recipe_from_states(state: EnvState, recipe_tag: str) -> str: @@ -1409,6 +1401,7 @@ def _save_observation_artifacts( ] +@readonly def view_env_state(step: int = -1, *, state: EnvState) -> dict: try: record = state.get(step) @@ -1543,6 +1536,7 @@ def _make_segment_overlay( return overlay +@readonly def view_camera_meta( camera: str = "agentview", step: int = -1, @@ -1567,6 +1561,7 @@ def view_camera_meta( return {"camera": "wrist", "step": record.step_idx, "camera_meta": meta} +@readonly def back_project( row: int | None = None, col: int | None = None, diff --git a/rpent/tools/common.py b/rpent/tools/common.py index fd959566..be790214 100644 --- a/rpent/tools/common.py +++ b/rpent/tools/common.py @@ -4,6 +4,7 @@ import os from pathlib import Path +from rpent.tools.toolkit import readonly from rpent.utils.config import get_repo_root from rpent.utils.logging import get_output_dir @@ -93,6 +94,7 @@ def _truncate(text: str, max_chars: int) -> str: ) +@readonly def read_text_file(path: str, max_chars: int = 40000) -> dict: p = _resolve(path) if not p.exists(): @@ -106,6 +108,7 @@ def read_text_file(path: str, max_chars: int = 40000) -> dict: return {"path": str(p), "size": len(text), "content": _truncate(text, max_chars)} +@readonly def write_text_file(path: str, content: str) -> dict: p = _resolve(path) p.parent.mkdir(parents=True, exist_ok=True) @@ -113,6 +116,7 @@ def write_text_file(path: str, content: str) -> dict: return {"path": str(p), "bytes_written": len(content.encode("utf-8"))} +@readonly def list_dir(path: str = "") -> dict: # Default to the current output dir (so parallel agents see their own). p = _resolve(path) if path else get_output_dir() @@ -122,6 +126,7 @@ def list_dir(path: str = "") -> dict: return {"path": str(p), "count": len(files), "files": files} +@readonly def finish(status: str, summary: str) -> dict: """Signal that the run is complete. Halts the agent loop. diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 6aacd4b7..2dc14cb8 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -13,6 +13,7 @@ import traceback from collections.abc import Callable from dataclasses import dataclass, field +from functools import partial from typing import TYPE_CHECKING, Any, ClassVar from rpent.dashboard.events import DashboardEventSink, StepRecordEvent @@ -32,35 +33,24 @@ class ToolCancelled(Exception): """Raised when an environment reaches a safe cancellation boundary.""" -def updatestate(func): - """Mark a tool handler as one that advances environment state. +def readonly(func): + """Mark a tool handler as not advancing environment state. - The toolkit captures a fresh observation (:meth:`Toolkit.get_env_state`) - after a handler carrying this marker runs (or raises). Apply it to - primitive-driver methods that move the robot; read-only tools (file IO, - ``view_env_state``, ``back_project``, ``segment``, ...) are left - unmarked and skip state capture. - - The marker is read off the underlying function, so registering a bound - method (``getattr(self._driver, name)``) inherits it automatically. - ``functools.partial`` does not delegate attribute access, so a partial - wrapping a marked method is treated as read-only (intentional -- see the - LIBERO ``segment`` tool). + Tool handlers capture a fresh observation (:meth:`Toolkit.get_env_state`) + by default. Apply this marker to observational and file/IO tools that do + not move the robot or otherwise change the environment. """ - func._updates_state = True + func._readonly = True return func -def _updates_state(handler: Callable[..., Any]) -> bool: - """Whether ``handler`` was marked with :func:`updatestate`. - - Resolves through ``__func__`` so bound methods (the common case for - primitive-driver tools) report the marker set on their underlying - function. Plain functions, closures, and ``functools.partial`` objects - do not delegate, so undecorated read-only handlers read as ``False``. - """ - target = getattr(handler, "__func__", handler) - return bool(getattr(target, "_updates_state", False)) +def _is_readonly(handler: Callable[..., Any]) -> bool: + """Whether ``handler`` was marked with :func:`readonly`.""" + target = handler + while isinstance(target, partial): + target = target.func + target = getattr(target, "__func__", target) + return bool(getattr(target, "_readonly", False)) @dataclass @@ -174,10 +164,10 @@ def add_tool( spec: Anthropic-shaped tool schema dict (``name``, ``description``, ``input_schema``). handler: Callable invoked with the tool's input kwargs; returns - a result dict. Decorate state-advancing primitives with - :func:`updatestate`. + a result dict. Decorate read-only handlers with + :func:`readonly`; all other handlers capture state. """ - self._tools[name] = (spec, handler, _updates_state(handler)) + self._tools[name] = (spec, handler, not _is_readonly(handler)) def _register_common_tools(self) -> None: """Register the file/IO tools shared by every run.""" From 5ece7fe9d2a38d73ee51da97b3db8f1a0e2ee4d7 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 03:54:51 +0000 Subject: [PATCH 16/25] fix: remove all non-log artifacts when resetting Signed-off-by: Jiaxing Qiu --- rpent/tools/state.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rpent/tools/state.py b/rpent/tools/state.py index c50842f5..02e44e33 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -150,14 +150,11 @@ def _write_manifest(self) -> None: # -- lifecycle and counters ----------------------------------------- def reset(self) -> None: - """Remove state-owned artifacts and start a fresh trace.""" + """Remove all artifacts and start a fresh trace.""" self._output_dir.mkdir(parents=True, exist_ok=True) - for record in self._steps: - for name in record.artifacts: - self._artifact_file(name, record.step_idx).unlink(missing_ok=True) - for name in self._run_artifacts: - self._artifact_file(name, None).unlink(missing_ok=True) - self._manifest_file().unlink(missing_ok=True) + for path in self._output_dir.iterdir(): + if path.is_file() and path.suffix.lower() != ".log": + path.unlink() self._steps = [] self._run_artifacts = set() self._open_count = 0 From 2e9b8e81e46863481f17546a16af425778c134fc Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 04:00:32 +0000 Subject: [PATCH 17/25] fix: resolve readonly state at tool execution Signed-off-by: Jiaxing Qiu --- rpent/tools/toolkit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 2dc14cb8..4464e140 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -139,7 +139,7 @@ def __init__( ) -> None: self._tools: dict[ str, - tuple[dict[str, Any], Callable[..., Any], bool], + tuple[dict[str, Any], Callable[..., Any]], ] = {} self._dashboard_events = dashboard_events self._state = state @@ -167,7 +167,7 @@ def add_tool( a result dict. Decorate read-only handlers with :func:`readonly`; all other handlers capture state. """ - self._tools[name] = (spec, handler, not _is_readonly(handler)) + self._tools[name] = (spec, handler) def _register_common_tools(self) -> None: """Register the file/IO tools shared by every run.""" @@ -192,7 +192,7 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: entry = self._tools.get(name) if entry is None: return ToolResult(name=name, result={"error": f"unknown tool: {name}"}) - _, handler, captures_state = entry + _, handler = entry with self._operation_lock: if self._active_operation is not None: @@ -227,7 +227,7 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: result = {"error": str(e), "traceback": traceback.format_exc()} failed = True - if captures_state: + if not _is_readonly(handler): elapsed_s = round(time.perf_counter() - started, 2) result_dict = result if isinstance(result, dict) else {"value": result} command = {"action": name, **input_dict} From 4404482ea801469f3d0b7ecb8eaf8520a5be7f21 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 06:31:21 +0000 Subject: [PATCH 18/25] chore: change libero_terminated -> terminated; episode_truncated -> truncated Signed-off-by: Jiaxing Qiu --- docs/source-en/rst_source/quickstart.rst | 2 +- docs/source-zh/rst_source/quickstart.rst | 2 +- robots/libero/env_client.py | 17 +++--- robots/libero/guides/pro_hybrid_guide.md | 8 +-- robots/libero/guides/strict_hybrid_guide.md | 24 ++++----- robots/libero/prompts/system.py | 24 ++++----- robots/libero/tools.py | 58 ++++++++++++--------- rpent/dashboard/state.py | 25 ++++++--- rpent/tools/state.py | 12 +++++ 9 files changed, 101 insertions(+), 71 deletions(-) diff --git a/docs/source-en/rst_source/quickstart.rst b/docs/source-en/rst_source/quickstart.rst index ac49ab68..574c29f5 100644 --- a/docs/source-en/rst_source/quickstart.rst +++ b/docs/source-en/rst_source/quickstart.rst @@ -106,7 +106,7 @@ A successful run: 4. By default, artifacts are saved under ``logs/__t_s/``. They include ``transcript_*.json`` (run record), ``states.json`` (the versioned ``EnvState`` manifest), ``recipe_*.jsonl`` (action sequence), and ``episode.mp4`` (episode video). Step artifact files use flat, zero-padded step prefixes with a minimum width of two digits and are managed internally by ``EnvState``. Inspect the final state through the Dashboard or -``view_env_state(step=-1)``. Its top-level ``libero_terminated`` value is the +``view_env_state(step=-1)``. Its top-level ``terminated`` value is the benchmark outcome. ``states.json`` is internal ``EnvState`` storage and should not be parsed by callers. You can also open ``episode.mp4`` to review the run. If something goes wrong, inspect the four log files described at the diff --git a/docs/source-zh/rst_source/quickstart.rst b/docs/source-zh/rst_source/quickstart.rst index 2b153eb0..c8c358c6 100644 --- a/docs/source-zh/rst_source/quickstart.rst +++ b/docs/source-zh/rst_source/quickstart.rst @@ -98,6 +98,6 @@ LIBERO-PRO 仿真资源。下面以 LIBERO-PRO 和 ``claude_code`` planner 4. 默认输出目录为 ``logs/__t_s/``，其中包含 ``transcript_*.json``\ （运行记录）、``states.json``\ （带版本号的 ``EnvState`` 清单）、``recipe_*.jsonl``\ （动作序列）和 ``episode.mp4``\ （回合录像）。每步工件文件采用至少两位、零填充的步骤前缀扁平命名，并由 ``EnvState`` 在内部管理。 通过 Dashboard 或 ``view_env_state(step=-1)`` 查看最终状态；其顶层 -``libero_terminated`` 即为基准任务结果。``states.json`` 是 ``EnvState`` 的内部 +``terminated`` 即为基准任务结果。``states.json`` 是 ``EnvState`` 的内部 存储，调用方不应直接解析。也可以打开 ``episode.mp4`` 复核运行过程。 出问题时，参考 :doc:`installation` 页底部提到的四份日志文件。 diff --git a/robots/libero/env_client.py b/robots/libero/env_client.py index 5c33776a..632c19ac 100644 --- a/robots/libero/env_client.py +++ b/robots/libero/env_client.py @@ -13,7 +13,6 @@ from rpent.utils.rpc import RpcClient - _TIMEOUT_S = { "default": 30.0, "env.reset": 120.0, @@ -35,8 +34,8 @@ def __init__( ): self._client = client self.return_all_frames = return_all_frames - self.episode_terminated = False - self.episode_truncated = False + self.terminated = False + self.truncated = False server_meta = self._client.call( "env.get_env_meta", timeout_s=_TIMEOUT_S["default"] ) @@ -49,17 +48,17 @@ def __init__( self.reset() def check_done(self, term, trunc) -> None: - self.episode_terminated |= bool(np.asarray(term).any()) - self.episode_truncated |= bool(np.asarray(trunc).any()) + self.terminated |= bool(np.asarray(term).any()) + self.truncated |= bool(np.asarray(trunc).any()) def reset(self) -> tuple[dict, Any]: ret = self._client.call("env.reset", timeout_s=_TIMEOUT_S["env.reset"]) - self.episode_terminated = False - self.episode_truncated = False + self.terminated = False + self.truncated = False return ret def step(self, action) -> tuple[dict, Any, np.ndarray, Any, Any]: - assert not (self.episode_terminated or self.episode_truncated), ( + assert not (self.terminated or self.truncated), ( "env.step called after the episode signaled term/trunc" ) ret = self._client.call( @@ -78,7 +77,7 @@ def chunk_step(self, actions, *, return_all_frames: bool | None = None) -> tuple Terminated / truncated have shape ``[chunk_size]`` after the server strips the env dim. """ - assert not (self.episode_terminated or self.episode_truncated), ( + assert not (self.terminated or self.truncated), ( "env.chunk_step called after the episode signaled term/trunc" ) if return_all_frames is None: diff --git a/robots/libero/guides/pro_hybrid_guide.md b/robots/libero/guides/pro_hybrid_guide.md index 13e36228..500ff22f 100644 --- a/robots/libero/guides/pro_hybrid_guide.md +++ b/robots/libero/guides/pro_hybrid_guide.md @@ -368,7 +368,7 @@ Pi0 never sees object coords in either mode, so there is no perception variant o the baseline. Expected behavior: - **P1 (task):** Pi0 "succeeds at the wrong task" — it picks the *base*-task - target object, places it on the plate, and `libero_terminated=False` because the + target object, places it on the plate, and `terminated=False` because the goal predicate names a different object. This is exactly the gap the hybrid closes. - **P2 (position):** Pi0 picks / places at the *base* (un-swapped) location. @@ -391,14 +391,14 @@ add the PRO fields: "strategy_notes": "HOW you localized — which pixel(s) in agentview_high.png, back-projected world xyz; for swap, how you found the relocated object/fixture", "pick_result": { /* the pi0_pick step's result */ }, "final_state": { /* latest view_env_state result's `state` field */ }, - "libero_terminated": true + "terminated": true } ``` `strategy_notes` **must** describe the localization (pixel → `back_project` → world xyz). For swap cells, explicitly note that the relocated object/fixture was found by perception, not by reading coords. If unrecoverable after honest -exploration, write `libero_terminated: false` with what you tried and which step +exploration, write `terminated: false` with what you tried and which step failed — never warp (teleport primitives are deleted; see Rule 4 in the perception protocol). Write the audit with `write_text_file` to `{output_dir}/{recipe_tag}.json`, then call `finish`. @@ -421,7 +421,7 @@ perception protocol). Write the audit with `write_text_file` to `finish`. `_swap` typically needs an in-episode retry — document it. - **Rule 4 (no teleport).** `set_object_pose`, `articulate_to`, `js_move_to`, `carry_object` are deleted. A goal past OSC reach with no physical approach → - honest `libero_terminated:false`. + honest `terminated:false`. - **Rule 5 (assume solvable).** A localization that moves the gripper into thin air means you picked a wrong pixel (a reflection, a rim, a decoy object under the Object perturbation), not that the cell is unreachable. Re-look, re-pick, diff --git a/robots/libero/guides/strict_hybrid_guide.md b/robots/libero/guides/strict_hybrid_guide.md index 786f85a4..2cc9d5e6 100644 --- a/robots/libero/guides/strict_hybrid_guide.md +++ b/robots/libero/guides/strict_hybrid_guide.md @@ -174,14 +174,14 @@ After a pick, decide "did I grab the target?" from two coord-free signals: `pi0_pick`'s returned `success` (an eef-lift + gripper-closure heuristic, pure proprioception) is a HINT, not proof — confirm with the wrist cam before carrying. The only authoritative TASK-success signal is -top-level `libero_terminated` (the benchmark predicate), which is name-independent. +top-level `terminated` (the benchmark predicate), which is name-independent. ## Rule 4 — NO TELEPORT primitives (physics-only) `set_object_pose`, `articulate_to`, `js_move_to`, `carry_object` are **deleted from the codebase**. They are not callable and are not in the tool list. If a goal is past OSC reach and no physical approach works, write an honest -`libero_terminated:false` audit — never warp. +`terminated:false` audit — never warp. ## Rule 0 — Use images for reasoning, not just JSON state @@ -205,7 +205,7 @@ and do NOT restart the episode. You MAY recover *within* this one episode grasp; walk the next rung of the Pi0 prompt ladder in Rule 3; re-firm the grip; `rotate_pitch`/`move_pose`) — that is all one continuous attempt. But the instant you would want to start over, **STOP instead and write the audit** -(success or an honest `libero_terminated:false`), then call `finish`. Never +(success or an honest `terminated:false`), then call `finish`. Never warp; never reset. ## Rule 5 — Assume every task is physically solvable @@ -273,7 +273,7 @@ sim) — do not start/stop it. You call MCP tools; begin by reading step 0 via | logical artifact or field | what's in it | |---|---| -| `view_env_state` result | `step`, top-level `task_language`, `libero_terminated`, `episode_truncated`, `state.{robot0_eef_pos, robot0_eef_quat, robot0_gripper_qpos, object_names}`, `artifacts`, and nested `log.{command,result,elapsed_s}`. **No object coordinates.** Step `-1` selects latest. | +| `view_env_state` result | `step`, top-level `task_language`, `terminated`, `truncated`, `state.{robot0_eef_pos, robot0_eef_quat, robot0_gripper_qpos, object_names}`, `artifacts`, and nested `log.{command,result,elapsed_s}`. **No object coordinates.** Step `-1` selects latest. | | `agentview_policy.png` | RGB in Pi0 frame. *Do not pick pixels here for back-projection.* | | `agentview.png` | 256×256 agentview RGB in **calibration frame**. Pick object pixels HERE only with `back_project(..., resolution:"low")`. | | `agentview_high.png` | **HI-RES 1024×1024** agentview RGB, calibration frame. **PREFER this for looking / identification**; `back_project` defaults to `resolution:"high"`. | @@ -341,7 +341,7 @@ pi0_pick({"prompt": "pick up the X", "max_chunks": 20, "lift_thresh": 0.05, "gripper_closed_thresh": 0.06}) // Pi0.5 for a contact skill (knob turn, drawer/door open-close — rare here). -// success mirrors libero_terminated only; inspect image/state for intermediates. +// success mirrors terminated only; inspect image/state for intermediates. pi0_doubled({"prompt": "turn off the stove", "max_chunks": 20}) // Open gripper to place. Triggers libero termination if the On/In predicate met. @@ -442,7 +442,7 @@ A typical bowl→plate cell looks like: 7. Localize the placement region (basket / plate / drawer slot) the same way. 8. `move_to([place_x, place_y, carry_z])` to traverse at constant height. 9. Optionally descend `move_to([place_x, place_y, place_z])`. -10. `release` — predicate (`On`/`In`) checks → `libero_terminated=True` if hit. +10. `release` — predicate (`On`/`In`) checks → `terminated=True` if hit. 11. Light retreat (`move_to` upward) so the next step's image is clean. > **Predicate fire timing.** Most LIBERO `On(X, Y)` predicates fire on @@ -491,7 +491,7 @@ you do **not** need a separate read. When you need an older step, call 2. Inspect the embedded `agentview_high.png` image → visual confirmation; pick pixels for any new localization. 3. Check `state.robot0_eef_pos`, `state.robot0_gripper_qpos`, and top-level - `libero_terminated` in the returned view. + `terminated` in the returned view. You **do not** open the depth maps yourself — feed a pixel to `back_project`. Don't call `view_env_state` immediately after a primitive that already @@ -504,7 +504,7 @@ returned the new state. lower or shifted; retry Pi0 with next prompt-ladder rung. - **Object slipped mid-carry.** `release` returns top-level - `libero_terminated=false` and the object + `terminated=false` and the object is no longer where you `release`d it. `release`, re-pre-pos above it, `pi0_pick` again, traverse again. - **OSC stuck.** `move_to` returns `log.result.final_dist_m > 0.05` at @@ -530,7 +530,7 @@ allowed physics primitives (`move_to`, `pi0_pick`, `pi0_doubled`, `release`, ## Persisting successful runs as audit JSONs -When top-level `libero_terminated == true`: +When top-level `terminated == true`: a. The working command recipe (`{output_dir}/recipe_{recipe_tag}.jsonl`) is **auto-exported by the runner** from non-error primitive commands in the @@ -542,11 +542,11 @@ b. Write a minimal audit JSON with `write_text_file` to `regime: "strict_perception"`, `strategy_notes` (mention HOW you localized — which pixel, depth, back-projected world xyz), `pick_result` (the `result` from your `pi0_pick`), `final_state` (the latest returned `state` field), - `libero_terminated: true`. + `terminated: true`. c. Call `finish({"status":"success","summary":"…"})`. If unrecoverable after honest exploration in this one episode, write -`{output_dir}/{recipe_tag}.json` with `libero_terminated: false` + +`{output_dir}/{recipe_tag}.json` with `terminated: false` + `strategy_notes` describing what you tried, the back-projected xyz you used, and which step failed. Then call `finish` (NO reset, NO second attempt). @@ -580,7 +580,7 @@ which step failed. Then call `finish` (NO reset, NO second attempt). - **Fully oracle-free, including the grasp.** There is no GT-lift oracle — `pi0_pick` reads NO GT object pose and takes no tracking argument. You judge the grasp from gripper width + the wrist cam, and TASK success from - top-level `libero_terminated` (the benchmark predicate). + top-level `terminated` (the benchmark predicate). - **Single attempt.** One episode, no reset (Rule 2). - The expected audit `regime` is `strict_perception`. diff --git a/robots/libero/prompts/system.py b/robots/libero/prompts/system.py index f0eba22d..657c4bfc 100644 --- a/robots/libero/prompts/system.py +++ b/robots/libero/prompts/system.py @@ -11,12 +11,12 @@ > ONE-SHOT evaluation: you get **exactly ONE episode**. You MUST NOT call > `reset`, and you must not restart the episode. Plan carefully, then execute > your single best manipulation sequence toward top-level -> `libero_terminated == true`. +> `terminated == true`. > You MAY recover *within* this one episode (re-pre-position, re-`pi0_pick` a > missed grasp, walk the Pi0 prompt ladder, `rotate_pitch`/`move_pose`) — that is > all one continuous attempt — but the instant you would want to reset/start over, > **STOP instead and write the audit** (success or honest -> `libero_terminated:false`). Do NOT call `reset`. Use the PROVEN LEVERS below to +> `terminated:false`). Do NOT call `reset`. Use the PROVEN LEVERS below to > get the single attempt right the first time.""" PROVEN_LEVERS = """These are battle-tested on seed 0 of THIS suite. You are now running a DIFFERENT @@ -55,7 +55,7 @@ a HIGH `lift_thresh` (e.g. 999) + `gripper_closed_thresh:0` turns it into a generic closed-loop CONTACT skill (used to turn the stove knob). - **`pi0_doubled`** = Pi0 closed-loop CONTACT skill (success := - `libero_terminated`). Use it for drawer/door open-close AND insertions; call it + `terminated`). Use it for drawer/door open-close AND insertions; call it repeatedly. DISAMBIGUATION / TARGETING: @@ -131,7 +131,7 @@ - mug→microwave + close (t9): the only UNSOLVED seed-0 cell — the round mug-in-hand walls ~3cm short of the In() threshold (deep narrow cavity). Try every lever (`pi0_doubled`, `move_pose`, push) and if it still walls, write an honest - `libero_terminated:false` with the max eef-y reached.""" + `terminated:false` with the max eef-y reached.""" RUNTIME = """A server process (`env_server.py`) is already running. It has Pi0.5 loaded and a single-env LIBERO sim. The runner manages the server and exposes structured @@ -148,7 +148,7 @@ name shown in your tool list, preserving the same arguments and semantics. Each state record exposes `step`, top-level `task_language`, -`libero_terminated`, `episode_truncated`, coord-free `state`, an `artifacts` +`terminated`, `truncated`, coord-free `state`, an `artifacts` list of logical base names, and a `log`. Storage paths are internal; do not construct or parse them. @@ -164,7 +164,7 @@ metadata and world maps without opening artifacts directly. Step `0` is the initial state; step `-1` selects the latest state.""" -GOAL = """YOUR GOAL: produce top-level `libero_terminated == true` in ONE episode. ⛔ NO +GOAL = """YOUR GOAL: produce top-level `terminated == true` in ONE episode. ⛔ NO `reset`, NO retry (SINGLE-ATTEMPT MODE — see the override at the very top; it supersedes any reset/retry wording in the Rules below).""" @@ -260,7 +260,7 @@ `rotate_pitch`/`move_pose`) — that is still one continuous attempt — but you may NOT restart the episode. When the task terminates, OR when your single best sequence is exhausted (you'd otherwise want to reset), STOP and write the audit - (success or honest `libero_terminated:false`), then call `finish`. + (success or honest `terminated:false`), then call `finish`. NO teleport primitives (set_object_pose / articulate_to / js_move_to / carry_object — deleted/forbidden; a goal past OSC reach is approached physically or honestly reported, never warped). NO object world coords are @@ -439,7 +439,7 @@ ⚠ INFRA NOTE: `pi0_doubled` IS implemented and callable in this runtime — verified. It runs the Pi0 VLA on a CONTACT skill (drawer/door open-close, knob -turn) with success := `libero_terminated` (no lift / no gripper-close +turn) with success := `terminated` (no lift / no gripper-close assumption — unlike `pi0_pick`). If ANY prior note or reference for this cell concluded that `pi0_doubled` is "unknown action" / missing / that drawer-or-door articulation is an unsolvable "structural dead-end" BECAUSE no @@ -473,16 +473,16 @@ traversals into <0.30 xy waypoints; for a door/drawer/knob use a SHORT capped OSC push or `pi0_doubled`, never one long push — it NaNs MuJoCo. If the task is unrecoverable within this one episode, do NOT reset — write an honest -stuck-audit (`libero_terminated:false`) and call `finish`. Never warp. +stuck-audit (`terminated:false`) and call `finish`. Never warp. """, - """WHEN top-level `libero_terminated == true` in the latest tool result: + """WHEN top-level `terminated == true` in the latest tool result: a. Write audit `{{output_dir}}/{{recipe_tag}}.json` with: suite, task_id, seed, regime:"strict_perception", strategy_notes (incl. how you localized), pick_result, final_state (latest state's `state`), - libero_terminated:true. + terminated:true. b. Call `finish`. If your single attempt does not solve it, write `{{output_dir}}/{{recipe_tag}}.json` with -libero_terminated:false + strategy_notes describing what you tried in this one +terminated:false + strategy_notes describing what you tried in this one episode and where it stalled. Then call `finish`. (NO reset, NO second attempt.)""", ) diff --git a/robots/libero/tools.py b/robots/libero/tools.py index 254c9342..921b9ab9 100644 --- a/robots/libero/tools.py +++ b/robots/libero/tools.py @@ -173,8 +173,8 @@ def pi0_pick( if descent_done and ascended and closed: success = True break - if self.env.episode_terminated or self.env.episode_truncated: - success = self.env.episode_terminated + if self.env.terminated or self.env.truncated: + success = self.env.terminated break return { @@ -186,7 +186,8 @@ def pi0_pick( "peak_lift_m": post_min_peak_z - min_z, # actual post-descent ascent "min_gripper_opening": min_grip, "final_gripper_opening": last_grip, - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, "diagnostics": { "start_eef_z": round(start_z, 4), "peak_eef_z": round(peak_z, 4), @@ -219,8 +220,8 @@ def pi0_doubled( for c in range(max_chunks): self._vlm_chunk(instr) chunks_used = c + 1 - if self.env.episode_terminated or self.env.episode_truncated: - task_success = self.env.episode_terminated + if self.env.terminated or self.env.truncated: + task_success = self.env.terminated break return { @@ -231,9 +232,10 @@ def pi0_doubled( "contact_skill_executed": chunks_used > 0, "chunks_used": chunks_used, "max_chunks": max_chunks, - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, "diagnostics": { - "mode": "contact_skill_success_by_libero_terminated", + "mode": "contact_skill_success_by_termination", "success_meaning": ( "`success` mirrors official LIBERO task termination only; " "for intermediate contact skills, inspect image/state evidence." @@ -292,7 +294,7 @@ def move_to( action[5] = float(np.clip(step_dyaw / 0.10, -1.0, 1.0)) action[6] = gripper self._step_env(action) - if self.env.episode_terminated or self.env.episode_truncated: + if self.env.terminated or self.env.truncated: break final = self._last_obs_eef_pos return { @@ -302,7 +304,8 @@ def move_to( "final_dist_m": round(float(np.linalg.norm(target - final)), 4), "steps_used": len(traj), "max_steps": max_steps, - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, } def rotate_wrist( @@ -365,7 +368,7 @@ def _yaw_of(quat_xyzw): action[5] = float(np.clip(action[5], -1.0, 1.0)) action[6] = float(gripper) self._step_env(action) - if self.env.episode_terminated or self.env.episode_truncated: + if self.env.terminated or self.env.truncated: break final_yaw = _yaw_of(self.env.raw_obs()["robot0_eef_quat"]) return { @@ -375,7 +378,8 @@ def _yaw_of(quat_xyzw): "final_yaw": round(final_yaw, 4), "final_err": round(float((target_yaw - final_yaw + np.pi) % (2 * np.pi) - np.pi), 4), "steps_used": len(traj), - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, } def rotate_pitch( @@ -445,7 +449,7 @@ def _pitch_of(quat_xyzw): action[3] = float(np.clip(action[3], -1.0, 1.0)) action[6] = float(gripper) self._step_env(action) - if self.env.episode_terminated or self.env.episode_truncated: + if self.env.terminated or self.env.truncated: break final_pitch = _pitch_of(self.env.raw_obs()["robot0_eef_quat"]) return { @@ -456,7 +460,8 @@ def _pitch_of(quat_xyzw): "final_err": round(float( (target_pitch - final_pitch + np.pi) % (2 * np.pi) - np.pi), 4), "steps_used": len(traj), - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, } def move_pose( @@ -516,7 +521,7 @@ def _yaw_of(q): action[5] = float(np.clip(np.clip(y_err, -yaw_step, yaw_step) / 0.10, -1.0, 1.0)) action[6] = float(gripper) self._step_env(action) - if self.env.episode_terminated or self.env.episode_truncated: + if self.env.terminated or self.env.truncated: break final = self._last_obs_eef_pos fq = self.env.raw_obs()["robot0_eef_quat"] @@ -526,7 +531,8 @@ def _yaw_of(q): "final_dist_m": round(float(np.linalg.norm(target - final)), 4), "final_pitch": round(_pitch_of(fq), 4), "steps_used": step + 1, - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, } def release( @@ -546,7 +552,7 @@ def release( action[6] = -1.0 # open self._step_env(action) peak_grip = max(peak_grip, self._last_obs_gripper) - if self.env.episode_terminated or self.env.episode_truncated: + if self.env.terminated or self.env.truncated: break return { "name": "release", @@ -554,7 +560,8 @@ def release( "start_gripper_opening": round(start_grip, 4), "peak_gripper_opening": round(peak_grip, 4), "final_gripper_opening": round(self._last_obs_gripper, 4), - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, } def set_gripper( @@ -570,13 +577,14 @@ def set_gripper( action = np.zeros(7, dtype=np.float32) action[6] = g self._step_env(action) - if self.env.episode_terminated or self.env.episode_truncated: + if self.env.terminated or self.env.truncated: break return { "name": "set_gripper", "gripper": g, "steps": n, - "libero_terminated": self.env.episode_terminated, + "terminated": self.env.terminated, + "truncated": self.env.truncated, } # ---- introspection helpers (for LLM-in-the-loop) ---- @@ -748,7 +756,7 @@ def _is_primitive_action(name: object) -> bool: def write_recipe_from_states(state: EnvState, recipe_tag: str) -> str: - """Find a command sequence that gets ``libero_terminated=True``. + """Find a command sequence that gets ``terminated=True``. Export non-error LIBERO primitive commands and successful segment calls. """ @@ -840,12 +848,12 @@ def dump_state( log = log or {} with env_state.record_step( state=state, + terminated=primitives.env.terminated, + truncated=primitives.env.truncated, command=log.get("command"), result=log.get("result"), elapsed_s=log.get("elapsed_s"), extras={ - "terminated": primitives.env.episode_terminated, - "episode_truncated": primitives.env.episode_truncated, "task_language": primitives.env.get_task_language(), }, ) as step_idx: @@ -1140,7 +1148,7 @@ def _save_observation_artifacts( "description": ( "Pi0.5 closed-loop contact skill for non-pick interactions " "(e.g. stove/knob/button/short push). Returned success/task_success " - "only mirrors official libero_terminated; for intermediate contact " + "only mirrors official termination; for intermediate contact " "skills, success=false does not necessarily mean the contact " "interaction failed. Inspect image/state evidence. Do not use it " "as a general pick/place shortcut." @@ -1412,12 +1420,12 @@ def view_env_state(step: int = -1, *, state: EnvState) -> dict: extras = record.extras out: dict = { "step": nn, + "terminated": record.terminated, + "truncated": record.truncated, "state": record.state, "artifacts": sorted(record.artifacts), } out["task_language"] = extras.get("task_language") - out["libero_terminated"] = extras.get("terminated") - out["episode_truncated"] = extras.get("episode_truncated") out["log"] = { "command": record.command, "result": record.result, diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py index 0a339ef6..32cebabf 100644 --- a/rpent/dashboard/state.py +++ b/rpent/dashboard/state.py @@ -115,6 +115,7 @@ def __init__( self._condition = threading.Condition(self._lock) self._task_state: str | None = None self._terminated = False + self._truncated = False self._error: str | None = None self._usage = {"in": 0, "out": 0, "tool_calls": 0} self._runtime = { @@ -280,7 +281,12 @@ def complete_task( raise ValueError(f"invalid terminal run state: {state!r}") with self._condition: self._task_state = state - self._terminated = any(item.get("terminated") for item in self._timeline) + self._terminated = any( + item.get("terminated") for item in self._timeline + ) + self._truncated = any( + item.get("truncated") for item in self._timeline + ) self._error = None if error is None else str(error) self._task_replacement_requested = False self._seal_interaction_locked() @@ -306,6 +312,7 @@ def _begin_task_locked( self._task_state = "starting" self._task_replacement_requested = False self._terminated = False + self._truncated = False self._error = None self._usage = {"in": 0, "out": 0, "tool_calls": 0} self._reset_task_runtime_locked() @@ -604,9 +611,8 @@ def _apply_tool_result(self, event: ToolResultEvent) -> None: except Exception: return action = str(command.get("action", name)) - terminated = bool( - result.get("terminated", result.get("libero_terminated", False)) - ) + terminated = bool(result.get("terminated")) + truncated = bool(result.get("truncated")) action_video_path = self._action_video_from_result( result, step=step, @@ -621,6 +627,7 @@ def _apply_tool_result(self, event: ToolResultEvent) -> None: "result": log.get("result"), "elapsed_s": log.get("elapsed_s"), "terminated": terminated, + "truncated": truncated, "action_video_path": action_video, "action_video_artifact": action_video_artifact, "has_action_video": bool(action_video_artifact or action_video_path), @@ -628,6 +635,7 @@ def _apply_tool_result(self, event: ToolResultEvent) -> None: with self._lock: self._timeline.append(item) self._terminated = self._terminated or terminated + self._truncated = self._truncated or truncated def _action_video_from_result( self, @@ -651,7 +659,6 @@ def on_step(self, record: StepRecord) -> None: command = record.command if not isinstance(command, dict) or not command.get("action"): return - terminated = bool(record.extras.get("terminated")) action_video = next( (name for name in sorted(record.artifacts) if name.endswith(".mp4")), None, @@ -662,13 +669,15 @@ def on_step(self, record: StepRecord) -> None: "args": {key: value for key, value in command.items() if key != "action"}, "result": record.result, "elapsed_s": record.elapsed_s, - "terminated": terminated, + "terminated": record.terminated, + "truncated": record.truncated, "action_video_artifact": action_video, "has_action_video": action_video is not None, } with self._lock: self._timeline.append(item) - self._terminated = self._terminated or terminated + self._terminated = self._terminated or record.terminated + self._truncated = self._truncated or record.truncated def _update_step_frames(self, record: StepRecord) -> None: """Load dashboard frame bytes from the step's canonical artifacts.""" @@ -874,6 +883,7 @@ def snapshot(self) -> dict[str, Any]: return { "state": self._visible_state_locked(), "terminated": self._terminated, + "truncated": self._truncated, "error": self._error, "usage": dict(self._usage), "runtime": self._runtime_snapshot(), @@ -896,6 +906,7 @@ def run_detail(self) -> dict[str, Any]: return { "state": self._visible_state_locked(), "terminated": self._terminated, + "truncated": self._truncated, "error": self._error, "usage": dict(self._usage), "runtime": self._runtime_snapshot(), diff --git a/rpent/tools/state.py b/rpent/tools/state.py index 02e44e33..0acf8f95 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -46,6 +46,8 @@ class StepRecord: step_idx: int state: dict[str, Any] + terminated: bool = False + truncated: bool = False artifacts: set[str] = field(default_factory=set) command: dict | None = None result: dict | None = None @@ -56,6 +58,8 @@ def to_blob(self) -> dict[str, Any]: blob: dict[str, Any] = { "step_idx": self.step_idx, "state": self.state, + "terminated": self.terminated, + "truncated": self.truncated, "artifacts": sorted(self.artifacts), } if self.command is not None: @@ -73,6 +77,8 @@ def from_blob(cls, blob: dict[str, Any]) -> "StepRecord": return cls( step_idx=int(blob["step_idx"]), state=dict(blob.get("state") or {}), + terminated=bool(blob["terminated"]), + truncated=bool(blob["truncated"]), artifacts={str(name) for name in blob.get("artifacts") or []}, command=blob.get("command"), result=blob.get("result"), @@ -350,6 +356,8 @@ def record_step( self, *, state: dict[str, Any], + terminated: bool = False, + truncated: bool = False, command: dict | None = None, result: dict | None = None, elapsed_s: float | None = None, @@ -366,6 +374,8 @@ def record_step( record = StepRecord( step_idx=self._next_step, state=copy.deepcopy(state), + terminated=terminated, + truncated=truncated, command=copy.deepcopy(command), result=copy.deepcopy(result), elapsed_s=elapsed_s, @@ -438,6 +448,8 @@ def view( out: dict[str, Any] = { "step": record.step_idx, + "terminated": record.terminated, + "truncated": record.truncated, "state": record.state, "artifacts": sorted(record.artifacts), "camera_meta": metadata, From 060eb4cdcdfece81ae49c668864f1cb8f388c017 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 08:39:45 +0000 Subject: [PATCH 19/25] fix: add missing libero dep Signed-off-by: Jiaxing Qiu --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0200869a..3b47e229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ openpi = [ ] libero = [ "rlinf-libero", + "h5py", ] libero-pro = [ "rpent[libero]", From 4f6e7b877c7670e23c3d0ceaad86a059344cb7bc Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 08:42:48 +0000 Subject: [PATCH 20/25] fix: update env state even with TypeError Signed-off-by: Jiaxing Qiu --- rpent/tools/toolkit.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 4464e140..fa587314 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -209,13 +209,11 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: try: result = handler(**input_dict) except TypeError as e: - return ToolResult( - name=name, - result={ - "error": f"bad arguments for {name}: {e}", - "got": input_dict, - }, - ) + result = { + "error": f"bad arguments for {name}: {e}", + "got": input_dict, + } + failed = True except ToolCancelled as e: result = { "error": str(e), @@ -249,9 +247,8 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: record = self._state.latest_record() result = captured if failed: - result.setdefault("error", result_dict["error"]) - if "traceback" in result_dict: - result.setdefault("traceback", result_dict["traceback"]) + for key, value in result_dict.items(): + result.setdefault(key, value) if record is not None: self._publish_step(record) From ef9d576133c90e412eba4f92be618f3670f14bbd Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 08:44:24 +0000 Subject: [PATCH 21/25] fix: tuple size mismatch bug Signed-off-by: Jiaxing Qiu --- rpent/tools/toolkit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index fa587314..e7c88a66 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -184,7 +184,7 @@ def _register_common_tools(self) -> None: def get_tools_spec(self) -> list[dict[str, Any]]: """Return the tool schemas the LLM sees.""" return substitute( - [spec for spec, _, _ in self._tools.values()] + [spec for spec, _ in self._tools.values()] ) def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: From e85221f1a194c149770ff6853d3e0b4a28d740b8 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 09:40:42 +0000 Subject: [PATCH 22/25] refactor: separate step artifacts and preserve existing outputs Signed-off-by: Jiaxing Qiu --- docs/source-en/rst_source/quickstart.rst | 2 +- docs/source-zh/rst_source/quickstart.rst | 2 +- rpent/tools/state.py | 17 +++++++++++------ rpent/utils/logging.py | 6 ++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/source-en/rst_source/quickstart.rst b/docs/source-en/rst_source/quickstart.rst index 574c29f5..05d60d5a 100644 --- a/docs/source-en/rst_source/quickstart.rst +++ b/docs/source-en/rst_source/quickstart.rst @@ -103,7 +103,7 @@ A successful run: by the elapsed time, token usage, and path to the run record. 3. With the Dashboard enabled, also streams agent output, camera views, the action timeline, and clip replays to the Dashboard. -4. By default, artifacts are saved under ``logs/__t_s/``. They include ``transcript_*.json`` (run record), ``states.json`` (the versioned ``EnvState`` manifest), ``recipe_*.jsonl`` (action sequence), and ``episode.mp4`` (episode video). Step artifact files use flat, zero-padded step prefixes with a minimum width of two digits and are managed internally by ``EnvState``. +4. By default, artifacts are saved under ``logs/__t_s/``. They include ``transcript_*.json`` (run record), ``states.json`` (the ``EnvState`` manifest), ``recipe_*.jsonl`` (action sequence), and ``episode.mp4`` (episode video). Each step artifact has a directory named after its logical artifact name; zero-padded step files live inside it, for example ``agentview_depth.npy/00.npy`` and ``agentview_depth.npy/01.npy``. Run-level artifacts remain at the output root. Inspect the final state through the Dashboard or ``view_env_state(step=-1)``. Its top-level ``terminated`` value is the diff --git a/docs/source-zh/rst_source/quickstart.rst b/docs/source-zh/rst_source/quickstart.rst index c8c358c6..4d4f14aa 100644 --- a/docs/source-zh/rst_source/quickstart.rst +++ b/docs/source-zh/rst_source/quickstart.rst @@ -95,7 +95,7 @@ LIBERO-PRO 仿真资源。下面以 LIBERO-PRO 和 ``claude_code`` planner 1. 终端会先显示 ``env_server``、``vla_server`` 和 ``sam3_server`` 的启动信息。 2. 智能体的逐轮输出和工具调用会显示在终端中；运行结束时还会显示耗时、token 用量和运行记录的路径。 3. 启用 Dashboard 后，智能体的输出、相机视图、动作时间线和片段回放也会实时显示在 Dashboard 中。 -4. 默认输出目录为 ``logs/__t_s/``，其中包含 ``transcript_*.json``\ （运行记录）、``states.json``\ （带版本号的 ``EnvState`` 清单）、``recipe_*.jsonl``\ （动作序列）和 ``episode.mp4``\ （回合录像）。每步工件文件采用至少两位、零填充的步骤前缀扁平命名，并由 ``EnvState`` 在内部管理。 +4. 默认输出目录为 ``logs/__t_s/``，其中包含 ``transcript_*.json``\ （运行记录）、``states.json``\ （``EnvState`` 清单）、``recipe_*.jsonl``\ （动作序列）和 ``episode.mp4``\ （回合录像）。每种逐步工件使用一个与逻辑工件同名的目录，目录内按步骤保存零填充文件，例如 ``agentview_depth.npy/00.npy`` 和 ``agentview_depth.npy/01.npy``；运行级工件仍保存在输出目录根部。 通过 Dashboard 或 ``view_env_state(step=-1)`` 查看最终状态；其顶层 ``terminated`` 即为基准任务结果。``states.json`` 是 ``EnvState`` 的内部 diff --git a/rpent/tools/state.py b/rpent/tools/state.py index 0acf8f95..7c6a1471 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -120,7 +120,8 @@ def _artifact_file(self, name: str, step: int | None) -> Path: return self._output_dir / name if step < 0: raise ValueError("artifact writes require a nonnegative step") - return self._output_dir / f"{step:02d}_{name}" + artifact = Path(name) + return self._output_dir / name / f"{step:02d}{artifact.suffix}" def _manifest_file(self) -> Path: return self._output_dir / _MANIFEST_NAME @@ -156,11 +157,8 @@ def _write_manifest(self) -> None: # -- lifecycle and counters ----------------------------------------- def reset(self) -> None: - """Remove all artifacts and start a fresh trace.""" + """Reset the in-memory trace without removing on-disk artifacts.""" self._output_dir.mkdir(parents=True, exist_ok=True) - for path in self._output_dir.iterdir(): - if path.is_file() and path.suffix.lower() != ".log": - path.unlink() self._steps = [] self._run_artifacts = set() self._open_count = 0 @@ -211,6 +209,7 @@ def save( """ step = self._resolve_read_step(step) destination = self._artifact_file(name, step) + destination.parent.mkdir(parents=True, exist_ok=True) temporary = self._temporary_file(destination) suffix = destination.suffix.lower() record: StepRecord | None = ( @@ -308,6 +307,11 @@ def remove(self, name: str, *, step: int | None) -> bool: else: self._record_for(step).artifacts.discard(name) destination.unlink() + if step is not None: + try: + destination.parent.rmdir() + except OSError: + pass self._write_manifest() return True @@ -389,7 +393,8 @@ def record_step( yield record.step_idx except BaseException: for name in list(record.artifacts): - self._artifact_file(name, record.step_idx).unlink(missing_ok=True) + destination = self._artifact_file(name, record.step_idx) + destination.unlink(missing_ok=True) if self._steps and self._steps[-1] is record: self._steps.pop() self._next_step = record.step_idx diff --git a/rpent/utils/logging.py b/rpent/utils/logging.py index a0a64f32..94a7c48d 100644 --- a/rpent/utils/logging.py +++ b/rpent/utils/logging.py @@ -87,6 +87,12 @@ def init_output_dir(log_dir: str | Path | None = None, verbose: bool = False) -> if log_dir is None: log_dir = get_repo_root() / "logs" _output_dir = Path(log_dir) + if not _log_initialized and _output_dir.exists() and any(_output_dir.iterdir()): + print( + f"Warning: RPent output directory is not empty: {_output_dir}; existing files may be overwritten!", + file=sys.stderr, + flush=True, + ) _output_dir.mkdir(parents=True, exist_ok=True) if _log_initialized: From 3f69c76023ac1700deae5e4eb941c29aa49059ea Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 10:24:05 +0000 Subject: [PATCH 23/25] chore: clean up state.py a bit Signed-off-by: Jiaxing Qiu --- rpent/tools/state.py | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/rpent/tools/state.py b/rpent/tools/state.py index 7c6a1471..81a4ef6a 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -5,7 +5,7 @@ import fnmatch import json import os -from collections.abc import Iterator +from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path @@ -92,11 +92,6 @@ class EnvState: def __init__(self, output_dir: Path | str): self._output_dir = Path(output_dir) - self._output_dir.mkdir(parents=True, exist_ok=True) - self._steps: list[StepRecord] = [] - self._run_artifacts: set[str] = set() - self._open_count = 0 - self._next_step = 0 self.reset() # -- private file resolution ----------------------------------------- @@ -133,9 +128,8 @@ def _temporary_file(self, destination: Path) -> Path: def _record_for(self, step: int) -> StepRecord: """Return the live step record at ``step`` for in-place updates.""" - for record in self._steps: - if record.step_idx == step: - return record + if 0 <= step < len(self._steps): + return self._steps[step] raise KeyError(f"step {step} not present in state trace") # -- manifest -------------------------------------------------------- @@ -159,14 +153,9 @@ def _write_manifest(self) -> None: def reset(self) -> None: """Reset the in-memory trace without removing on-disk artifacts.""" self._output_dir.mkdir(parents=True, exist_ok=True) - self._steps = [] - self._run_artifacts = set() - self._open_count = 0 - self._next_step = 0 - - @property - def next_step_idx(self) -> int: - return self._next_step + self._steps: list[StepRecord] = [] + self._run_artifacts: set[str] = set() + self._step_open = False @property def latest_step(self) -> int | None: @@ -366,17 +355,17 @@ def record_step( result: dict | None = None, elapsed_s: float | None = None, extras: dict[str, Any] | None = None, - ) -> Iterator[int]: + ) -> Generator[int, None, None]: """Append a new step record and yield its index. The step is committed to the trace immediately, so any subsequent :meth:`save` without an explicit ``step`` attaches to it. If the block raises, the step and any artifacts written to it are rolled back. """ - if self._open_count: + if self._step_open: raise RuntimeError("a step record is already open") record = StepRecord( - step_idx=self._next_step, + step_idx=len(self._steps), state=copy.deepcopy(state), terminated=terminated, truncated=truncated, @@ -386,8 +375,7 @@ def record_step( extras=copy.deepcopy(extras or {}), ) self._steps.append(record) - self._next_step = record.step_idx + 1 - self._open_count += 1 + self._step_open = True self._write_manifest() try: yield record.step_idx @@ -397,7 +385,6 @@ def record_step( destination.unlink(missing_ok=True) if self._steps and self._steps[-1] is record: self._steps.pop() - self._next_step = record.step_idx try: self._write_manifest() except Exception as exc: @@ -406,16 +393,13 @@ def record_step( ) raise finally: - self._open_count -= 1 + self._step_open = False def get(self, step: int = -1) -> StepRecord: resolved_step = self._resolve_read_step(step) if resolved_step is None: raise ValueError("step records are not run-level artifacts") - for record in self._steps: - if record.step_idx == resolved_step: - return copy.deepcopy(record) - raise KeyError(f"step {resolved_step} not present in state trace") + return copy.deepcopy(self._record_for(resolved_step)) def records(self) -> list[StepRecord]: return copy.deepcopy(self._steps) From b3b2145eb65e00f438cb12c2921b1ded98c101b0 Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 10:29:58 +0000 Subject: [PATCH 24/25] chore: remove unused state functions for simplicity Signed-off-by: Jiaxing Qiu --- rpent/tools/state.py | 127 ------------------------------------------- 1 file changed, 127 deletions(-) diff --git a/rpent/tools/state.py b/rpent/tools/state.py index 81a4ef6a..b6feb499 100644 --- a/rpent/tools/state.py +++ b/rpent/tools/state.py @@ -2,7 +2,6 @@ from __future__ import annotations import copy -import fnmatch import json import os from collections.abc import Generator @@ -72,21 +71,6 @@ def to_blob(self) -> dict[str, Any]: blob["extras"] = self.extras return blob - @classmethod - def from_blob(cls, blob: dict[str, Any]) -> "StepRecord": - return cls( - step_idx=int(blob["step_idx"]), - state=dict(blob.get("state") or {}), - terminated=bool(blob["terminated"]), - truncated=bool(blob["truncated"]), - artifacts={str(name) for name in blob.get("artifacts") or []}, - command=blob.get("command"), - result=blob.get("result"), - elapsed_s=blob.get("elapsed_s"), - extras=dict(blob.get("extras") or {}), - ) - - class EnvState: """Own a run's step trace and all state-related files in its output root.""" @@ -287,61 +271,6 @@ def exists(self, name: str, *, step: int | None = -1) -> bool: return False return self._artifact_file(name, resolved_step).exists() - def remove(self, name: str, *, step: int | None) -> bool: - destination = self._artifact_file(name, step) - if not destination.exists(): - return False - if step is None: - self._run_artifacts.discard(name) - else: - self._record_for(step).artifacts.discard(name) - destination.unlink() - if step is not None: - try: - destination.parent.rmdir() - except OSError: - pass - self._write_manifest() - return True - - def list( - self, - pattern: str = "*", - *, - step: int | None = -1, - ) -> list[str]: - resolved_step = self._resolve_read_step(step) - if resolved_step is None: - names = [ - name - for name in self._run_artifacts - if self._artifact_file(name, None).exists() - ] - else: - record = self.get(resolved_step) - names = [ - name - for name in record.artifacts - if self._artifact_file(name, resolved_step).exists() - ] - return sorted(name for name in names if fnmatch.fnmatch(name, pattern)) - - def list_all(self, pattern: str = "*") -> list[tuple[int | None, str]]: - artifacts: list[tuple[int | None, str]] = [ - (None, name) for name in self.list(pattern, step=None) - ] - for record in self._steps: - artifacts.extend( - (record.step_idx, name) - for name in record.artifacts - if fnmatch.fnmatch(name, pattern) - and self._artifact_file(name, record.step_idx).exists() - ) - return sorted( - artifacts, - key=lambda item: (-1 if item[0] is None else item[0], item[1]), - ) - # -- step records ---------------------------------------------------- @contextmanager @@ -403,59 +332,3 @@ def get(self, step: int = -1) -> StepRecord: def records(self) -> list[StepRecord]: return copy.deepcopy(self._steps) - - # -- LLM-facing view ------------------------------------------------- - - def view( - self, - step: int = -1, - *, - image_slots: dict[str, str] | None = None, - ) -> dict[str, Any]: - try: - record = self.get(step) - except Exception as exc: - return {"error": f"state step not available: {exc}"} - - metadata: dict[str, Any] = {} - metadata_suffix = "_metadata.json" - for name in sorted(record.artifacts): - if not name.endswith(metadata_suffix): - continue - key = name.removesuffix(metadata_suffix) - try: - loaded = self.load(name, step=record.step_idx) - if isinstance(loaded, dict): - loaded = { - field: value - for field, value in loaded.items() - if field not in {"K", "T_base_cam"} - } - metadata[key] = loaded - except Exception as exc: - metadata[key] = {"error": str(exc)} - - out: dict[str, Any] = { - "step": record.step_idx, - "terminated": record.terminated, - "truncated": record.truncated, - "state": record.state, - "artifacts": sorted(record.artifacts), - "camera_meta": metadata, - "log": { - "command": record.command, - "result": record.result, - "elapsed_s": record.elapsed_s, - }, - } - if record.extras: - out["extras"] = record.extras - if image_slots: - for slot, name in image_slots.items(): - if name not in record.artifacts: - continue - try: - out[slot] = self.load_bytes(name, step=record.step_idx) - except FileNotFoundError: - continue - return out From 17aeb2d434c971281912575e670d9f9f86b1decb Mon Sep 17 00:00:00 2001 From: Jiaxing Qiu Date: Fri, 7 Aug 2026 10:41:58 +0000 Subject: [PATCH 25/25] feat: rewrite read_image tool to use EnvState API Signed-off-by: Jiaxing Qiu --- rpent/cli/main.py | 2 +- rpent/planner/api_loop.py | 74 ++++++++++++++++++++++++++++++++------- rpent/tools/toolkit.py | 9 ++++- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/rpent/cli/main.py b/rpent/cli/main.py index 92b083f7..94b22f1b 100644 --- a/rpent/cli/main.py +++ b/rpent/cli/main.py @@ -113,7 +113,7 @@ def _build_argparser() -> argparse.ArgumentParser: help="Never send image bytes to the model (api planner only). " "Use for text-only models that reject image input " "(e.g. 400 \"message type 'image_url' is not supported\"); " - "read_image then returns the file path with a notice.") + "read_image then returns the image name instead, with a notice.") ap.add_argument("--planner-timeout-s", type=int, default=None, help="Wall-clock cap for api/claude_code/codex planner runs. " "Terminal interactive API/Claude sessions are exempt. " diff --git a/rpent/planner/api_loop.py b/rpent/planner/api_loop.py index f649fd15..2a96a43d 100644 --- a/rpent/planner/api_loop.py +++ b/rpent/planner/api_loop.py @@ -15,6 +15,8 @@ import json import queue from collections import deque +from collections.abc import Callable +from pathlib import Path from typing import Any from pydantic_ai import Agent, BinaryContent, ModelSettings, Tool, ToolReturn @@ -42,6 +44,7 @@ from rpent.dashboard.interaction import DashboardInteractionPort, DashboardMessage from rpent.dashboard.planner_control import DashboardPlannerControl from rpent.planner.base import PlannerResult +from rpent.tools.state import EnvState from rpent.tools.toolkit import Toolkit from rpent.utils.logging import get_logger @@ -689,7 +692,7 @@ def _api_error_text(error: Exception, *, no_images: bool) -> str: def _build_tools(toolkit: Toolkit, *, no_images: bool = False) -> list[Tool]: """Build the API-only image reader plus pydantic-ai toolkit wrappers.""" - image_reader = read_image_text_only if no_images else read_image + image_reader = _make_image_reader(toolkit.state, no_images=no_images) tools: list[Tool] = [Tool(image_reader, name="read_image")] for spec in toolkit.get_tools_spec(): name = spec["name"] @@ -706,27 +709,72 @@ def _build_tools(toolkit: Toolkit, *, no_images: bool = False) -> list[Tool]: return tools -def read_image(path: str) -> ToolReturn: - """Read an explicitly provided local image file as visual input. +def _make_image_reader( + state: EnvState, + *, + no_images: bool, +) -> Callable[[str, int], ToolReturn | str]: + if no_images: - Environment observations are already embedded by ``view_env_state``; - this helper is only for other user-selected local files. - """ + def read_image_tool(name: str, step: int = -1) -> str: + return read_image_text_only(name, step, state=state) + + read_image_tool.__name__ = "read_image" + read_image_tool.__doc__ = read_image_text_only.__doc__ + return read_image_tool + + def read_image_tool(name: str, step: int = -1) -> ToolReturn: + return read_image(name, step, state=state) + + read_image_tool.__name__ = "read_image" + read_image_tool.__doc__ = read_image.__doc__ + return read_image_tool + + +def read_image(name: str, step: int = -1, *, state: EnvState) -> ToolReturn: + """Read a step-scoped image artifact as visual input.""" + resolved_step, path = _resolve_image_artifact(state, name, step) return ToolReturn( - return_value=path, - content=[BinaryContent.from_path(path)], + return_value={"artifact": name, "step": resolved_step}, + content=[ + BinaryContent( + data=state.load_bytes(name, step=resolved_step), + media_type=_image_media_type(path), + ) + ], ) -def read_image_text_only(path: str) -> str: - """``read_image`` stub for ``--no-images``: acknowledge, send no bytes.""" +def read_image_text_only(name: str, step: int = -1, *, state: EnvState) -> str: + """Acknowledge an image artifact without sending bytes to the model.""" + resolved_step, _ = _resolve_image_artifact(state, name, step) return ( - f"{path} exists, but image input is disabled (--no-images, text-only " - "model). Reason from textual state instead: view_env_state, " - "back_project, and the numeric fields in tool results." + f"Image artifact {name!r} exists at step {resolved_step}, but image " + "input is disabled (--no-images, text-only model). Reason from textual " + "state instead: view_env_state, back_project, and numeric tool results." ) +def _resolve_image_artifact( + state: EnvState, + name: str, + step: int, +) -> tuple[int, Path]: + record = state.get(step) + path = state.artifact_path(name, step=record.step_idx) + if name not in record.artifacts or not path.is_file(): + raise FileNotFoundError( + f"image artifact {name!r} is not available at step {step}" + ) + if path.suffix.lower() not in {".png", ".jpg", ".jpeg"}: + raise ValueError(f"artifact {name!r} is not an image") + return record.step_idx, path + + +def _image_media_type(path: Path) -> str: + return "image/jpeg" if path.suffix.lower() in {".jpg", ".jpeg"} else "image/png" + + def _make_tool_function(toolkit: Toolkit, name: str, *, no_images: bool = False): """Return a callable that dispatches one tool call to the toolkit.""" diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index e7c88a66..690f60af 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -20,7 +20,7 @@ from rpent.utils.templates import substitute if TYPE_CHECKING: - from rpent.tools.state import StepRecord + from rpent.tools.state import EnvState, StepRecord @dataclass(slots=True) @@ -181,6 +181,13 @@ def _register_common_tools(self) -> None: # Planner-facing API # ------------------------------------------------------------------ + @property + def state(self) -> EnvState: + """Return the run's artifact and step store.""" + if self._state is None: + raise RuntimeError("toolkit has no environment state") + return self._state + def get_tools_spec(self) -> list[dict[str, Any]]: """Return the tool schemas the LLM sees.""" return substitute(