Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
173a518
Support raw image offload in v1 train client
eligotts Jun 18, 2026
de37650
Enforce strict raw multimodal descriptors
eligotts Jun 20, 2026
e6b13dc
Simplify v1 raw multimodal: drop the cache-miss retry subsystem
S1ro1 Jun 27, 2026
9430999
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Jun 27, 2026
4a7b37a
feat: support inline multimodal images
eligotts Jun 28, 2026
0b1d73f
Simplify v1 raw image offload path
eligotts Jun 29, 2026
2d4969b
Preserve v1 node usage in trace dumps
eligotts Jun 29, 2026
7ade0b2
Surface request preparation failures on traces
eligotts Jun 29, 2026
0dc57a1
Require raw image URIs in v1 sidecars
eligotts Jun 29, 2026
18b0fbe
Share multimodal image preparation across clients
eligotts Jun 29, 2026
22c7cf4
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Jun 29, 2026
9b3e7ee
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Jul 1, 2026
2c2824a
Cover every image part shape at multimodal ingress
eligotts Jul 4, 2026
9bc3cc3
Merge commit '5885ab9c54' into codex/v1-raw-image-offload
eligotts Jul 5, 2026
2b1627d
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Jul 23, 2026
3d2068b
Drop the orphaned prepare_messages client hook
eligotts Jul 23, 2026
d9e79d6
Sort renderer client imports
eligotts Jul 23, 2026
ae516c1
Cover kept_tokens in the _NODE_DUMP_EXCLUDE docstring
eligotts Jul 23, 2026
0e41666
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Jul 31, 2026
88c759b
style: sort in-function imports in the multimodal client type test (r…
eligotts Jul 31, 2026
ebc0e2f
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Aug 4, 2026
c9d1f13
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Aug 5, 2026
97b6af8
fix: repair interception image-offload ingress and image-part parsing
eligotts Aug 6, 2026
4c6f31e
refactor: canonicalize image parts at v1 training ingress
eligotts Aug 6, 2026
bef9fd2
chore: drop unused asyncio import in renderer_client
eligotts Aug 6, 2026
1f3159f
Merge remote-tracking branch 'origin/main' into codex/v1-raw-image-of…
eligotts Aug 6, 2026
62b103f
refactor: drop processed-mm key scan from v1 sidecar validation
eligotts Aug 6, 2026
8dd92a0
refactor: drop resurrected MessageNode finish_reason/usage
eligotts Aug 6, 2026
9a85a77
refactor: fail fast on multimodal through the v0 legacy bridge
eligotts Aug 6, 2026
35714d3
refactor: reject legacy multimodal at message/token conversion
eligotts Aug 6, 2026
07deb99
style: inline legacy multimodal error strings at raise sites
eligotts Aug 7, 2026
f569896
test: drop thin multimodal unit tests
eligotts Aug 7, 2026
d754774
docs: restore the short Branch.multi_modal_data docstring
eligotts Aug 7, 2026
333052c
chore: relock for the nemo-gym extra
eligotts Aug 7, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions verifiers/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ class ResponseTokens(CustomBaseModel):
completion_logprobs: list[float]
routed_experts: RoutedExpertsPayload | None = None
# Renderer-emitted multimodal sidecar (renderers.base.MultiModalData)
# carrying processed pixel_values / placeholder ranges per modality.
# carrying raw image descriptors / placeholder ranges per modality.
# Populated by the renderer client when the rollout went through a
# multimodal-aware renderer; ``None`` otherwise. Stored as ``Any`` to
# avoid a hard import dependency on ``renderers`` at this layer.
Expand Down Expand Up @@ -263,7 +263,7 @@ class TrajectoryStepTokens(TypedDict):
is_truncated: bool
routed_experts: RoutedExpertsPayload | None
# Renderer-emitted multimodal sidecar (renderers.base.MultiModalData)
# carrying processed pixel_values / placeholder ranges per modality.
# carrying raw image descriptors / placeholder ranges per modality.
# ``NotRequired`` because text-only rollouts (and non-renderer client
# types) never populate it.
multi_modal_data: NotRequired[Any]
Expand Down
66 changes: 66 additions & 0 deletions verifiers/utils/multimodal.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""Multimodal ingress: canonicalize image parts to
``{"type": "image_url", "image_url": {"url": ...}}`` with the URL offloaded
to a ``file://`` run image asset."""

from __future__ import annotations

from importlib import import_module
from pathlib import Path
from typing import Any


def _offload_image_url(url: str, image_dir: Path | None) -> str | None:
try:
offload_image_to_run_assets = import_module(
"renderers.mm_store"
).offload_image_to_run_assets
except (
ImportError,
AttributeError,
) as exc: # pragma: no cover - dependency-version guard
raise RuntimeError(
"Multimodal training requires a renderers version with raw image asset offload support."
) from exc

return offload_image_to_run_assets(url, image_dir=image_dir)


def _prepare_image_part(part: dict[str, Any], *, image_dir: Path | None) -> None:
"""Rewrite one image part to the canonical shape with an offloaded URL."""
if part.get("type") == "image": # HF-style: URL directly under ``image``
part["type"] = "image_url"
part["image_url"] = {"url": part.pop("image", None)}
image_url = part.get("image_url")
if not isinstance(image_url, dict):
raise TypeError(
"v1 multimodal training requires the OpenAI image part shape "
f"{{'image_url': {{'url': ...}}}}; got image_url of type {type(image_url).__name__}"
)
url = image_url.get("url")
if not isinstance(url, str):
raise TypeError(
f"v1 multimodal training requires string image URLs; got {url!r}"
)
if url.startswith("file://"):
return
offloaded = _offload_image_url(url, image_dir)
if offloaded is None:
raise RuntimeError(
"v1 multimodal training accepts data:image/...;base64 or file:// "
f"image sources; got {url.split(',', 1)[0]!r}"
)
image_url["url"] = offloaded


def prepare_images_inplace(value: Any, *, image_dir: Path | None = None) -> None:
"""Rewrite every image part reachable from a request body to the
canonical shape with a ``file://`` URL; reject unsupported sources."""
if isinstance(value, dict):
if value.get("type") in ("image", "image_url"):
_prepare_image_part(value, image_dir=image_dir)
return
for child in value.values():
prepare_images_inplace(child, image_dir=image_dir)
elif isinstance(value, (list, tuple)):
for child in value:
prepare_images_inplace(child, image_dir=image_dir)
9 changes: 9 additions & 0 deletions verifiers/v1/clients/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ class RelayReply:


class Client(ABC):
async def prepare_request_body(self, dialect: Dialect, body: dict) -> dict:
"""Normalize a provider request before the interception server parses/traces it.

Relay clients keep the request verbatim. Training clients may rewrite heavy
in-process payloads (for example base64 images) into stable run-asset refs so the
trace, renderer, and trainer all see the same cheap message content.
"""
return body

@abstractmethod
async def get_response(
self,
Expand Down
33 changes: 15 additions & 18 deletions verifiers/v1/clients/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
from openai import OpenAIError
from renderers import OverlongPromptError as RendererOverlongPromptError
from renderers import RenderedTokens, Renderer, RendererConfig
from renderers.base import ToolCallParseStatus
from renderers.base import ToolCallParseStatus, is_multimodal

from verifiers.utils.multimodal import prepare_images_inplace
from verifiers.v1.clients.base import build_async_openai
from verifiers.v1.clients.client import SESSION_ID_HEADER, Client
from verifiers.v1.configs.client import TrainClientConfig
Expand Down Expand Up @@ -175,16 +176,6 @@ def _is_valid_incremental_tail(messages: list[dict[str, Any]]) -> bool:
return all(role == "tool" for role in roles)


def _has_multimodal_content(messages) -> bool:
for message in messages:
content = getattr(message, "content", None)
if not isinstance(content, list):
continue
if any(getattr(part, "type", None) == "image_url" for part in content):
return True
return False


@dataclass
class RendererSlot:
"""One renderer and the rollouts currently holding it. Encoding mutates a fast
Expand Down Expand Up @@ -314,6 +305,11 @@ def __init__(self, config: TrainClientConfig) -> None:
multiplex=config.multiplex,
).warm()

async def prepare_request_body(self, dialect: Dialect, body: dict) -> dict:
if isinstance(dialect, ChatDialect):
await asyncio.to_thread(prepare_images_inplace, body)
return body
Comment thread
macroscopeapp[bot] marked this conversation as resolved.

async def get_response(
self,
dialect: Dialect,
Expand Down Expand Up @@ -369,22 +365,23 @@ async def get_response(
async with pool.acquire() as slot:
renderer = slot.renderer
# Only build the (O(context)) previous-turn token ids once the cheap guards pass — a
# multimodal prompt or a tail that isn't a clean `[tool*, user?]` extension can't bridge.
can_bridge = (
turn is not None
and not _has_multimodal_content(prompt)
and _is_valid_incremental_tail(wire_messages)
)
# tail that isn't a clean `[tool*, user?]` extension can't bridge.
can_bridge = turn is not None and _is_valid_incremental_tail(wire_messages)
previous_ids = turn.previous_token_ids() if can_bridge else None
if previous_ids is not None:
previous_prompt_ids, previous_completion_ids = previous_ids

def bridge():
kwargs: dict[str, Any] = {"tools": wire_tools}
if is_multimodal(renderer):
kwargs["previous_multi_modal_data"] = (
turn.previous_multi_modal_data()
)
return renderer.bridge_to_next_turn(
previous_prompt_ids,
previous_completion_ids,
wire_messages,
tools=wire_tools,
**kwargs,
)

bridged = await slot.run(bridge)
Expand Down
113 changes: 82 additions & 31 deletions verifiers/v1/graph.py
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,28 @@ def _decode_ndarray(d: dict) -> np.ndarray:
return np.frombuffer(d["data"], dtype=np.dtype(d["dtype"])).reshape(d["shape"])


def _validate_raw_mm_item(item: Any) -> dict[str, Any]:
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
if not isinstance(item, dict):
raise TypeError(
"v1 multimodal sidecars must be raw image descriptor dicts, "
f"got {type(item).__name__}"
)
if not isinstance(item.get("raw_image_uri"), str) or not item["raw_image_uri"]:
raise ValueError("v1 multimodal sidecars require raw_image_uri")
return dict(item)


def _validate_raw_mm_data(mmd: MultiModalData) -> MultiModalData:
return MultiModalData(
mm_hashes={k: list(v) for k, v in mmd.mm_hashes.items()},
mm_placeholders={k: list(v) for k, v in mmd.mm_placeholders.items()},
mm_items={
modality: [_validate_raw_mm_item(item) for item in items]
for modality, items in mmd.mm_items.items()
},
)


class MessageNode(BaseModel):
"""One message in the graph: a message plus the tokens it adds to the cumulative
sequence. Concatenating a root→leaf path's nodes reconstructs that branch's full token
Expand Down Expand Up @@ -107,11 +129,12 @@ class MessageNode(BaseModel):
rewards were all equal is assigned zeros and carries no gradient, while an unassigned node was
never scored at all."""
multi_modal_data: SkipJsonSchema[MultiModalData | None] = None
"""The renderer items for the images this message's content introduces (pixel tensors,
grids, hashes, placeholders) — the only carrier of the pixels from the env server to the
trainer. `Branch.multi_modal_data` concatenates them along the path into the training
`mm_kwargs`. Rides the wire as raw bytes (msgpack `bin`) since pydantic can't JSON the numpy;
kept off disk by the dump-site `exclude` in prime-rl (the tensors bloat the rollout jsonl)."""
"""The renderer items for images this message introduces.

Items are raw descriptors (hashes, grid metadata, ``raw_image_uri``), not
image-processor tensors. ``Branch.multi_modal_data`` concatenates them along
the path for the trainer.
"""
routed_experts: SkipJsonSchema[np.ndarray | None] = None
"""This node's slice of the MoE expert-routing array — uint8 `[len(token_ids), layers,
top_k]`, the expert ids inference selected for exactly this node's tokens. Attributed from
Expand All @@ -129,46 +152,48 @@ class MessageNode(BaseModel):

@field_serializer("multi_modal_data")
def serialize_multi_modal_data(self, mmd: MultiModalData | None) -> dict | None:
"""`MultiModalData` -> msgpack-safe dict so the pixel tensors ride the wire; numpy
`mm_items` values become raw-bytes `__nd__` dicts (every renderer emits `return_tensors="np"`)."""
"""`MultiModalData` -> msgpack-safe raw descriptor dict."""
if mmd is None:
return None
mmd = _validate_raw_mm_data(mmd)
return {
"mm_hashes": {k: list(v) for k, v in mmd.mm_hashes.items()},
"mm_placeholders": {
modality: [{"offset": p.offset, "length": p.length} for p in ranges]
for modality, ranges in mmd.mm_placeholders.items()
},
"mm_items": {
modality: [
{k: _encode_ndarray(v) for k, v in item.items()} for item in items
]
modality: [dict(item) for item in items]
for modality, items in mmd.mm_items.items()
},
}

@field_validator("multi_modal_data", mode="before")
@classmethod
def deserialize_multi_modal_data(cls, value: Any) -> MultiModalData | None:
if value is None or isinstance(value, MultiModalData):
if value is None:
return value
if isinstance(value, MultiModalData):
return _validate_raw_mm_data(value)
if not isinstance(value, dict):
raise TypeError(f"cannot build MultiModalData from {type(value).__name__}")
return MultiModalData(
mm_hashes={k: list(v) for k, v in (value.get("mm_hashes") or {}).items()},
mm_placeholders={
modality: [
PlaceholderRange(offset=p["offset"], length=p["length"])
for p in ranges
]
for modality, ranges in (value.get("mm_placeholders") or {}).items()
},
mm_items={
modality: [
{k: _decode_ndarray(v) for k, v in item.items()} for item in items
]
for modality, items in (value.get("mm_items") or {}).items()
},
return _validate_raw_mm_data(
MultiModalData(
mm_hashes={
k: list(v) for k, v in (value.get("mm_hashes") or {}).items()
},
mm_placeholders={
modality: [
PlaceholderRange(offset=p["offset"], length=p["length"])
for p in ranges
]
for modality, ranges in (value.get("mm_placeholders") or {}).items()
},
mm_items={
modality: list(items)
for modality, items in (value.get("mm_items") or {}).items()
},
)
)

@field_serializer("routed_experts")
Expand Down Expand Up @@ -368,6 +393,23 @@ def prompt_message_spans(
for span in tail_spans
]

def previous_multi_modal_data(self) -> MultiModalData | None:
"""Concatenate multimodal sidecars attached to the reusable prefix."""
merged = MultiModalData()
found = False
for nid in self.prefix_node_ids:
mmd = self.trace.nodes[nid].multi_modal_data
if mmd is None or mmd.is_empty():
continue
found = True
for modality, items in mmd.mm_items.items():
merged.mm_items.setdefault(modality, []).extend(items)
for modality, hashes in mmd.mm_hashes.items():
merged.mm_hashes.setdefault(modality, []).extend(hashes)
for modality, placeholders in mmd.mm_placeholders.items():
merged.mm_placeholders.setdefault(modality, []).extend(placeholders)
return merged if found else None

def commit(self, response: Response, tools: list[Tool] | None = None) -> int:
"""Add this turn to the graph; returns the committed assistant node's id."""
assistant_id = _commit_turn(self, response)
Expand Down Expand Up @@ -428,8 +470,9 @@ def _attribute_mm(
renderer emits items per modality in prompt order (message order, then content-part order),
so we walk the path advancing a per-modality cursor over every message's media but write
only the nodes created this turn — `path[:num_reused]` is the reused prefix, already
attributed when first created. Item order is all training needs; placeholder offsets aren't
carried."""
attributed when first created. Each node gets the hashes/items/placeholders for exactly the
media it introduced, preserving vLLM multimodal-list alignment when those node sidecars are
later merged for bridge or training."""
if mmd is None or mmd.is_empty():
return
cursors: dict[str, int] = {}
Expand All @@ -439,6 +482,7 @@ def _attribute_mm(
continue
node_items: dict[str, list] = {}
node_hashes: dict[str, list] = {}
node_placeholders: dict[str, list[PlaceholderRange]] = {}
for part in content:
modality = _part_modality(part)
if modality is None:
Expand All @@ -450,13 +494,20 @@ def _attribute_mm(
continue
items = mmd.mm_items.get(modality) or []
hashes = mmd.mm_hashes.get(modality) or []
placeholders = mmd.mm_placeholders.get(modality) or []
if k < len(items):
node_items.setdefault(modality, []).append(items[k])
if k < len(hashes):
node_hashes.setdefault(modality, []).append(hashes[k])
if node_items:
trace.nodes[node_id].multi_modal_data = MultiModalData(
mm_items=node_items, mm_hashes=node_hashes
if k < len(placeholders):
node_placeholders.setdefault(modality, []).append(placeholders[k])
if node_items or node_hashes or node_placeholders:
trace.nodes[node_id].multi_modal_data = _validate_raw_mm_data(
MultiModalData(
mm_items=node_items,
mm_hashes=node_hashes,
mm_placeholders=node_placeholders,
)
)


Expand Down
Loading
Loading