diff --git a/src/agentpool/agents/native_agent/agent.py b/src/agentpool/agents/native_agent/agent.py index 36646d303..ff676e2a0 100644 --- a/src/agentpool/agents/native_agent/agent.py +++ b/src/agentpool/agents/native_agent/agent.py @@ -5,6 +5,7 @@ import asyncio from collections.abc import Awaitable, Callable, Sequence from contextlib import AsyncExitStack, asynccontextmanager +from dataclasses import replace from datetime import datetime, timedelta import inspect from pathlib import Path @@ -1297,6 +1298,24 @@ async def get_agentlet[AgentOutputType]( # noqa: PLR0915 ) registry.register(populated, turn_scope) + # Populate VikingCapability.model_capabilities with resolved + # model capabilities so viking_read can auto-detect whether + # to return image bytes (via _should_return_image_bytes). + # Like ModalityFilterCapability this is a capability-level + # population, not auto-injection of new capabilities. + from agentpool.capabilities.viking import VikingCapability + + if isinstance(cap, VikingCapability): + populated_viking = replace( + cap, + model_capabilities=resolved_caps, + ) + tool_capabilities[i] = populated_viking + for j, ext_cap in enumerate(self._external_capabilities): + if ext_cap is cap: + self._external_capabilities[j] = populated_viking + break + # Handle retries parameter: newer pydantic-ai uses dict form for output_retries if AgentRetries is not None and self._output_retries is not None: retries_param: int | dict[str, int] = { diff --git a/src/agentpool/capabilities/viking/__init__.py b/src/agentpool/capabilities/viking/__init__.py index 772c1df51..d35c1e608 100644 --- a/src/agentpool/capabilities/viking/__init__.py +++ b/src/agentpool/capabilities/viking/__init__.py @@ -26,6 +26,10 @@ from pydantic_ai.capabilities import AbstractCapability from pydantic_ai.toolsets import AgentToolset, FunctionToolset +from agentpool.capabilities.viking.constants import ( + IMAGE_EXTENSIONS as IMAGE_EXTENSIONS, + IMAGE_MIME_TYPES as IMAGE_MIME_TYPES, +) from agentpool.capabilities.viking.identity import VikingIdentity, _try_decode_api_key from agentpool.log import get_logger @@ -90,6 +94,23 @@ class VikingCapability(AbstractCapability[Any]): multimodal_bridge: bool = False """Enable multimodal bridge — auto-upload binary content to Viking before sending to the model.""" + support_vision: bool | None = None + """Result of ``viking_read`` for image URIs. + + Tri-state control over how image resources are returned to the model: + + - ``True`` — return image bytes (``BinaryImage``) regardless of model. + - ``False`` — return a text URI description, never image bytes. + - ``None`` (default) — auto-detect from ``model_capabilities.image_input``; + treated as text-only when capabilities are unknown (not injected or + field is ``None``). + + Note: unlike ``ModalityFilterCapability._is_modality_supported``, which + treats ``capabilities=None`` as pass-through, this capability treats an + unset/model ``None`` capability as text-only (safe degradation) — it is + the *producer* of image content and must not emit ``BinaryImage`` it + cannot guarantee the model accepts. + """ uploads_uri: str | None = None public_download_base_url: str | None = None enable_link: bool = False @@ -1151,6 +1172,30 @@ async def _handle_multimodal_bridge( return request_context return replace(request_context, messages=new_messages) + def _should_return_image_bytes(self) -> bool: + """Whether ``viking_read`` should return image bytes for image URIs. + + Decision order: + + 1. ``support_vision`` explicitly set — return its value. + 2. ``model_capabilities`` injected — return ``image_input`` (``None`` + counts as text-only). + 3. Otherwise — text-only (safe degradation). + + Note: unlike ``ModalityFilterCapability._is_modality_supported`` + (which treats ``capabilities=None`` as pass-through), this treats an + unavailable capability as text-only: this capability *produces* + image content, so it must never emit ``BinaryImage`` it cannot + guarantee the model accepts. + + Returns: + ``True`` when image bytes should be returned, ``False`` for text. + """ + if self.support_vision is not None: + return self.support_vision + caps = self.model_capabilities + return bool(caps and caps.image_input) + def _supports_modality(self, media_type: str) -> bool: """Check if the model supports the given media type. diff --git a/src/agentpool/capabilities/viking/constants.py b/src/agentpool/capabilities/viking/constants.py new file mode 100644 index 000000000..d87e39de2 --- /dev/null +++ b/src/agentpool/capabilities/viking/constants.py @@ -0,0 +1,46 @@ +"""Constants for the Viking capability — image extension detection. + +Kept in a dedicated module (not ``__init__``) so ``tools.py`` can import +them at runtime without importing the full capability package (avoiding +import cycles, since the capability module lazily imports tools). +""" + +from __future__ import annotations + + +# Image extensions recognized by the openviking server parser layer +# (``parse/parsers/media/constants.py``). MUST be kept in sync manually with +# the server's ``IMAGE_EXTENSIONS`` — extension is the authoritative signal +# since the server's ``stat`` API exposes no MIME field. +# +# ``.svg`` IS included (matching the server), but is a vector format most +# vision APIs reject, so ``viking_read`` downgrades SVG URIs to a text hint +# and never returns bytes for them. See ``_should_return_image_bytes``. +IMAGE_EXTENSIONS: frozenset[str] = frozenset({ + ".png", + ".jpg", + ".jpeg", + ".gif", + ".bmp", + ".webp", + ".svg", + ".tiff", + ".tif", + ".ico", + ".jp2", +}) + +# MIME mapping for the byte-return image extensions above. Unknown +# extensions fall back to ``application/octet-stream``. +IMAGE_MIME_TYPES: dict[str, str] = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".bmp": "image/bmp", + ".webp": "image/webp", + ".tiff": "image/tiff", + ".tif": "image/tiff", + ".ico": "image/x-icon", + ".jp2": "image/jp2", +} diff --git a/src/agentpool/capabilities/viking/tools.py b/src/agentpool/capabilities/viking/tools.py index b22e0cfc7..611b0a30d 100644 --- a/src/agentpool/capabilities/viking/tools.py +++ b/src/agentpool/capabilities/viking/tools.py @@ -9,12 +9,14 @@ from __future__ import annotations import asyncio +from pathlib import PurePosixPath from typing import TYPE_CHECKING, Any, Literal import uuid -from pydantic_ai.messages import ToolReturn +from pydantic_ai.messages import BinaryImage, ToolReturn from pydantic_ai.tools import RunContext # noqa: TC002 - needed at runtime for get_type_hints() +from agentpool.capabilities.viking.constants import IMAGE_EXTENSIONS, IMAGE_MIME_TYPES from agentpool.capabilities.viking.utils import ( add_line_numbers, format_glob_results, @@ -39,6 +41,29 @@ def _get_session_id(ctx: RunContext[Any]) -> str | None: return None +def _is_image_resource(uri: str) -> bool: + """Whether a URI points to an image resource by its file extension. + + Matches the openviking server's extension-based image detection + (``IMAGE_EXTENSIONS``). SVG is deliberately excluded — it is a vector + format most vision APIs reject, so it never enters the byte path. + """ + return PurePosixPath(uri).suffix.lower() in IMAGE_EXTENSIONS + + +def _image_uri_hint(uri: str) -> str: + """Text hint for an image URI when image bytes are not returned. + + Used when the model cannot consume image bytes (text-only) or bytes + are forced off. Mentions the URI so the model can still reference it. + """ + return ( + f"[Image resource: {uri}]\n" + f"The file is an image and cannot be shown as text. The image is " + f"stored at the URI above — reference it when discussing the content." + ) + + def build_tools(cap: VikingCapability) -> list[Callable[..., Any]]: """Build the list of tool functions for the Viking capability. @@ -403,7 +428,36 @@ async def viking_read( client = await cap._ensure_client() uri_list = [uris] if isinstance(uris, str) else uris sections: list[str] = [] + image_parts: list[BinaryImage] = [] for u in uri_list: + is_image = _is_image_resource(u) + suffix = PurePosixPath(u).suffix.lower() + # SVG is a vector format most vision APIs reject — it + # never enters the byte path, always degrades to a text + # hint, regardless of the support_vision / model caps. + if is_image and (not cap._should_return_image_bytes() or suffix == ".svg"): + # Image resource but the model can't consume image + # bytes (or forced text / vector SVG) — text URI hint. + if len(uri_list) > 1: + sections.append(f"=== {u} ===\n{_image_uri_hint(u)}") + else: + sections.append(_image_uri_hint(u)) + continue + + if is_image: + # Image resource and the model accepts image bytes. + data = await client.download_bytes(u) + media_type = IMAGE_MIME_TYPES.get( + PurePosixPath(u).suffix.lower(), "application/octet-stream" + ) + image_idx = len(image_parts) + 1 # 1-based, matches content order + image_parts.append(BinaryImage(data=data, media_type=media_type)) + if len(uri_list) > 1: + sections.append(f"=== {u} ===\n[Image #{image_idx}: {media_type}]") + else: + sections.append(f"[Image #{image_idx}: {media_type}]") + continue + if level == "abstract": content = await client.abstract(u) elif level == "overview": @@ -422,6 +476,16 @@ async def viking_read( sections.append(f"=== {u} ===\n{numbered}") else: sections.append(numbered) + + if image_parts: + # Mixed content: text sections describe each file; image + # bytes follow as BinaryImage parts the model can view. + tool_content: list[Any] = ["\n\n".join(sections)] + tool_content.extend(image_parts) + return ToolReturn( + return_value="\n\n".join(sections), + content=tool_content, + ) return ToolReturn(return_value="\n\n".join(sections)) except Exception as e: return ToolReturn(return_value=f"viking_read error: {e}") diff --git a/src/agentpool_config/capabilities.py b/src/agentpool_config/capabilities.py index cc23bfaae..cbafeb876 100644 --- a/src/agentpool_config/capabilities.py +++ b/src/agentpool_config/capabilities.py @@ -184,6 +184,20 @@ class VikingCapabilityConfig(BaseModel): """Override for sessions URI. Default: viking://user/{user}/sessions/""" multimodal_bridge: bool = False """Enable multimodal bridge (Phase 6, not yet implemented).""" + support_vision: bool | None = None + """Result of viking_read for image URIs. + + Tri-state control over how image resources are returned to the model: + + - ``True`` — return image bytes (``BinaryImage``) regardless of model. + - ``False`` — return a text URI description, never image bytes. + - ``None`` (default) — auto-detect from resolved model capabilities + (``image_input``); text-only when unknown. + + When forcing ``True`` on a model that does not actually accept image + input, configure ``type: modality_filter`` as a safety net so the + image is degraded before reaching the model API. + """ uploads_uri: str | None = None """Override for uploads URI.""" public_download_base_url: str | None = None diff --git a/tests/capabilities/viking/test_viking.py b/tests/capabilities/viking/test_viking.py index 87817373f..e0382cdca 100644 --- a/tests/capabilities/viking/test_viking.py +++ b/tests/capabilities/viking/test_viking.py @@ -13,6 +13,7 @@ from unittest.mock import AsyncMock, MagicMock from pydantic import ValidationError +from pydantic_ai.messages import BinaryImage from pydantic_ai.models import ModelRequestContext, ModelRequestParameters from pydantic_ai.models.test import TestModel import pytest @@ -37,7 +38,7 @@ is_viking_uri, truncate_text, ) -from agentpool_config.capabilities import VikingCapabilityConfig +from agentpool_config.capabilities import VikingCapabilityConfig, build_capability pytestmark = pytest.mark.unit @@ -138,6 +139,32 @@ def test_default_config(self) -> None: assert cfg.public_download_base_url is None assert cfg.resource_read_level == "overview" + def test_default_support_vision_none(self) -> None: + """support_vision defaults to None (auto-detect from model capabilities).""" + cfg = VikingCapabilityConfig() + assert cfg.support_vision is None + + def test_support_vision_true(self) -> None: + """support_vision=True forces image bytes for image URIs.""" + cfg = VikingCapabilityConfig(support_vision=True) + assert cfg.support_vision is True + + def test_support_vision_false(self) -> None: + """support_vision=False forces text URI descriptions for image URIs.""" + cfg = VikingCapabilityConfig(support_vision=False) + assert cfg.support_vision is False + + def test_support_vision_build_passthrough(self) -> None: + """build_capability passes support_vision=... to VikingCapability. + + Explicit False must reach the capability (only None is filtered by + _import_and_instantiate), so forced-text mode survives the build. + """ + cap = build_capability(VikingCapabilityConfig(support_vision=False)) + assert cap.support_vision is False + cap_none = build_capability(VikingCapabilityConfig()) + assert cap_none.support_vision is None + def test_mode_retrieve(self) -> None: """Mode 'retrieve' is accepted.""" cfg = VikingCapabilityConfig(mode="retrieve") @@ -204,6 +231,71 @@ def test_discriminator_works(self) -> None: assert VikingCapabilityConfig in member_types +# --------------------------------------------------------------------------- +# support_vision — _should_return_image_bytes tri-state matrix +# --------------------------------------------------------------------------- + + +class TestShouldReturnImageBytes: + """Tri-state matrix for ``_should_return_image_bytes``.""" + + @staticmethod + def _cap( + support_vision: bool | None = None, image_input: bool | None = None + ) -> VikingCapability: + from agentpool_config.model_capabilities import ModelCapabilities + + cap = VikingCapability(mode="all", support_vision=support_vision) + cap.model_capabilities = ModelCapabilities(image_input=image_input) + return cap + + @pytest.mark.parametrize( + "image_input", + [None, False, True], + ids=["unknown", "false", "true"], + ) + def test_explicit_true_overrides_all(self, image_input: bool | None) -> None: + """support_vision=True forces bytes regardless of model capabilities.""" + cap = self._cap(support_vision=True, image_input=image_input) + assert cap._should_return_image_bytes() is True + + @pytest.mark.parametrize( + "image_input", + [None, False, True], + ids=["unknown", "false", "true"], + ) + def test_explicit_false_overrides_all(self, image_input: bool | None) -> None: + """support_vision=False forces text regardless of model capabilities.""" + cap = self._cap(support_vision=False, image_input=image_input) + assert cap._should_return_image_bytes() is False + + def test_auto_vision_model(self) -> None: + """support_vision=None + image_input=True auto-detects vision.""" + cap = self._cap(support_vision=None, image_input=True) + assert cap._should_return_image_bytes() is True + + def test_auto_text_only_model(self) -> None: + """support_vision=None + image_input=False auto-detects text-only.""" + cap = self._cap(support_vision=None, image_input=False) + assert cap._should_return_image_bytes() is False + + def test_auto_unknown_capability_field(self) -> None: + """support_vision=None + image_input=None (cache miss) degrades to text.""" + cap = self._cap(support_vision=None, image_input=None) + assert cap._should_return_image_bytes() is False + + def test_auto_uninjected_capabilities(self) -> None: + """support_vision=None + model_capabilities=None (not injected) → text. + + Safe degradation: the capability *produces* image content and must + not emit BinaryImage it cannot guarantee the model accepts (unlike + ModalityFilterCapability which passes through on None). + """ + cap = VikingCapability(mode="all", support_vision=None) + cap.model_capabilities = None + assert cap._should_return_image_bytes() is False + + # --------------------------------------------------------------------------- # 8.2 — Test __aenter__/__aexit__ lifecycle # --------------------------------------------------------------------------- @@ -652,6 +744,219 @@ async def test_viking_read_multi_uri_no_header_for_single( assert "===" not in result.return_value + # ------------------------------------------------------------------ + # viking_read image branch (support_vision) + # ------------------------------------------------------------------ + + @pytest.mark.asyncio + async def test_viking_read_image_support_vision_true( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """support_vision=True returns BinaryImage with correct data & mime.""" + viking_cap.support_vision = True + mock_client.download_bytes = AsyncMock(return_value=b"\x89PNG-fake-image-bytes") + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris="viking://photo.png") + + mock_client.download_bytes.assert_called_once_with("viking://photo.png") + mock_client.read.assert_not_called() + assert result.content is not None + parts = list(result.content) + assert any(isinstance(p, BinaryImage) for p in parts) + img = next(p for p in parts if isinstance(p, BinaryImage)) + assert img.data == b"\x89PNG-fake-image-bytes" + assert img.media_type == "image/png" + + @pytest.mark.asyncio + async def test_viking_read_image_support_vision_false( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """support_vision=False returns text URI hint, no download.""" + viking_cap.support_vision = False + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris="viking://photo.png") + + mock_client.download_bytes.assert_not_called() + mock_client.read.assert_not_called() + assert result.content is None + assert "viking://photo.png" in result.return_value + assert "Image resource" in result.return_value + + @pytest.mark.asyncio + async def test_viking_read_image_auto_vision_model( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """support_vision=None + image_input=True returns image bytes.""" + from agentpool_config.model_capabilities import ModelCapabilities + + viking_cap.support_vision = None + viking_cap.model_capabilities = ModelCapabilities(image_input=True) + mock_client.download_bytes = AsyncMock(return_value=b"webp-data") + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris="viking://pic.webp") + + parts = list(result.content) if result.content is not None else [] + assert any(isinstance(p, BinaryImage) for p in parts) + img = next(p for p in parts if isinstance(p, BinaryImage)) + assert img.media_type == "image/webp" + + @pytest.mark.asyncio + async def test_viking_read_image_auto_text_only_model( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """support_vision=None + image_input=False returns text URI hint.""" + from agentpool_config.model_capabilities import ModelCapabilities + + viking_cap.support_vision = None + viking_cap.model_capabilities = ModelCapabilities(image_input=False) + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris="viking://photo.png") + + assert result.content is None + assert "Image resource" in result.return_value + mock_client.download_bytes.assert_not_called() + + @pytest.mark.asyncio + async def test_viking_read_non_image_ignores_support_vision( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """Non-image URIs keep the text path regardless of the switch.""" + viking_cap.support_vision = True + mock_client.read = AsyncMock(return_value="text content") + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris="viking://doc.md") + + mock_client.download_bytes.assert_not_called() + mock_client.read.assert_called_once() + assert result.content is None + assert "text content" in result.return_value + + @pytest.mark.asyncio + async def test_viking_read_image_download_error( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """download_bytes failure returns viking_read error text, no raise.""" + viking_cap.support_vision = True + mock_client.download_bytes = AsyncMock(side_effect=RuntimeError("boom")) + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris="viking://photo.png") + + assert "viking_read error" in result.return_value + assert "boom" in result.return_value + + @pytest.mark.asyncio + async def test_viking_read_image_mixed_uris( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """Mixed image + text URIs: image bytes + text sections, order kept.""" + viking_cap.support_vision = True + mock_client.read = AsyncMock(return_value="doc body") + mock_client.download_bytes = AsyncMock(return_value=b"img-bytes") + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris=["viking://a.md", "viking://photo.jpg", "viking://b.md"]) + + assert mock_client.read.call_count == 2 + mock_client.download_bytes.assert_called_once_with("viking://photo.jpg") + assert "=== viking://photo.jpg ===" in result.return_value + parts = list(result.content) if result.content is not None else [] + assert any(isinstance(p, BinaryImage) for p in parts) + + @pytest.mark.asyncio + async def test_viking_read_multi_image_index_maps_to_content_order( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """Multi-image reads: #N markers in return_value map to content order. + + The text return_value must carry indexed markers ([Image #1], [#2], ...) + in URIs order, and the BinaryImage parts in ToolReturn.content must + follow the same order — so the model can disambiguate which image + belongs to which URI. + """ + viking_cap.support_vision = True + mock_client.download_bytes = AsyncMock(side_effect=[b"a-bytes", b"b-bytes", b"c-bytes"]) + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool( + ctx, + uris=[ + "viking://one.png", + "viking://two.png", + "viking://three.png", + ], + ) + + # Markers appear in URI order, 1-based. + rv = result.return_value + i1, i2, i3 = rv.index("[Image #1"), rv.index("[Image #2"), rv.index("[Image #3") + assert i1 < i2 < i3 + # Content mirrors the same order. + imgs = [p for p in (result.content or []) if isinstance(p, BinaryImage)] + assert [p.data for p in imgs] == [b"a-bytes", b"b-bytes", b"c-bytes"] + assert [p.media_type for p in imgs] == ["image/png"] * 3 + + @pytest.mark.asyncio + async def test_viking_read_image_svg_never_returns_bytes( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """SVG images degrade to text URI hint even with support_vision=True.""" + viking_cap.support_vision = True + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + result = await read_tool(ctx, uris="viking://diagram.svg") + + mock_client.download_bytes.assert_not_called() + assert result.content is None + assert "Image resource" in result.return_value + + @pytest.mark.asyncio + async def test_viking_read_image_jpeg_mime_extension_map( + self, viking_cap: VikingCapability, mock_client: AsyncMock + ) -> None: + """Unknown image extension falls back to application/octet-stream.""" + viking_cap.support_vision = True + mock_client.download_bytes = AsyncMock(return_value=b"raw-bytes") + tools = build_tools(viking_cap) + read_tool = _get_tool(tools, "viking_read") + + ctx = _make_ctx() + await read_tool(ctx, uris="viking://data.xyz") + + # .xyz is not a known image extension → text path untouched. + mock_client.read = AsyncMock(return_value="content") + result2 = await read_tool(ctx, uris="viking://data.xyz") + assert result2.content is None + + # .jpeg maps to image/jpeg. + result3 = await read_tool(ctx, uris="viking://pic.jpeg") + parts = list(result3.content) if result3.content is not None else [] + img = next(p for p in parts if isinstance(p, BinaryImage)) + assert img.media_type == "image/jpeg" + # --------------------------------------------------------------------------- # 8.5 — Test each write tool with mocked client diff --git a/tests/test_agent_factory_modality.py b/tests/test_agent_factory_modality.py index e393329fd..8efd7d98e 100644 --- a/tests/test_agent_factory_modality.py +++ b/tests/test_agent_factory_modality.py @@ -200,6 +200,131 @@ async def test_no_inject_when_no_config() -> None: assert len(filter_caps) == 0 +# --------------------------------------------------------------------------- +# VikingCapability.model_capabilities population +# --------------------------------------------------------------------------- + + +async def test_viking_cap_populated_with_resolved_caps() -> None: + """VikingCapability.model_capabilities is populated by the factory. + + A user-configured VikingCapability must receive the resolved model + capabilities so ``viking_read`` can auto-detect image-byte returns. + """ + from agentpool.capabilities.viking import VikingCapability + + viking = VikingCapability(mode="all") + config = NativeAgentConfig( + model=_TestModelConfig(capabilities=_all_true_caps()), + ) + agent = Agent( + name="test", + model="test", + agent_config=config, + capabilities=[viking], + ) + pydantic_agent = await agent.get_agentlet(model=None, output_type=str) + viking_caps = [ + cap + for cap in pydantic_agent.root_capability.capabilities + if isinstance(cap, VikingCapability) + ] + assert len(viking_caps) == 1 + assert viking_caps[0].model_capabilities is not None + assert viking_caps[0].model_capabilities.image_input is True + assert viking_caps[0]._should_return_image_bytes() is True + + +async def test_viking_cap_text_only_model_returns_false() -> None: + """Text-only model resolution prevents image-byte returns in viking_read.""" + from agentpool.capabilities.viking import VikingCapability + + viking = VikingCapability(mode="all") + config = NativeAgentConfig( + model=_TestModelConfig(capabilities=_text_only_caps()), + ) + agent = Agent( + name="test", + model="test", + agent_config=config, + capabilities=[viking], + ) + pydantic_agent = await agent.get_agentlet(model=None, output_type=str) + viking_caps = [ + cap + for cap in pydantic_agent.root_capability.capabilities + if isinstance(cap, VikingCapability) + ] + assert len(viking_caps) == 1 + assert viking_caps[0]._should_return_image_bytes() is False + + +async def test_viking_cap_no_model_name_gets_all_none_caps() -> None: + """No resolvable model name injects ModelCapabilities() (all None fields). + + _should_return_image_bytes must degrade to text-only (safe default), + even though model_capabilities is a non-None object. + """ + from agentpool.capabilities.viking import VikingCapability + + viking = VikingCapability(mode="all") + agent = Agent( + name="test", + model="test", + capabilities=[viking], + ) + pydantic_agent = await agent.get_agentlet(model=None, output_type=str) + viking_caps = [ + cap + for cap in pydantic_agent.root_capability.capabilities + if isinstance(cap, VikingCapability) + ] + assert len(viking_caps) == 1 + # Injected (non-None), but all fields None → text-only degradation. + assert viking_caps[0].model_capabilities is not None + assert viking_caps[0].model_capabilities.image_input is None + assert viking_caps[0]._should_return_image_bytes() is False + + +async def test_viking_injected_caps_feed_multimodal_bridge() -> None: + """Injected ModelCapabilities also drive multimodal bridge modality checks. + + _supports_modality must reflect the resolved model: True for a vision + model (bridge keeps images as HTTP URLs), False for text-only (bridge + replaces images with viking:// text links). + """ + from agentpool.capabilities.viking import VikingCapability + + for caps in (_all_true_caps(), _text_only_caps()): + viking = VikingCapability(mode="all", multimodal_bridge=True) + config = NativeAgentConfig( + model=_TestModelConfig(capabilities=caps), + ) + agent = Agent( + name="test", + model="test", + agent_config=config, + capabilities=[viking], + ) + pydantic_agent = await agent.get_agentlet(model=None, output_type=str) + viking_caps = [ + cap + for cap in pydantic_agent.root_capability.capabilities + if isinstance(cap, VikingCapability) + ] + assert len(viking_caps) == 1 + cap = viking_caps[0] + # Bridge modality check follows resolved capabilities. + assert cap._supports_modality("image/png") is bool(caps.image_input) + + # A for_run() copy must preserve the injected caps so the bridge + # sees them inside a run. + from unittest.mock import MagicMock + + copy_cap = await cap.for_run(MagicMock()) # type: ignore[arg-type] + assert copy_cap._supports_modality("image/png") is bool(caps.image_input) + + # --------------------------------------------------------------------------- # 5.6 — FallbackModelConfig intersection (pessimistic) # --------------------------------------------------------------------------- diff --git a/uv.lock b/uv.lock index 5bd842928..228b6e2fb 100644 --- a/uv.lock +++ b/uv.lock @@ -281,7 +281,7 @@ requires-dist = [ { name = "yamling", specifier = ">=2.0.2" }, { name = "zstandard", marker = "extra == 'zed'", specifier = ">=0.23.0" }, ] -provides-extras = ["a2a", "bot", "braintrust", "clipboard", "coding", "composio", "events", "langfuse", "markitdown", "mcp-discovery", "mcp-run", "notifications", "promptlayer", "tiktoken", "tts", "watchdog", "zed", "viking"] +provides-extras = ["a2a", "bot", "braintrust", "clipboard", "coding", "composio", "events", "langfuse", "markitdown", "mcp-discovery", "mcp-run", "notifications", "promptlayer", "tiktoken", "tts", "viking", "watchdog", "zed"] [package.metadata.requires-dev] benchmark = [{ name = "pyinstrument" }] @@ -3395,14 +3395,14 @@ wheels = [ [[package]] name = "openviking-sdk" -version = "0.1.5" +version = "0.1.6" source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } dependencies = [ { name = "httpx" }, ] -sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e4/9c/b75da8d956bbfaf39288dd3cf7c734ba9225fba34e44362d02fbdb9a871d/openviking_sdk-0.1.5.tar.gz", hash = "sha256:f017dec938267aaea659122165f2872e189933a5604ba6d9ec0ae867b558ecaf", size = 33652, upload-time = "2026-07-23T06:18:56.211Z" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6b/65/021ba0a777750d536b70e7ca5c14888d9e906627b80a9332b2c644082480/openviking_sdk-0.1.6.tar.gz", hash = "sha256:7ecfdebdbe3538556584e4348f19f1c990831c739404ee4e06e251f3d8a6641d", size = 40041, upload-time = "2026-08-03T07:15:38.963Z" } wheels = [ - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/53/c0/481ec44e2f98cf1afd47d4c115756dd4b1fa674caa4b99b0f72723d6fc33/openviking_sdk-0.1.5-py3-none-any.whl", hash = "sha256:f29af1c56e1a4477c881d584fa5178b5a34600847926cfe21fddd333ccb576ec", size = 21740, upload-time = "2026-07-23T06:18:54.864Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6b/86/e0a9ff16d5b5eb38871450110b0ba64bff7a46020c617c482d4ce0ea3e7a/openviking_sdk-0.1.6-py3-none-any.whl", hash = "sha256:61e55f51f2733b80950488bb57299c5fd1bface388242ebff02c245ab6e04bd5", size = 25058, upload-time = "2026-08-03T07:15:37.776Z" }, ] [[package]]