diff --git a/vision-bridge/README.md b/vision-bridge/README.md
index 53fa3de..c9ae72d 100644
--- a/vision-bridge/README.md
+++ b/vision-bridge/README.md
@@ -2,7 +2,7 @@
-Give a **text-only model the ability to work with images**, with no core changes. A filter takes the image out of the request (so the text-only model never breaks on an image it cannot accept) and leaves a marker in its place. A tool then lets the model send that image to a separate vision model on demand, asking whatever it wants, as many times as it wants. The image itself stays in the chat untouched.
+Give a **text-only model the ability to work with images**, with no core changes. A filter takes the image out of the request (so the text-only model never breaks on an image it cannot accept) and leaves a marker in its place. The tools then let the model send either a chat attachment or an image file in the connected Open Terminal to a separate vision model on demand. Chat attachments stay untouched and can be inspected as many times as needed.
> [!IMPORTANT]
> **Requires Open WebUI `0.11.0` or newer.** Both parts resolve chat ids through the core helper added in that release. They will not load on older versions.
@@ -46,8 +46,16 @@ Vision Bridge keeps the image and defers the looking. The text-only model drives
## How it works
1. **Filter** runs on the request to the text-only model. Each image part is replaced with a text marker: `[Image attached — file_id: . Call analyze_image(...) to inspect it.]`, or `[Image attached. Call analyze_image(query="…") to inspect the most recent image.]` when the id is not in the request (Open WebUI inlines uploaded images before filters run). The model receives the marker, never the image. The image stays in the chat and in storage.
-2. **Model calls `analyze_image(file_id, query)`** whenever it needs to see something. The tool resolves the file id to the stored image, sends it plus the question to the configured vision model, and returns the answer as text.
-3. **Re-query any time.** Because the image is never consumed or deleted, the model can call the tool again with a new question and get a fresh, different answer about the same image.
+2. **For chat attachments, the model calls `analyze_image(file_id, query)`.** The tool resolves the file id to the stored image, sends it plus the question to the configured vision model, and returns the answer as text.
+3. **For files in Open Terminal, the model calls `analyze_terminal_image(path, query)`.** The tool reads the image from the currently connected terminal and sends it directly to the configured vision model.
+4. **The filter adds tool-selection instructions.** They tell the model which vision tool to use, require real visual verification after generating or modifying an image, and reserve `read_file` for non-image content. Inline images returned by tools are also persisted before they are replaced with markers when possible.
+5. **Re-query any time.** Because an image is not consumed by analysis, the model can call the appropriate tool again with a new question.
+
+### Images generated in Open Terminal
+
+The additional `analyze_terminal_image` tool is required for the **Open WebUI → Open Terminal** workflow. A terminal-generated image exists at a terminal path, not as a chat attachment with a file id, so `analyze_image` could not find it. Previously this made the workflow stall: the bridge returned a missing-image error and the model could not continue with visual verification.
+
+Use `analyze_image` for images attached to the conversation and `analyze_terminal_image(path="...", query="...")` for PNG, JPEG, WebP, or other image files created or stored in the connected terminal. Do not call `read_file` first for a terminal image—the new tool performs that read internally and forwards the image to the vision model. For animations or GIFs, extract representative frames and inspect them with `analyze_terminal_image` when reliable visual verification is needed.
```
┌──────────────┐ image stripped ┌──────────────┐
diff --git a/vision-bridge/filter.py b/vision-bridge/filter.py
index 066c967..6eb9188 100644
--- a/vision-bridge/filter.py
+++ b/vision-bridge/filter.py
@@ -19,7 +19,7 @@
from open_webui.models.chats import Chats
from open_webui.storage.provider import Storage
from open_webui.socket.main import get_event_emitter
-from open_webui.utils.misc import get_message_list
+from open_webui.utils.misc import get_message_list, add_or_update_system_message
from open_webui.utils.chat_id import is_saved_chat_id
from open_webui.utils.chat import generate_chat_completion
from open_webui.utils.files import get_image_base64_from_file_id
@@ -106,6 +106,70 @@ async def inlet(
__event_emitter__: Optional[Callable[[dict], Any]] = None,
) -> dict:
messages = body.get("messages") or []
+ vision_bridge_note = """
+ [VISION BRIDGE TOOL POLICY]
+ You are a text-only primary model with access to a separate vision model through Vision Bridge.
+ Use the correct vision workflow depending on where the image is located.
+ 1. IMAGES ATTACHED TO THE CONVERSATION
+ If an image is attached to the conversation or represented by a marker such as:
+ [Image attached — file_id: ...]
+ or:
+ [Image attached. Call analyze_image(...) ...]
+ use `analyze_image`.
+ If a file_id is available, pass that exact file_id to `analyze_image`.
+ Do not attempt to inspect such images yourself.
+ 2. IMAGES STORED IN THE OPEN TERMINAL
+ If an image exists as a file inside the currently connected Open Terminal, use:
+ `analyze_terminal_image(path="...", query="...")`
+ to inspect it.
+ Examples include PNG, JPG, JPEG and WebP files created by Python, Pillow, ImageMagick, ffmpeg, rendering scripts, or other terminal commands.
+ IMPORTANT:
+ Do NOT call `read_file` before `analyze_terminal_image`.
+ Do NOT use `read_file` for visual inspection of terminal image files.
+ `analyze_terminal_image` reads the image from the terminal internally and sends it directly to the separate vision model.
+ Use `read_file` normally for text files, source code, JSON, logs, configuration files, and other non-image content.
+ 3. VISUAL VERIFICATION
+ When a task requires understanding, describing, evaluating, comparing, or verifying the actual appearance of an image, you MUST use the appropriate vision tool before completing the task.
+ Do not infer what an image looks like solely from:
+ - the code that generated it,
+ - the prompt used to create it,
+ - file metadata,
+ - filenames,
+ - image dimensions,
+ - or successful command execution.
+ A message such as:
+ "Image file read successfully"
+ is NOT visual analysis.
+ 4. GENERATED IMAGES
+ When you generate or modify an image in the terminal and visual verification is relevant:
+ 1. Create or modify the image.
+ 2. Call `analyze_terminal_image` on the resulting image file.
+ 3. Evaluate the vision model's response.
+ 4. If necessary, modify the image again.
+ 5. Re-analyze the new result.
+ 6. Continue until the requested visual result is achieved or further iteration is unnecessary.
+ Do not stop after merely generating the file if the task requires checking its appearance.
+ 5. ANIMATIONS AND GIFS
+ If reliable visual inspection of an animation or GIF is required and direct inspection is unsuitable, use the terminal to extract one or more representative frames as PNG files and inspect those frames using `analyze_terminal_image`.
+ Use the vision model's actual observations as visual evidence.
+ """.strip()
+
+ system_content = next(
+ (
+ str(msg.get("content", ""))
+ for msg in messages
+ if msg.get("role") == "system"
+ ),
+ "",
+ )
+
+ if "[VISION BRIDGE TOOL POLICY]" not in system_content:
+ messages = add_or_update_system_message(
+ vision_bridge_note,
+ messages,
+ append=True,
+ )
+ body["messages"] = messages
# Describe mode covers the newest image message; the strip below catches every other image.
if not self.valves.strip_only and self.valves.vision_model_id and (__user__ or {}).get("id"):
@@ -117,10 +181,53 @@ async def inlet(
user = await Users.get_user_by_id(__user__["id"])
if user:
await self._describe(target, user, __request__, __chat_id__, __event_emitter__)
+ user = None
+ if self.valves.strip_only and __request__ and (__user__ or {}).get("id"):
+ user = await Users.get_user_by_id(__user__["id"])
for msg in messages:
if _has_image(msg.get("content")):
- msg["content"] = _strip_images(msg["content"], self.valves.strip_only)
+ if self.valves.strip_only and user and msg.get("role") == "user":
+ content = msg.get("content") or []
+
+ text = " ".join(
+ part.get("text", "")
+ for part in content
+ if isinstance(part, dict) and part.get("type") == "text"
+ )
+
+ if "images from the tool results above" in text:
+ for part in content:
+ if (
+ not isinstance(part, dict)
+ or part.get("type") != "image_url"
+ ):
+ continue
+
+ url = (part.get("image_url") or {}).get("url", "")
+
+ if url.startswith("data:image/"):
+ try:
+ stored_url = await get_image_url_from_base64(
+ __request__,
+ url,
+ {},
+ user,
+ )
+
+ if stored_url:
+ part["image_url"]["url"] = stored_url
+
+ except Exception:
+ log.exception(
+ "Vision Bridge: failed to persist tool-result image"
+ )
+
+ msg["content"] = _strip_images(
+ msg["content"],
+ self.valves.strip_only,
+ )
+
return body
async def _describe(self, target, user, request, chat_id, event_emitter):
diff --git a/vision-bridge/tool.py b/vision-bridge/tool.py
index d72bc65..0916783 100644
--- a/vision-bridge/tool.py
+++ b/vision-bridge/tool.py
@@ -18,6 +18,7 @@
from open_webui.utils.misc import get_message_list
from open_webui.utils.chat_id import is_saved_chat_id
from open_webui.utils.chat import generate_chat_completion
+from open_webui.utils.tools import get_terminal_tools
from open_webui.utils.files import get_image_base64_from_file_id
log = logging.getLogger(__name__)
@@ -111,6 +112,134 @@ async def status(d, done=False):
await status("Done", done=True)
return answer
+ async def analyze_terminal_image(
+ self,
+ path: str,
+ query: str = "",
+ __request__: Any = None,
+ __user__: Optional[dict] = None,
+ __metadata__: Optional[dict] = None,
+ __event_emitter__: Optional[Callable[[dict], Any]] = None,
+ ) -> str:
+ """
+ Inspect an image file located in the currently connected Open Terminal.
+ Use this tool for images created or stored in the terminal instead of
+ calling read_file followed by analyze_image.
+ :param path: Full path to the image in the terminal, e.g. /home/user/output.png
+ :param query: The specific visual question to ask about the image.
+ :return: The vision model's analysis as text.
+ """
+
+ async def status(description, done=False):
+ if __event_emitter__:
+ await __event_emitter__(
+ {
+ "type": "status",
+ "data": {
+ "description": description,
+ "done": done,
+ },
+ }
+ )
+
+ if not self.valves.vision_model_id:
+ return "Vision Bridge is not configured: set a vision_model_id in the tool valves."
+
+ if not __user__ or not __user__.get("id"):
+ return "No user context available."
+
+ if not __request__:
+ return "No request context available."
+
+ terminal_id = (__metadata__ or {}).get("terminal_id")
+ if not terminal_id:
+ return "No active terminal is attached to this chat."
+
+ user = await Users.get_user_by_id(__user__["id"])
+ if not user:
+ return "User not found."
+
+ await status(f"Reading image from terminal: {path}")
+
+ try:
+ terminal_result = await get_terminal_tools(
+ __request__,
+ terminal_id,
+ user,
+ {
+ "__user__": __user__,
+ "__metadata__": __metadata__ or {},
+ "__request__": __request__,
+ },
+ )
+
+ if isinstance(terminal_result, tuple):
+ terminal_tools = terminal_result[0]
+ else:
+ terminal_tools = terminal_result
+
+ read_file_tool = terminal_tools.get("read_file")
+ if not read_file_tool:
+ return "The active terminal does not provide a read_file tool."
+
+ result = await read_file_tool["callable"](path=path)
+
+ if isinstance(result, tuple):
+ image_data = result[0]
+ else:
+ image_data = result
+
+ if isinstance(image_data, dict) and image_data.get("error"):
+ return f"Terminal read_file failed: {image_data['error']}"
+
+ if not isinstance(image_data, str) or not image_data.startswith(
+ "data:image/"
+ ):
+ return (
+ "Terminal read_file did not return image data. "
+ f"Received: {str(image_data)[:300]}"
+ )
+
+ except Exception as e:
+ log.exception("Vision Bridge terminal image read failed")
+ return f"Could not read image from terminal: {e}"
+
+ prompt = query.strip() or self.valves.default_query
+
+ await status(f"Looking at terminal image with {self.valves.vision_model_id}…")
+
+ try:
+ response = await generate_chat_completion(
+ __request__,
+ {
+ "model": self.valves.vision_model_id,
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": prompt},
+ {
+ "type": "image_url",
+ "image_url": {"url": image_data},
+ },
+ ],
+ }
+ ],
+ "stream": False,
+ },
+ user=user,
+ bypass_filter=True,
+ )
+
+ answer = response["choices"][0]["message"]["content"]
+
+ except Exception as e:
+ log.exception("Vision Bridge terminal image analysis failed")
+ return f"Vision analysis failed: {e}"
+
+ await status("Done", done=True)
+ return answer
+
async def _resolve(self, file_id, user, chat_id, messages):
if file_id:
return await get_image_base64_from_file_id(file_id, user=user)