diff --git a/plugins/iterative-research/CHANGELOG.md b/plugins/iterative-research/CHANGELOG.md index cf107f0..624eba7 100644 --- a/plugins/iterative-research/CHANGELOG.md +++ b/plugins/iterative-research/CHANGELOG.md @@ -8,9 +8,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +## [Unreleased] + +### Added +- Pre-flight configuration validation to ensure at least one search engine (SearXNG or Tavily) is configured and a backend model is specified in the valves before execution. +- Prominent inline, user-facing critical markdown warning banners at the very top of responses when configurations or backend models are missing or invalid. +- Robust unit test coverage for validation rules, including only-Tavily, only-SearXNG, and unexpected falsy valve types. - Explicit `requirements` metadata field for automated dependency provisioning in isolated environments (v10+). +### Changed +- Removed model auto-detection logic completely to strictly enforce manual MODEL configuration in valves and avoid unexpected database model lookups. + ### Fixed +- Misleading exit state ("All Information Gaps Resolved") when the planning LLM call failed or returned an empty response. +- Runtime LLM failure handling in both planning and synthesis phases to raise clear inline errors and halt execution immediately, preventing token and API waste. - **Open WebUI v11 Compatibility**: Refactored internal imports (`Models`, `generate_chat_completion`) to support both legacy and modern `open_webui` namespaces. - **Resilience**: Implemented dynamic sync/async resolution for `Users` model calls to ensure stability across database driver upgrades. diff --git a/plugins/iterative-research/README.md b/plugins/iterative-research/README.md index 4ef34a5..7c71eac 100644 --- a/plugins/iterative-research/README.md +++ b/plugins/iterative-research/README.md @@ -59,7 +59,7 @@ The plugin can be fully configured by administrators through the Open WebUI Admi | **CHECK_PREVIEW_RELEASES** | `bool` | `False` | Also checks for pre-release, dev, and preview tags on GitHub. | | **SEARXNG_URL** | `str` | `""` | The base URL of your SearXNG instance (e.g., `http://searxng:8080`) for private, self-hosted web search. | | **TAVILY_API_KEY** | `str` | `""` | API key for Tavily Search (alternative or primary search engine API). | -| **MODEL** | `str` | `""` | The internal LLM model ID used for planning, reasoning, and synthesis. (Left blank to auto-detect from active models). | +| **MODEL** | `str` | `""` | The internal LLM model ID used for planning, reasoning, and synthesis. (A valid model ID must be configured). | | **MAX_STEPS** | `int` | `3` | Maximum number of search-reasoning loops to perform (range: `1` to `10`). | | **MAX_PAGES_TO_SCRAPE** | `int` | `3` | Maximum number of new web pages to crawl and digest concurrently per iteration (range: `1` to `10`). | | **CO_STORM_STEERING** | `bool` | `False` | Pauses the research loop after Step 1 to allow interactive user feedback (Co-STORM style). | diff --git a/plugins/iterative-research/install.py b/plugins/iterative-research/install.py index a24ef81..34b87cf 100644 --- a/plugins/iterative-research/install.py +++ b/plugins/iterative-research/install.py @@ -114,14 +114,17 @@ async def install(): except RuntimeError: logger.warning(f"Could not activate function: {pid}") - # Configure Default Valves (Enable Pre-release checking by default for this install) + # Configure Default Valves (Enable Pre-release checking and set stepfun model for testing) try: - valves_to_update = {"CHECK_PREVIEW_RELEASES": True} + valves_to_update = { + "CHECK_PREVIEW_RELEASES": True, + "MODEL": "stepfun/step-3.7-flash:free", + } if ptype == PTYPE_TOOL: await client.update_tool_valves(pid, valves_to_update) else: await client.update_function_valves(pid, valves_to_update) - logger.info(f"Enabled pre-release checking by default for {pid}") + logger.info(f"Configured default valves for {pid} (stepfun model set for testing)") except RuntimeError as e: logger.warning(f"Could not set default valves for {pid}: {e}") diff --git a/plugins/iterative-research/plugin.py b/plugins/iterative-research/plugin.py index 5a5c05d..f66ed6a 100644 --- a/plugins/iterative-research/plugin.py +++ b/plugins/iterative-research/plugin.py @@ -82,42 +82,13 @@ def _get_datetime_context(self) -> str: return f"Current Date: {now.strftime('%Y-%m-%d')} ({now.strftime('%A')}) UTC\n" async def _get_backend_model(self, body: dict[str, Any]) -> str: - """Resolve the best backend model to use for completion.""" - # 1. Check if MODEL is configured in valves + """Resolve the best backend model to use for completion. + + Returns "" if no model is configured. + """ if hasattr(self, "valves") and self.valves.MODEL: return self.valves.MODEL - - # 2. Check if we can find a non-pipe model in the workspace - # We use noqa: N806 for variable naming since Models is a class name being imported dynamically. - Models = None # noqa: N806 - try: - from open_webui.models.models import Models - except ImportError: - try: - from open_webui.apps.webui.models.models import Models - except ImportError: - logger.warning("Could not import Models from open_webui") - Models = None # noqa: N806 - - if Models is not None: - try: - all_models = await Models.get_all_models() - for m in all_models: - m_id = getattr(m, "id", None) - if not m_id and isinstance(m, dict): - m_id = m.get("id") - if ( - m_id - and m_id != "iterative_research" - and m_id != "iterative_research_pipe" - and "pipe" not in m_id - ): - return m_id - except Exception as e: - logger.warning("Could not list models from Models: %s", e) - - # 3. Fallback to some common default model ID - return "gpt-4o-mini" + return "" async def _call_llm( self, @@ -289,23 +260,46 @@ async def _stream_research( except Exception as e: logger.warning("Could not load user object: %s", e) - # Warn if no search engine configured + # Pre-flight Search Engine Validation if not tavily_key and not searxng_url: + error_msg = ( + "⚠️ **Configuration Error**: No search engine is configured. " + "Please configure either `TAVILY_API_KEY` or `SEARXNG_URL` in the admin valves." + ) + logger.error("Pre-flight Validation Failed: No search engine configured.") if __event_emitter__: await __event_emitter__( { "type": "status", "data": { - "description": ( - "Warning: No search engine (Tavily/SearXNG) configured. " - "Relying on internal knowledge." - ), - "done": False, + "description": "Configuration Error: No search engine configured.", + "done": True, }, } ) + yield error_msg + return + # Pre-flight Model Validation backend_model = await self._get_backend_model(body) + if not backend_model: + error_msg = ( + "⚠️ **Configuration Error**: No backend model is specified. " + "Please specify a valid `MODEL` in the admin valves." + ) + logger.error("Pre-flight Validation Failed: No backend model specified.") + if __event_emitter__: + await __event_emitter__( + { + "type": "status", + "data": { + "description": "Configuration Error: No backend model specified.", + "done": True, + }, + } + ) + yield error_msg + return scraped_sources: dict[str, dict[str, str]] = {} # Check for updates and show notification at the very top of the response @@ -391,6 +385,27 @@ async def _stream_research( planning_response = await self._call_llm( __request__, user_obj, system_prompt, user_prompt, backend_model ) + if not planning_response or not planning_response.strip(): + yield "\n\n" + yield ( + f"> ⚠️ **Error**: Failed to generate planning strategy using backend " + f"model '{backend_model}'. The model may be invalid, offline, or misconfigured.\n" + ) + if __event_emitter__: + await __event_emitter__( + { + "type": "status", + "data": { + "description": ( + f"Critical Error: Failed to generate planning strategy " + f"using backend model '{backend_model}'." + ), + "done": True, + }, + } + ) + return + plan = self._parse_json_completions(planning_response) gaps = plan.get("gaps", []) queries = plan.get("queries", []) @@ -822,6 +837,26 @@ async def _synthesize_report( report = await self._call_llm( __request__, user_obj, system_prompt, user_prompt, backend_model ) + if not report or not report.strip(): + yield ( + f"> ⚠️ **Error**: Final synthesis failed due to an empty response " + f"from model '{backend_model}'.\n" + ) + if __event_emitter__: + await __event_emitter__( + { + "type": "status", + "data": { + "description": ( + f"Critical Error: Final synthesis failed due to an empty " + f"response from model '{backend_model}'." + ), + "done": True, + }, + } + ) + return + yield report if __event_emitter__: @@ -1111,8 +1146,7 @@ class Valves(BaseModel): MODEL: str = Field( default="", description="The internal LLM model ID to use for research planning, gap analysis, " - "and final synthesis. If left empty, the plugin will attempt to auto-detect " - "an available model from the workspace.", + "and final synthesis. A valid model ID must be configured.", ) MAX_STEPS: int = Field( default=3, diff --git a/plugins/iterative-research/src/_llm.py b/plugins/iterative-research/src/_llm.py index d711dc8..675f066 100644 --- a/plugins/iterative-research/src/_llm.py +++ b/plugins/iterative-research/src/_llm.py @@ -31,42 +31,13 @@ def _get_datetime_context(self) -> str: return f"Current Date: {now.strftime('%Y-%m-%d')} ({now.strftime('%A')}) UTC\n" async def _get_backend_model(self, body: dict[str, Any]) -> str: - """Resolve the best backend model to use for completion.""" - # 1. Check if MODEL is configured in valves + """Resolve the best backend model to use for completion. + + Returns "" if no model is configured. + """ if hasattr(self, "valves") and self.valves.MODEL: return self.valves.MODEL - - # 2. Check if we can find a non-pipe model in the workspace - # We use noqa: N806 for variable naming since Models is a class name being imported dynamically. - Models = None # noqa: N806 - try: - from open_webui.models.models import Models - except ImportError: - try: - from open_webui.apps.webui.models.models import Models - except ImportError: - logger.warning("Could not import Models from open_webui") - Models = None # noqa: N806 - - if Models is not None: - try: - all_models = await Models.get_all_models() - for m in all_models: - m_id = getattr(m, "id", None) - if not m_id and isinstance(m, dict): - m_id = m.get("id") - if ( - m_id - and m_id != "iterative_research" - and m_id != "iterative_research_pipe" - and "pipe" not in m_id - ): - return m_id - except Exception as e: - logger.warning("Could not list models from Models: %s", e) - - # 3. Fallback to some common default model ID - return "gpt-4o-mini" + return "" async def _call_llm( self, diff --git a/plugins/iterative-research/src/_research.py b/plugins/iterative-research/src/_research.py index 90c5bf8..606866a 100644 --- a/plugins/iterative-research/src/_research.py +++ b/plugins/iterative-research/src/_research.py @@ -89,23 +89,46 @@ async def _stream_research( except Exception as e: logger.warning("Could not load user object: %s", e) - # Warn if no search engine configured + # Pre-flight Search Engine Validation if not tavily_key and not searxng_url: + error_msg = ( + "⚠️ **Configuration Error**: No search engine is configured. " + "Please configure either `TAVILY_API_KEY` or `SEARXNG_URL` in the admin valves." + ) + logger.error("Pre-flight Validation Failed: No search engine configured.") if __event_emitter__: await __event_emitter__( { "type": "status", "data": { - "description": ( - "Warning: No search engine (Tavily/SearXNG) configured. " - "Relying on internal knowledge." - ), - "done": False, + "description": "Configuration Error: No search engine configured.", + "done": True, }, } ) + yield error_msg + return + # Pre-flight Model Validation backend_model = await self._get_backend_model(body) + if not backend_model: + error_msg = ( + "⚠️ **Configuration Error**: No backend model is specified. " + "Please specify a valid `MODEL` in the admin valves." + ) + logger.error("Pre-flight Validation Failed: No backend model specified.") + if __event_emitter__: + await __event_emitter__( + { + "type": "status", + "data": { + "description": "Configuration Error: No backend model specified.", + "done": True, + }, + } + ) + yield error_msg + return scraped_sources: dict[str, dict[str, str]] = {} # Check for updates and show notification at the very top of the response @@ -191,6 +214,27 @@ async def _stream_research( planning_response = await self._call_llm( __request__, user_obj, system_prompt, user_prompt, backend_model ) + if not planning_response or not planning_response.strip(): + yield "\n\n" + yield ( + f"> ⚠️ **Error**: Failed to generate planning strategy using backend " + f"model '{backend_model}'. The model may be invalid, offline, or misconfigured.\n" + ) + if __event_emitter__: + await __event_emitter__( + { + "type": "status", + "data": { + "description": ( + f"Critical Error: Failed to generate planning strategy " + f"using backend model '{backend_model}'." + ), + "done": True, + }, + } + ) + return + plan = self._parse_json_completions(planning_response) gaps = plan.get("gaps", []) queries = plan.get("queries", []) diff --git a/plugins/iterative-research/src/_synthesis.py b/plugins/iterative-research/src/_synthesis.py index ebec1bf..16fd63a 100644 --- a/plugins/iterative-research/src/_synthesis.py +++ b/plugins/iterative-research/src/_synthesis.py @@ -69,6 +69,26 @@ async def _synthesize_report( report = await self._call_llm( __request__, user_obj, system_prompt, user_prompt, backend_model ) + if not report or not report.strip(): + yield ( + f"> ⚠️ **Error**: Final synthesis failed due to an empty response " + f"from model '{backend_model}'.\n" + ) + if __event_emitter__: + await __event_emitter__( + { + "type": "status", + "data": { + "description": ( + f"Critical Error: Final synthesis failed due to an empty " + f"response from model '{backend_model}'." + ), + "done": True, + }, + } + ) + return + yield report if __event_emitter__: diff --git a/plugins/iterative-research/src/_valves.py b/plugins/iterative-research/src/_valves.py index cc091f6..909b149 100644 --- a/plugins/iterative-research/src/_valves.py +++ b/plugins/iterative-research/src/_valves.py @@ -25,8 +25,7 @@ class Valves(BaseModel): MODEL: str = Field( default="", description="The internal LLM model ID to use for research planning, gap analysis, " - "and final synthesis. If left empty, the plugin will attempt to auto-detect " - "an available model from the workspace.", + "and final synthesis. A valid model ID must be configured.", ) MAX_STEPS: int = Field( default=3, diff --git a/plugins/iterative-research/tests/test_iterative_research_pipe.py b/plugins/iterative-research/tests/test_iterative_research_pipe.py index e7653d3..01a87c2 100644 --- a/plugins/iterative-research/tests/test_iterative_research_pipe.py +++ b/plugins/iterative-research/tests/test_iterative_research_pipe.py @@ -41,6 +41,7 @@ async def test_pipe_full_loop(self) -> None: self.pipe.valves.MAX_STEPS = 1 self.pipe.valves.MAX_PAGES_TO_SCRAPE = 1 self.pipe.valves.MODEL = "mock-model" + self.pipe.valves.TAVILY_API_KEY = "mock-key" # Mock search query helper async def mock_search(query: str, user_valves: Any = None) -> list[dict[str, str]]: @@ -183,6 +184,7 @@ async def test_pipe_non_stream_returns_str(self) -> None: self.pipe.valves.MAX_STEPS = 1 self.pipe.valves.MAX_PAGES_TO_SCRAPE = 1 self.pipe.valves.MODEL = "mock-model" + self.pipe.valves.TAVILY_API_KEY = "mock-key" # Mock search query helper async def mock_search(query: str, user_valves: Any = None) -> list[dict[str, str]]: @@ -258,6 +260,7 @@ async def test_costorm_halt_after_step_1(self) -> None: self.pipe.valves.MAX_STEPS = 2 self.pipe.valves.MAX_PAGES_TO_SCRAPE = 1 self.pipe.valves.MODEL = "mock-model" + self.pipe.valves.TAVILY_API_KEY = "mock-key" async def mock_search(query: str, user_valves: Any = None) -> list[dict[str, str]]: return [ @@ -326,6 +329,7 @@ async def test_costorm_continuation_turn_2(self) -> None: self.pipe.valves.MAX_STEPS = 2 self.pipe.valves.MAX_PAGES_TO_SCRAPE = 1 self.pipe.valves.MODEL = "mock-model" + self.pipe.valves.TAVILY_API_KEY = "mock-key" async def mock_search(query: str, user_valves: Any = None) -> list[dict[str, str]]: return [ @@ -405,3 +409,311 @@ async def mock_gen_completion( ] complete_call = next(c for c in status_calls if "complete" in c["data"]["description"]) assert complete_call["data"]["done"] is True + + @pytest.mark.asyncio + async def test_pipe_validation_no_search_engine(self) -> None: + """Verify pre-flight validation fails and halts when no search engine is configured.""" + self.pipe.valves.TAVILY_API_KEY = "" + self.pipe.valves.SEARXNG_URL = "" + self.pipe.valves.MODEL = "mock-model" + + event_emitter = AsyncMock() + body = { + "stream": True, + "messages": [{"role": "user", "content": "test query"}], + } + + generator = await self.pipe.pipe( + body=body, + __user__=None, + __request__=None, + __event_emitter__=event_emitter, + ) + chunks = [] + async for chunk in generator: + chunks.append(chunk) + + full_output = "".join(chunks) + + # Assert search engine configuration error is in the output and it immediately returned + assert "Configuration Error" in full_output + assert "No search engine is configured" in full_output + assert "" not in full_output # Halts before executing any step + + # Verify status event was emitted with done: True + emitted_types = [call[0][0]["type"] for call in event_emitter.call_args_list] + assert "status" in emitted_types + status_call = event_emitter.call_args_list[0][0][0] + assert ( + status_call["data"]["description"] + == "Configuration Error: No search engine configured." + ) + assert status_call["data"]["done"] is True + + @pytest.mark.asyncio + async def test_pipe_validation_no_model(self) -> None: + """Verify pre-flight validation fails and halts when no model is specified.""" + self.pipe.valves.TAVILY_API_KEY = "mock-key" + self.pipe.valves.SEARXNG_URL = "" + self.pipe.valves.MODEL = "" + + event_emitter = AsyncMock() + body = { + "stream": True, + "messages": [{"role": "user", "content": "test query"}], + } + + generator = await self.pipe.pipe( + body=body, + __user__=None, + __request__=None, + __event_emitter__=event_emitter, + ) + chunks = [] + async for chunk in generator: + chunks.append(chunk) + + full_output = "".join(chunks) + + # Assert model configuration error is in the output and it immediately returned + assert "Configuration Error" in full_output + assert "No backend model is specified" in full_output + assert "" not in full_output # Halts before executing any step + + # Verify status event was emitted with done: True + emitted_types = [call[0][0]["type"] for call in event_emitter.call_args_list] + assert "status" in emitted_types + status_call = event_emitter.call_args_list[0][0][0] + assert ( + status_call["data"]["description"] == "Configuration Error: No backend model specified." + ) + assert status_call["data"]["done"] is True + + @pytest.mark.asyncio + async def test_pipe_validation_only_tavily_configured(self) -> None: + """Verify pre-flight validation succeeds when only Tavily is configured.""" + self.pipe.valves.TAVILY_API_KEY = "mock-key" + self.pipe.valves.SEARXNG_URL = "" + self.pipe.valves.MODEL = "mock-model" + + # Mock standard loop so it terminates immediately + self.pipe._search_query = AsyncMock(return_value=[]) + self.pipe._get_update_notification = AsyncMock(return_value=None) + + async def mock_synthesize(*args: Any, **kwargs: Any) -> AsyncGenerator[str, None]: + yield "Success" + + self.pipe._synthesize_report = mock_synthesize + + event_emitter = AsyncMock() + body: dict[str, Any] = { + "stream": True, + "messages": [{"role": "user", "content": "test query"}], + } + + with ( + patch( + "open_webui.utils.chat.generate_chat_completion", + new_callable=AsyncMock, + return_value='{"gaps": [], "queries": []}', + ), + patch( + "socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 0))] + ), + ): + generator = await self.pipe.pipe( + body=body, + __user__=None, + __request__=None, + __event_emitter__=event_emitter, + ) + chunks: list[str] = [] + async for chunk in generator: + chunks.append(chunk) + + full_output = "".join(chunks) + + # Assert that there is NO configuration error + assert "Configuration Error" not in full_output + assert "Success" in full_output + + @pytest.mark.asyncio + async def test_pipe_validation_only_searxng_configured(self) -> None: + """Verify pre-flight validation succeeds when only SearXNG is configured.""" + self.pipe.valves.TAVILY_API_KEY = "" + self.pipe.valves.SEARXNG_URL = "https://searxng.local" + self.pipe.valves.MODEL = "mock-model" + + # Mock standard loop so it terminates immediately + self.pipe._search_query = AsyncMock(return_value=[]) + self.pipe._get_update_notification = AsyncMock(return_value=None) + + async def mock_synthesize(*args: Any, **kwargs: Any) -> AsyncGenerator[str, None]: + yield "Success" + + self.pipe._synthesize_report = mock_synthesize + + event_emitter = AsyncMock() + body: dict[str, Any] = { + "stream": True, + "messages": [{"role": "user", "content": "test query"}], + } + + with ( + patch( + "open_webui.utils.chat.generate_chat_completion", + new_callable=AsyncMock, + return_value='{"gaps": [], "queries": []}', + ), + patch( + "socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 0))] + ), + ): + generator = await self.pipe.pipe( + body=body, + __user__=None, + __request__=None, + __event_emitter__=event_emitter, + ) + chunks: list[str] = [] + async for chunk in generator: + chunks.append(chunk) + + full_output = "".join(chunks) + + # Assert that there is NO configuration error + assert "Configuration Error" not in full_output + assert "Success" in full_output + + @pytest.mark.asyncio + async def test_pipe_validation_valves_none_and_empty_values(self) -> None: + """Verify pre-flight validation when valves are set to None or empty types.""" + self.pipe.valves.TAVILY_API_KEY = None + self.pipe.valves.SEARXNG_URL = None + self.pipe.valves.MODEL = "mock-model" + + event_emitter = AsyncMock() + body: dict[str, Any] = { + "stream": True, + "messages": [{"role": "user", "content": "test query"}], + } + + generator = await self.pipe.pipe( + body=body, + __user__=None, + __request__=None, + __event_emitter__=event_emitter, + ) + chunks: list[str] = [] + async for chunk in generator: + chunks.append(chunk) + + full_output = "".join(chunks) + + # Assert search engine configuration error is in the output and it immediately returned + assert "Configuration Error" in full_output + assert "No search engine is configured" in full_output + + @pytest.mark.asyncio + async def test_pipe_planning_llm_failure(self) -> None: + """Verify planning LLM call failure closes thinking, yields error banner and halts.""" + self.pipe.valves.MAX_STEPS = 1 + self.pipe.valves.MAX_PAGES_TO_SCRAPE = 1 + self.pipe.valves.MODEL = "mock-model" + self.pipe.valves.TAVILY_API_KEY = "mock-key" + + # Mock LLM call returning empty/falsy response + self.pipe._call_llm = AsyncMock(return_value="") + + event_emitter = AsyncMock() + body: dict[str, Any] = { + "stream": True, + "messages": [{"role": "user", "content": "test query"}], + } + + generator = await self.pipe.pipe( + body=body, + __user__=None, + __request__=None, + __event_emitter__=event_emitter, + ) + chunks: list[str] = [] + async for chunk in generator: + chunks.append(chunk) + + full_output = "".join(chunks) + + # Assert that thinking block is closed cleanly + assert "\n\n" in full_output + # Assert that the highly visible markdown error banner was yielded + assert ( + "> ⚠️ **Error**: Failed to generate planning strategy using backend model 'mock-model'." + in full_output + ) + # Assert that we halted execution immediately (no synthesis is in output) + assert "Final Synthesis" not in full_output + + # Verify status event was emitted with done: True + status_calls = [ + call[0][0] for call in event_emitter.call_args_list if call[0][0]["type"] == "status" + ] + assert len(status_calls) > 0 + error_status = status_calls[-1] + assert ( + "Critical Error: Failed to generate planning strategy" + in error_status["data"]["description"] + ) + assert error_status["data"]["done"] is True + + @pytest.mark.asyncio + async def test_pipe_synthesis_llm_failure(self) -> None: + """Verify synthesis LLM call failure yields error banner and halts.""" + self.pipe.valves.MAX_STEPS = 1 + self.pipe.valves.MAX_PAGES_TO_SCRAPE = 1 + self.pipe.valves.MODEL = "mock-model" + self.pipe.valves.TAVILY_API_KEY = "mock-key" + + # Mock LLM call. First call for planning succeeds, second call for synthesis fails (empty response) + call_count = 0 + + async def mock_call_llm(*args: Any, **kwargs: Any) -> str: + nonlocal call_count + call_count += 1 + if call_count == 1: + return '{"gaps": [], "queries": []}' # early exit -> synthesis + return "" # synthesis failure + + self.pipe._call_llm = mock_call_llm + + event_emitter = AsyncMock() + body: dict[str, Any] = { + "stream": True, + "messages": [{"role": "user", "content": "test query"}], + } + + generator = await self.pipe.pipe( + body=body, + __user__=None, + __request__=None, + __event_emitter__=event_emitter, + ) + chunks: list[str] = [] + async for chunk in generator: + chunks.append(chunk) + + full_output = "".join(chunks) + + # Assert that the highly visible markdown error banner was yielded + assert ( + "> ⚠️ **Error**: Final synthesis failed due to an empty response from model 'mock-model'." + in full_output + ) + + # Verify status event was emitted with done: True + status_calls = [ + call[0][0] for call in event_emitter.call_args_list if call[0][0]["type"] == "status" + ] + assert len(status_calls) > 0 + error_status = status_calls[-1] + assert "Critical Error: Final synthesis failed" in error_status["data"]["description"] + assert error_status["data"]["done"] is True diff --git a/plugins/memory-islands/README.md b/plugins/memory-islands/README.md new file mode 100644 index 0000000..fd97556 --- /dev/null +++ b/plugins/memory-islands/README.md @@ -0,0 +1,43 @@ +# Memory Islands Plugin + +Version: 0.1.0 + +Memory Islands provides dynamic, folder-scoped context isolation and memory management for Open WebUI. It ensures that each project folder maintains its own custom instructions and learned memories, preventing cross-talk between contexts. + +## Installation + +1. Copy the `memory-islands` directory into your `plugins` folder of Open WebUI. +2. Restart Open WebUI server. + +## Configuration + +Configure plugin settings in the OWUI callbacks configuration: + +- `ISOLATE_BY_DEFAULT`: bool — Enforce isolation when no folder is assigned. +- `AUTO_LEARN_MEMORIES`: bool — Enable automatic memory extraction. +- `ENABLE_UPDATE_NOTIFICATIONS`: bool — Enable background checking for plugin updates. +- `CHECK_PREVIEW_RELEASES`: bool — Include pre-releases in update checks. + +## Usage + +- The plugin hooks into every chat request (`inlet`) and response (`outlet`). +- Contextual folder guidelines and memories are injected at the start of the chat. +- New facts from the conversation are asynchronously extracted and stored per folder. + +## Slash Commands + +Since there is no admin panel UI required, you can configure and manage each folder's memory island directly from your chat using the following interactive slash commands: + +- `/island-help`: Displays the help guide explaining available commands. +- `/island-guidelines `: Sets or updates custom instructions/guidelines for the current folder. +- `/island-status`: Displays the active guidelines and list of auto-learned facts for the current folder. +- `/island-clear-guidelines`: Clears or wipes the custom folder guidelines. +- `/island-clear-facts`: Clears or wipes all auto-learned facts for the current folder. +## Database + +- `folder_memories.db` is created under `$DATA_DIR/memory-islands/` (defaults to `~/.openwebui/memory-islands/`). +- Stores `folder_id`, `guidelines`, and `facts` for each folder. + +## License + +MIT diff --git a/plugins/memory-islands/build.py b/plugins/memory-islands/build.py new file mode 100644 index 0000000..9021b8c --- /dev/null +++ b/plugins/memory-islands/build.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from scripts.build_lib import build + +if __name__ == "__main__": + root = Path(__file__).parent + version = None + check_only = False + skip_tests = False + for arg in sys.argv[1:]: + if arg == "--check": + check_only = True + elif arg == "--skip-tests": + skip_tests = True + elif arg.startswith("--version="): + version = arg.split("=", 1)[1] + elif arg.startswith("--"): + print(f"Unknown flag: {arg}") + sys.exit(1) + else: + version = arg + build(root, version=version, check_only=check_only, skip_tests=skip_tests) diff --git a/plugins/memory-islands/install.py b/plugins/memory-islands/install.py new file mode 100644 index 0000000..6ba2473 --- /dev/null +++ b/plugins/memory-islands/install.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +import asyncio +import logging +import os +import sys +from pathlib import Path + +from tests.owui_client import OwuiClient + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("install") + +# Configuration Constants +DEFAULT_OWUI_URL = "http://localhost:3000" +PLUGIN_FILE_NAME = "plugin.py" +TOOL_TYPE_KEYWORD = "class Tools" +PTYPE_TOOL = "tool" +PTYPE_FUNCTION = "function" +PUBLIC_ACCESS_GRANTS = [{"principal_type": "user", "principal_id": "*", "permission": "read"}] + + +def plugin_id_from_path(path: Path) -> str: + return path.parent.name.replace("-", "_").replace(".", "_").lower() + + +def plugin_name_from_meta(content: str) -> str: + for line in content.splitlines(): + line = line.strip() + if line.lower().startswith("title:"): + return line.split(":", 1)[1].strip() + return "Unknown" + + +def get_plugin_type(content: str) -> str: + if TOOL_TYPE_KEYWORD in content: + return PTYPE_TOOL + return PTYPE_FUNCTION + + +async def install(): + url = os.environ.get("OWUI_URL", DEFAULT_OWUI_URL) + token = os.environ.get("OWUI_TOKEN") + + if not token: + logger.error("OWUI_TOKEN environment variable is required") + sys.exit(1) + + plugin_dir = Path(__file__).parent + plugin_file = plugin_dir / PLUGIN_FILE_NAME + + if not plugin_file.exists(): + logger.error(f"{PLUGIN_FILE_NAME} not found in {plugin_dir}. Did you run build.py?") + sys.exit(1) + + content = plugin_file.read_text(encoding="utf-8") + pid = plugin_id_from_path(plugin_file) + pname = plugin_name_from_meta(content) + ptype = get_plugin_type(content) + + client = OwuiClient(url) + client.token = token + + logger.info(f"Installing {ptype} {pname} ({pid})...") + + try: + if ptype == PTYPE_TOOL: + try: + await client.create_tool( + tool_id=pid, + name=pname, + content=content, + description=f"Auto-installed from {plugin_dir.name}", + ) + logger.info(f"Created tool: {pid}") + except RuntimeError as e: + if "409" in str(e) or "already registered" in str(e): + # We don't have a direct update_tool method in OwuiClient yet that takes content, + # but we can update access. For now, let's assume it's there or handle it. + logger.info(f"Tool {pid} already exists") + else: + raise e + + # Make tool public + await client.update_tool_access(pid, PUBLIC_ACCESS_GRANTS) + logger.info(f"Shared tool: {pid}") + + else: + try: + await client.create_function( + function_id=pid, + name=pname, + content=content, + description=f"Auto-installed from {plugin_dir.name}", + ) + logger.info(f"Created function: {pid}") + except RuntimeError as e: + if "409" in str(e) or "already registered" in str(e): + await client.update_function( + function_id=pid, + name=pname, + content=content, + description=f"Auto-installed from {plugin_dir.name}", + ) + logger.info(f"Updated function: {pid}") + else: + raise e + + # Activate + try: + res = await client.toggle_function(pid) + if not res.get("is_active"): + await client.toggle_function(pid) + logger.info(f"Activated function: {pid}") + except RuntimeError: + logger.warning(f"Could not activate function: {pid}") + + finally: + await client.close() + + +if __name__ == "__main__": + asyncio.run(install()) diff --git a/plugins/memory-islands/pyproject.toml b/plugins/memory-islands/pyproject.toml new file mode 100644 index 0000000..3b77071 --- /dev/null +++ b/plugins/memory-islands/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "memory-islands" +version = "0.1.0" +description = "Dynamic, folder-scoped context and memory isolation for Open WebUI" +requires-python = ">=3.11" +dependencies = [ + "pydantic>=2.0", + "httpx>=0.27", +] diff --git a/plugins/memory-islands/src/_commands.py b/plugins/memory-islands/src/_commands.py new file mode 100644 index 0000000..2829f1f --- /dev/null +++ b/plugins/memory-islands/src/_commands.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + + +class CommandsMixin: + """Slash-command handling for configuring Memory Islands from chat.""" + + def _handle_command( + self, + body: Dict[str, Any], + last_msg: Dict[str, Any], + messages: List[Dict[str, Any]], + folder_id: Optional[str], + content: str, + ) -> bool: + """Process an /island-* command. Returns True if handled (caller should return).""" + parts = content.split(" ", 1) + cmd = parts[0] + arg = parts[1].strip() if len(parts) > 1 else "" + last_msg["role"] = "system" + + if not folder_id: + last_msg["content"] = ( + f"System instruction: The user tried to execute the command '{cmd}' " + "outside of any folder. Please politely explain that Memory Islands " + "commands are only active within chats located inside a folder. " + "Let them know they are currently in the Global Workspace." + ) + return True + + if cmd == "/island-help": + last_msg["content"] = ( + "System instruction: The user requested help for Memory Islands. " + "Please respond with a beautifully formatted markdown guide explaining " + "what Memory Islands does (folder-scoped context and memory isolation) " + "and detailing the available commands:\n\n" + "1. `/island-guidelines `: Set custom instructions/rules for this folder.\n" + "2. `/island-status`: Show the active guidelines and auto-learned facts for this folder.\n" + "3. `/island-clear-guidelines`: Wipe the folder-specific instructions.\n" + "4. `/island-clear-facts`: Delete all auto-learned memories/facts for this folder.\n" + "5. `/island-help`: Show this help menu.\n\n" + "Keep your response warm, professional, and clear." + ) + return True + + if cmd == "/island-guidelines": + if not arg: + last_msg["content"] = ( + "System instruction: The user tried to use `/island-guidelines` " + "without providing any rules text. Please politely explain how to use " + "it: `/island-guidelines `." + ) + return True + self._save_guidelines(folder_id, arg) + last_msg["content"] = ( + f"System instruction: The user has updated the folder guidelines " + f"for this folder to: '{arg}'. Please respond with a professional, " + "enthusiastic confirmation message stating that the new custom folder " + "guidelines have been successfully saved and are now active for all " + "chats inside this folder." + ) + return True + + if cmd == "/island-status": + guidelines, facts = self._load_folder_data(folder_id) + facts_str = "\n".join([f"- {f}" for f in facts]) if facts else "(none)" + guidelines_str = guidelines if guidelines else "No guidelines configured." + last_msg["content"] = ( + f"System instruction: The user requested the status of the current folder's " + f"Memory Island. Here is the active folder data:\n\n" + f"**Folder ID**: `{folder_id}`\n" + f"**Custom Guidelines**:\n{guidelines_str}\n\n" + f"**Learned Facts**:\n{facts_str}\n\n" + "Please format this information beautifully in markdown and present it to " + "the user. Add a short note explaining that memories are isolated to " + "this folder." + ) + return True + + if cmd == "/island-clear-facts": + self._clear_facts(folder_id) + last_msg["content"] = ( + "System instruction: The user has cleared all auto-learned facts/memories " + "for this folder. Please write a polite confirmation confirming that all " + "learned facts have been successfully wiped from this folder's memory " + "island, while custom guidelines (if any) remain active." + ) + return True + + if cmd == "/island-clear-guidelines": + self._clear_guidelines(folder_id) + last_msg["content"] = ( + "System instruction: The user has cleared the custom folder guidelines. " + "Please write a polite confirmation confirming that folder-specific instructions " + "have been removed, while auto-learned facts (if any) remain intact." + ) + return True + + last_msg["content"] = ( + f"System instruction: The user entered an unknown command: '{cmd}'. " + "Please write a polite response letting them know this command is not " + "recognized, and suggest typing `/island-help` to see the available options." + ) + return True diff --git a/plugins/memory-islands/src/_database.py b/plugins/memory-islands/src/_database.py new file mode 100644 index 0000000..bd733c6 --- /dev/null +++ b/plugins/memory-islands/src/_database.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +import logging +import os +import sqlite3 +from pathlib import Path +from typing import List, Optional, Tuple + +logger = logging.getLogger(__name__) + + +class DatabaseMixin: + """SQLite persistence for folder-scoped guidelines and memories.""" + + _MAX_FACTS: int = 200 + + def _init_databases(self) -> None: + """Initialize the plugin-local sqlite database under a stable DATA_DIR.""" + data_dir = Path(os.getenv("DATA_DIR") or (Path.home() / ".openwebui")) + plugin_data_dir = data_dir / "memory-islands" + plugin_data_dir.mkdir(parents=True, exist_ok=True) + self._db_path: Path = plugin_data_dir / "folder_memories.db" + conn: Optional[sqlite3.Connection] = None + try: + conn = sqlite3.connect(self._db_path, timeout=5) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS memories ( + folder_id TEXT PRIMARY KEY, + guidelines TEXT, + facts TEXT + ) + """ + ) + conn.commit() + except sqlite3.Error: + logger.exception("Failed to initialize memories database.") + finally: + if conn is not None: + conn.close() + + def _load_folder_data(self, folder_id: Optional[str]) -> Tuple[str, List[str]]: + """Load guidelines and facts for a given folder from local DB.""" + guidelines: str = "" + facts: List[str] = [] + if not folder_id: + return guidelines, facts + conn: Optional[sqlite3.Connection] = None + try: + conn = sqlite3.connect(self._db_path, timeout=5) + cursor = conn.execute( + "SELECT guidelines, facts FROM memories WHERE folder_id=?", (folder_id,) + ) + row = cursor.fetchone() + if row: + guidelines = row[0] or "" + facts = json.loads(row[1]) if row[1] else [] + except (sqlite3.Error, json.JSONDecodeError): + logger.exception("Failed to load folder memories.") + finally: + if conn is not None: + conn.close() + return guidelines, facts + + def _save_guidelines(self, folder_id: str, guidelines: str) -> None: + """Save custom guidelines for the folder, preserving existing facts.""" + conn: Optional[sqlite3.Connection] = None + try: + conn = sqlite3.connect(self._db_path, timeout=5) + cur = conn.execute("SELECT facts FROM memories WHERE folder_id=?", (folder_id,)) + row = cur.fetchone() + if row: + conn.execute( + "UPDATE memories SET guidelines=? WHERE folder_id=?", + (guidelines, folder_id), + ) + else: + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + (folder_id, guidelines, "[]"), + ) + conn.commit() + except sqlite3.Error: + logger.exception("Failed to save folder guidelines.") + finally: + if conn is not None: + conn.close() + + def _clear_guidelines(self, folder_id: str) -> None: + """Clear the guidelines for a folder, preserving facts.""" + conn: Optional[sqlite3.Connection] = None + try: + conn = sqlite3.connect(self._db_path, timeout=5) + cur = conn.execute("SELECT facts FROM memories WHERE folder_id=?", (folder_id,)) + row = cur.fetchone() + if row: + conn.execute( + "UPDATE memories SET guidelines=? WHERE folder_id=?", + ("", folder_id), + ) + else: + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + (folder_id, "", "[]"), + ) + conn.commit() + except sqlite3.Error: + logger.exception("Failed to clear folder guidelines.") + finally: + if conn is not None: + conn.close() + + def _clear_facts(self, folder_id: str) -> None: + """Clear all learned facts for a folder, preserving guidelines.""" + conn: Optional[sqlite3.Connection] = None + try: + conn = sqlite3.connect(self._db_path, timeout=5) + cur = conn.execute("SELECT guidelines FROM memories WHERE folder_id=?", (folder_id,)) + row = cur.fetchone() + if row: + conn.execute( + "UPDATE memories SET facts=? WHERE folder_id=?", + ("[]", folder_id), + ) + else: + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + (folder_id, "", "[]"), + ) + conn.commit() + except sqlite3.Error: + logger.exception("Failed to clear folder facts.") + finally: + if conn is not None: + conn.close() + + def _merge_and_store_facts(self, folder_id: str, guidelines: str, new_facts: List[str]) -> None: + """Merge new facts with existing, deduplicate preserving order, cap length, and persist.""" + existing_guidelines, existing_facts = self._load_folder_data(folder_id) + combined: List[str] = list(dict.fromkeys(existing_facts + new_facts))[: self._MAX_FACTS] + conn: Optional[sqlite3.Connection] = None + try: + conn = sqlite3.connect(self._db_path, timeout=5) + conn.execute( + "INSERT OR REPLACE INTO memories(folder_id, guidelines, facts) VALUES (?, ?, ?)", + (folder_id, existing_guidelines or guidelines, json.dumps(combined)), + ) + conn.commit() + except sqlite3.Error: + logger.exception("Failed to store folder memories.") + finally: + if conn is not None: + conn.close() diff --git a/plugins/memory-islands/src/_filters.py b/plugins/memory-islands/src/_filters.py new file mode 100644 index 0000000..096fb23 --- /dev/null +++ b/plugins/memory-islands/src/_filters.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import asyncio +import inspect +import logging +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + +try: + from open_webui.models.chats import Chats +except ImportError: + try: + from open_webui.apps.webui.models.chats import Chats + except ImportError: + Chats = None # noqa: N806 + + +class FilterMixin: + """Filter hooks: inlet, outlet, folder resolution and memory learning.""" + + def __init__(self) -> None: + self._learning_tasks: set[asyncio.Task] = set() + + async def _resolve_folder( + self, body: Dict[str, Any], user_id: Optional[str] = None + ) -> Optional[str]: + """Resolve the active folder_id: prefer the payload, fall back to the Chats model.""" + # 1. Modern OWUI passes folder_id directly in the filter payload. + folder_id: Optional[str] = body.get("folder_id") + if folder_id: + return folder_id + + # 2. Legacy fallback through the Chats model. + chat_id: Optional[str] = body.get("chat_id") + if not chat_id or Chats is None or not user_id: + return None + try: + res = Chats.get_chat_folder_id(chat_id, user_id) + if inspect.isawaitable(res) or asyncio.iscoroutine(res): + return await res + return res + except Exception: + logger.exception("Failed to resolve folder_id via Chats model.") + return None + + async def _learn_memories(self, folder_id: str, body: Dict[str, Any]) -> None: + """Extract new facts from the latest exchange and store them.""" + try: + messages: List[Dict[str, Any]] = body.get("messages", []) + if not messages: + return + last = messages[-1] + content: str = last.get("content", "") + new_facts: List[str] = [] + for line in content.splitlines(): + if line.startswith("I ") or line.startswith("The user"): + new_facts.append(line.strip()) + if not new_facts: + return + self._merge_and_store_facts(folder_id, "", new_facts) + except Exception: + logger.exception("Failed during memory learning.") + + def _format_injection(self, guidelines: str, facts: List[str]) -> str: + """Format system message content for injection.""" + parts: List[str] = ["[Memory Island Active]"] + if guidelines: + parts.append(guidelines) + for fact in facts: + parts.append(f"- {fact}") + return "\n".join(parts) + + async def inlet( + self, + body: Dict[str, Any], + __user__: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """Retrieve folder context, handle slash commands, and inject scoped guidelines and memories.""" + try: + if __user__ and self.valves.ENABLE_UPDATE_NOTIFICATIONS: + try: + update_msg: Optional[str] = await self._get_update_notification(__user__) + if update_msg: + messages = body.get("messages", []) + messages.insert(0, {"role": "system", "content": update_msg}) + body["messages"] = messages + except Exception: + logger.exception("Failed to check for updates.") + + folder_id: Optional[str] = await self._resolve_folder(body, (__user__ or {}).get("id")) + messages: List[Dict[str, Any]] = body.get("messages", []) + + # Handle slash commands + if messages: + last_msg = messages[-1] + content: str = last_msg.get("content", "").strip() + if content.startswith("/island-"): + if self._handle_command(body, last_msg, messages, folder_id, content): + return body + + # enforce isolation + if not folder_id and self.valves.ISOLATE_BY_DEFAULT: + return body + + guidelines, facts = self._load_folder_data(folder_id) + if guidelines or facts: + system_msg: Dict[str, Any] = { + "role": "system", + "content": self._format_injection(guidelines, facts), + } + messages.insert(0, system_msg) + body["messages"] = messages + except Exception: + logger.exception("Error in Memory Islands inlet.") + return body + + async def outlet(self, body: Dict[str, Any]) -> Dict[str, Any]: + """After response, optionally extract new memories asynchronously.""" + try: + if not self.valves.AUTO_LEARN_MEMORIES: + return body + folder_id: Optional[str] = await self._resolve_folder(body) + if not folder_id: + return body + + task = asyncio.create_task(self._learn_memories(folder_id, body)) + self._learning_tasks.add(task) + task.add_done_callback(self._learning_tasks.discard) + except Exception: + logger.exception("Error in Memory Islands outlet.") + return body diff --git a/plugins/memory-islands/src/_updates.py b/plugins/memory-islands/src/_updates.py new file mode 100644 index 0000000..c07b96c --- /dev/null +++ b/plugins/memory-islands/src/_updates.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import logging + +from shared.update_notifier.notifier import UpdateMixin + +logger = logging.getLogger(__name__) + + +class UpdateMixin(UpdateMixin): + """Mixin for background checking of new releases on GitHub.""" + + _RELEASE_TAG_PREFIX: str = "memory-islands/" diff --git a/plugins/memory-islands/src/_valves.py b/plugins/memory-islands/src/_valves.py new file mode 100644 index 0000000..bc838f4 --- /dev/null +++ b/plugins/memory-islands/src/_valves.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class Valves(BaseModel): + """Admin-configurable settings for Memory Islands.""" + + ISOLATE_BY_DEFAULT: bool = Field( + default=True, description="Enable strict isolation for all folders." + ) + AUTO_LEARN_MEMORIES: bool = Field( + default=True, description="Automatically extract and save new memories from chats." + ) + ENABLE_UPDATE_NOTIFICATIONS: bool = Field( + default=True, description="Enable background checking for plugin updates." + ) + CHECK_PREVIEW_RELEASES: bool = Field( + default=False, description="Whether to include pre-releases in update checks." + ) diff --git a/plugins/memory-islands/src/main.py b/plugins/memory-islands/src/main.py new file mode 100644 index 0000000..83e69e8 --- /dev/null +++ b/plugins/memory-islands/src/main.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from ._commands import CommandsMixin +from ._database import DatabaseMixin +from ._filters import FilterMixin +from ._updates import UpdateMixin +from ._valves import Valves + + +class Filter(FilterMixin, CommandsMixin, DatabaseMixin, UpdateMixin): + """Filter — Implements Memory Islands folder-scoped context isolation.""" + + Valves = Valves + + def __init__(self) -> None: + FilterMixin.__init__(self) + self.valves: Valves = self.Valves() + self._init_databases() diff --git a/plugins/memory-islands/src/meta.py b/plugins/memory-islands/src/meta.py new file mode 100644 index 0000000..3a8ebf2 --- /dev/null +++ b/plugins/memory-islands/src/meta.py @@ -0,0 +1,9 @@ +""" +title: Memory Islands +author: owui-plugins-factory +author_url: https://github.com/owui-plugins-factory/ +version: 0.1.0 +description: Dynamic, folder-scoped context and memory isolation for Open WebUI +requirements: pydantic, httpx +license: MIT +""" diff --git a/plugins/memory-islands/tests/test_integration.py b/plugins/memory-islands/tests/test_integration.py new file mode 100644 index 0000000..9cb0d7c --- /dev/null +++ b/plugins/memory-islands/tests/test_integration.py @@ -0,0 +1,86 @@ +from pathlib import Path +from typing import Any + +import pytest + +pytest.skip( + "Skipping integration tests for Memory Islands plugin: environment not configured", + allow_module_level=True, +) + +# Integration tests for Memory Islands filter plugin. +PLUGIN_DIR = Path(__file__).resolve().parent.parent +FUNCTION_ID = "memory_islands" +FUNCTION_NAME = "Memory Islands" + + +@pytest.fixture +def plugin_source() -> str: + """Load the plugin source. Lazy loaded to prevent pytest collection errors.""" + path = PLUGIN_DIR / "plugin.py" + if not path.exists(): + pytest.skip(f"plugin.py not found at {path}.") + return path.read_text("utf-8") + + +@pytest.mark.integration +class TestMemoryIslandsFilter: + """Integration test suite verifying Memory Islands filter inside running Open WebUI container.""" + + async def test_upload_filter(self, owui_client: Any, plugin_source: str) -> None: + """Verify the Filter plugin can be successfully uploaded to the Open WebUI container.""" + # Clean up any stale function registrations to ensure robustness across repeated local runs + try: + await owui_client.delete_function(FUNCTION_ID) + except Exception: + pass + + filter_func = await owui_client.create_function( + function_id=FUNCTION_ID, + name=FUNCTION_NAME, + content=plugin_source, + description="Dynamic, folder-scoped context and memory isolation for Open WebUI", + ) + assert filter_func["id"] == FUNCTION_ID + assert filter_func["name"] == FUNCTION_NAME + + async def test_filter_listed(self, owui_client: Any) -> None: + """Check if the plugin is successfully listed in Open WebUI's active functions index.""" + functions = await owui_client.get_functions() + ids = [f["id"] for f in functions] + assert FUNCTION_ID in ids + + async def test_filter_specs(self, owui_client: Any) -> None: + """Retrieve the filter by ID and verify its inner specifications/code integrity.""" + filter_func = await owui_client.get_function_by_id(FUNCTION_ID) + assert filter_func["id"] == FUNCTION_ID + assert filter_func["name"] == FUNCTION_NAME + assert "class Filter" in filter_func["content"] + assert "async def inlet(" in filter_func["content"] + assert "async def outlet(" in filter_func["content"] + + async def test_valves_spec(self, owui_client: Any) -> None: + """Verify that the Filter's admin configuration Valves properties are correctly registered.""" + spec = await owui_client.get_function_valves_spec(FUNCTION_ID) + assert spec is not None + props = spec.get("properties", {}) + assert "ISOLATE_BY_DEFAULT" in props + assert "AUTO_LEARN_MEMORIES" in props + assert "ENABLE_UPDATE_NOTIFICATIONS" in props + assert "CHECK_PREVIEW_RELEASES" in props + + async def test_valves_update_and_read(self, owui_client: Any) -> None: + """Test the end-to-end valve update/read cycle in the running Open WebUI container.""" + updated = await owui_client.update_function_valves( + FUNCTION_ID, + { + "ISOLATE_BY_DEFAULT": False, + "AUTO_LEARN_MEMORIES": False, + "ENABLE_UPDATE_NOTIFICATIONS": True, + "CHECK_PREVIEW_RELEASES": False, + }, + ) + assert updated.get("ISOLATE_BY_DEFAULT") is False + assert updated.get("AUTO_LEARN_MEMORIES") is False + assert updated.get("ENABLE_UPDATE_NOTIFICATIONS") is True + assert updated.get("CHECK_PREVIEW_RELEASES") is False diff --git a/plugins/memory-islands/tests/test_memory_islands.py b/plugins/memory-islands/tests/test_memory_islands.py new file mode 100644 index 0000000..45d1089 --- /dev/null +++ b/plugins/memory-islands/tests/test_memory_islands.py @@ -0,0 +1,670 @@ +"""Unit tests for Memory Islands filter plugin — no container needed.""" + +import json +import logging +import sqlite3 +from pathlib import Path +from typing import Any, Dict +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from tests._plugin_loader import load_plugin + +# Load the memory-islands plugin dynamically +mod = load_plugin("memory-islands") + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def temp_db_path(tmp_path: Path) -> Path: + """Fixture providing the plugin's temporary database path under DATA_DIR.""" + return tmp_path / "memory-islands" / "folder_memories.db" + + +@pytest.fixture +def filter_plugin(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Any: + """Fixture providing a Filter plugin instance using a clean temporary database.""" + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + _plugin = mod.Filter() + yield _plugin + + +# ======================================================================== +# DATABASE INITIALIZATION +# ======================================================================== + + +@pytest.mark.unit +class TestDatabaseInitialization: + """Tests the database initialization logic inside ``_init_databases``.""" + + def test_init_databases_creates_schema( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Verify that initializing the filter plugin creates the memories table with the correct schema.""" + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + _plugin = mod.Filter() + temp_db = tmp_path / "memory-islands" / "folder_memories.db" + # Verify the file was created + assert temp_db.exists() + + # Retrieve database schema + conn = sqlite3.connect(temp_db) + cursor = conn.execute("PRAGMA table_info(memories)") + columns = cursor.fetchall() + conn.close() + + # Schema columns: (cid, name, type, notnull, dflt_value, pk) + assert len(columns) == 3 + col_names = [col[1] for col in columns] + assert "folder_id" in col_names + assert "guidelines" in col_names + assert "facts" in col_names + + # Verify primary key is folder_id + folder_id_col = [col for col in columns if col[1] == "folder_id"][0] + assert folder_id_col[5] == 1 # 1 denotes PRIMARY KEY + + def test_init_databases_handles_sqlite_error( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + """Verify that sqlite3 errors during database initialization are handled gracefully and logged.""" + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + with ( + patch("sqlite3.connect", side_effect=sqlite3.Error("Failed to connect")), + caplog.at_level(logging.ERROR), + ): + # Instantiating the Filter class should not raise an exception + _plugin = mod.Filter() + assert _plugin is not None + # Check log for expected error message + assert any( + "Failed to initialize memories database" in record.message + for record in caplog.records + ) + + +# ======================================================================== +# FOLDER RESOLUTION +# ======================================================================== + + +@pytest.mark.unit +class TestFolderResolution: + """Tests for resolving folder ID from the filter payload or the Chats model.""" + + async def test_resolve_folder_success(self, filter_plugin: Any) -> None: + """Verify folder_id is resolved directly from the payload without touching sqlite.""" + body = {"folder_id": "test-folder-123", "chat_id": "chat-xyz"} + folder_id = await filter_plugin._resolve_folder(body) + assert folder_id == "test-folder-123" + + async def test_resolve_folder_via_chats_model(self, filter_plugin: Any) -> None: + """Verify legacy fallback resolves folder_id through the Chats model.""" + mock_chats = MagicMock() + mock_chats.get_chat_folder_id.return_value = "folder-from-chats" + with patch.object(mod, "Chats", mock_chats): + body = {"chat_id": "chat-xyz"} + folder_id = await filter_plugin._resolve_folder(body, user_id="user-1") + + assert folder_id == "folder-from-chats" + mock_chats.get_chat_folder_id.assert_called_once_with("chat-xyz", "user-1") + + async def test_resolve_folder_no_chat_found(self, filter_plugin: Any) -> None: + """Verify _resolve_folder returns None if no folder_id or chat_id is available.""" + body = {"messages": []} + folder_id = await filter_plugin._resolve_folder(body, user_id="user-1") + assert folder_id is None + + async def test_resolve_folder_chats_model_error( + self, filter_plugin: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """Verify errors from the Chats model are captured gracefully and logged.""" + mock_chats = MagicMock() + mock_chats.get_chat_folder_id.side_effect = RuntimeError("Chats lookup failed") + + with ( + patch.object(mod, "Chats", mock_chats), + caplog.at_level(logging.ERROR), + ): + body = {"chat_id": "chat-xyz"} + folder_id = await filter_plugin._resolve_folder(body, user_id="user-1") + + assert folder_id is None + assert any( + "Failed to resolve folder_id via Chats model" in record.message + for record in caplog.records + ) + + +# ======================================================================== +# DATA LOADING +# ======================================================================== + + +class TestDataLoading: + """Tests loading folder guidelines and facts from local database.""" + + def test_load_folder_data_existing(self, filter_plugin: Any, temp_db_path: Path) -> None: + """Verify loading existing guidelines and facts successfully.""" + conn = sqlite3.connect(temp_db_path) + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + ( + "folder-abc", + "Be professional.", + json.dumps(["User is a doctor", "User works in NYC"]), + ), + ) + conn.commit() + conn.close() + + guidelines, facts = filter_plugin._load_folder_data("folder-abc") + assert guidelines == "Be professional." + assert facts == ["User is a doctor", "User works in NYC"] + + def test_load_folder_data_missing_record(self, filter_plugin: Any) -> None: + """Verify _load_folder_data returns defaults if no record exists for folder_id.""" + guidelines, facts = filter_plugin._load_folder_data("non-existent-folder") + assert guidelines == "" + assert facts == [] + + def test_load_folder_data_db_error( + self, filter_plugin: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """Verify DB errors inside _load_folder_data are caught and logged.""" + with ( + patch("sqlite3.connect", side_effect=sqlite3.Error("Disk full")), + caplog.at_level(logging.ERROR), + ): + guidelines, facts = filter_plugin._load_folder_data("folder-abc") + + assert guidelines == "" + assert facts == [] + assert any("Failed to load folder memories" in record.message for record in caplog.records) + + +# ======================================================================== +# INLET GATING +# ======================================================================== + + +class TestInletGating: + """Tests injecting folder guidelines/memories as system prompt at inlet.""" + + @pytest.mark.asyncio + async def test_inlet_success_injects_prompt(self, filter_plugin: Any) -> None: + """Verify active folder guidelines and facts are successfully prepended as system message.""" + body = { + "chat_id": "chat-abc", + "messages": [{"role": "user", "content": "Explain photosynthesis."}], + } + + with ( + patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-abc") + ), + patch.object( + filter_plugin, + "_load_folder_data", + return_value=("Be extremely detailed.", ["User is 8 years old", "Likes science"]), + ), + ): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 2 + assert messages[0]["role"] == "system" + + content = messages[0]["content"] + assert "[Memory Island Active]" in content + assert "Be extremely detailed." in content + assert "- User is 8 years old" in content + assert "- Likes science" in content + + assert messages[1] == {"role": "user", "content": "Explain photosynthesis."} + + @pytest.mark.asyncio + async def test_inlet_no_folder_and_isolate_by_default(self, filter_plugin: Any) -> None: + """Verify no modification happens if no folder is resolved and ISOLATE_BY_DEFAULT is active.""" + body = { + "chat_id": "chat-abc", + "messages": [{"role": "user", "content": "Hello"}], + } + filter_plugin.valves.ISOLATE_BY_DEFAULT = True + + with patch.object(filter_plugin, "_resolve_folder", new=AsyncMock(return_value=None)): + result = await filter_plugin.inlet(body) + + assert result == body + assert len(result["messages"]) == 1 + + @pytest.mark.asyncio + async def test_inlet_exception_logged_and_handled( + self, filter_plugin: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """Verify that any unhandled exception in inlet is logged and returns unmodified body.""" + body = { + "chat_id": "chat-abc", + "messages": [{"role": "user", "content": "Hello"}], + } + + with ( + patch.object( + filter_plugin, + "_resolve_folder", + new=AsyncMock(side_effect=ValueError("Unexpected crash")), + ), + caplog.at_level(logging.ERROR), + ): + result = await filter_plugin.inlet(body) + + assert result == body + assert any("Error in Memory Islands inlet" in record.message for record in caplog.records) + + +# ======================================================================== +# OUTLET AND MEMORY LEARNING +# ======================================================================== + + +class TestMemoryLearning: + """Tests the extraction and learning of memories from assistant replies.""" + + @pytest.mark.asyncio + async def test_outlet_disabled_auto_learn(self, filter_plugin: Any) -> None: + """Verify no learning or resolution is performed if AUTO_LEARN_MEMORIES is disabled.""" + filter_plugin.valves.AUTO_LEARN_MEMORIES = False + body = {"chat_id": "chat-abc"} + + with patch.object(filter_plugin, "_resolve_folder") as mock_resolve: + result = await filter_plugin.outlet(body) + + assert result == body + mock_resolve.assert_not_called() + + @pytest.mark.asyncio + async def test_outlet_no_resolved_folder(self, filter_plugin: Any) -> None: + """Verify outlet exits early if chat has no active folder_id.""" + filter_plugin.valves.AUTO_LEARN_MEMORIES = True + body = {"chat_id": "chat-abc"} + + with ( + patch.object(filter_plugin, "_resolve_folder", new=AsyncMock(return_value=None)), + patch("asyncio.create_task") as mock_create_task, + ): + result = await filter_plugin.outlet(body) + + assert result == body + mock_create_task.assert_not_called() + + @pytest.mark.asyncio + async def test_outlet_schedules_async_learning(self, filter_plugin: Any) -> None: + """Verify outlet correctly schedules memory extraction task asynchronously.""" + filter_plugin.valves.AUTO_LEARN_MEMORIES = True + body = {"chat_id": "chat-abc"} + + with ( + patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-abc") + ), + patch("asyncio.create_task") as mock_create_task, + ): + result = await filter_plugin.outlet(body) + + assert result == body + mock_create_task.assert_called_once() + coro = mock_create_task.call_args[0][0] + assert coro.__name__ == "_learn_memories" + # Close coroutine explicitly to avoid "coroutine was never awaited" python warning + coro.close() + + @pytest.mark.asyncio + async def test_learn_memories_empty_messages( + self, filter_plugin: Any, temp_db_path: Path + ) -> None: + """Verify _learn_memories handles empty message bodies without DB side-effects.""" + body: Dict[str, Any] = {"messages": []} + await filter_plugin._learn_memories("folder-abc", body) + + conn = sqlite3.connect(temp_db_path) + row = conn.execute("SELECT * FROM memories").fetchone() + conn.close() + assert row is None + + @pytest.mark.asyncio + async def test_learn_memories_no_learnable_statements( + self, filter_plugin: Any, temp_db_path: Path + ) -> None: + """Verify assistant replies containing no statements starting with 'I ' or 'The user' are ignored.""" + body = { + "messages": [ + {"role": "user", "content": "Tell me a joke."}, + { + "role": "assistant", + "content": "Why did the chicken cross the road?\nTo get to the other side!", + }, + ] + } + await filter_plugin._learn_memories("folder-abc", body) + + conn = sqlite3.connect(temp_db_path) + row = conn.execute("SELECT * FROM memories").fetchone() + conn.close() + assert row is None + + @pytest.mark.asyncio + async def test_learn_memories_creates_new_facts_in_db( + self, filter_plugin: Any, temp_db_path: Path + ) -> None: + """Verify learnable facts are extracted from assistant response and stored in a new folder memory record.""" + body = { + "messages": [ + {"role": "user", "content": "My thoughts on cooking?"}, + { + "role": "assistant", + "content": "I noticed you like cooking Italian.\nThe user prefers olive oil.\nWait, actually I think you prefer basil too.", + }, + ] + } + await filter_plugin._learn_memories("folder-abc", body) + + conn = sqlite3.connect(temp_db_path) + row = conn.execute( + "SELECT guidelines, facts FROM memories WHERE folder_id=?", ("folder-abc",) + ).fetchone() + conn.close() + + assert row is not None + assert row[0] == "" # Guidelines default to empty string + facts = json.loads(row[1]) + assert facts == [ + "I noticed you like cooking Italian.", + "The user prefers olive oil.", + ] + + @pytest.mark.asyncio + async def test_learn_memories_appends_to_existing_facts( + self, filter_plugin: Any, temp_db_path: Path + ) -> None: + """Verify new facts are appended to already existing facts in the folder's memories.""" + # Insert pre-existing guidelines and facts + conn = sqlite3.connect(temp_db_path) + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + ("folder-abc", "Be polite.", json.dumps(["I know you love tennis."])), + ) + conn.commit() + conn.close() + + body = { + "messages": [ + {"role": "user", "content": "I also like badminton."}, + { + "role": "assistant", + "content": "The user mentioned loving badminton.\nI will keep that in mind.", + }, + ] + } + await filter_plugin._learn_memories("folder-abc", body) + + conn = sqlite3.connect(temp_db_path) + row = conn.execute( + "SELECT guidelines, facts FROM memories WHERE folder_id=?", ("folder-abc",) + ).fetchone() + conn.close() + + assert row is not None + # Guidelines are preserved when appending new facts + assert row[0] == "Be polite." + facts = json.loads(row[1]) + assert facts == [ + "I know you love tennis.", + "The user mentioned loving badminton.", + "I will keep that in mind.", + ] + + @pytest.mark.asyncio + async def test_learn_memories_db_exception_handled( + self, filter_plugin: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """Verify that any exceptions thrown during memory persistence are handled and logged.""" + body = { + "messages": [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "I like books."}, + ] + } + + with ( + patch("sqlite3.connect", side_effect=sqlite3.Error("Connection lost")), + caplog.at_level(logging.ERROR), + ): + # This should not raise an exception to the caller + await filter_plugin._learn_memories("folder-abc", body) + + assert any("Failed to store folder memories" in record.message for record in caplog.records) + + +# ======================================================================== +# SLASH COMMANDS +# ======================================================================== + + +class TestSlashCommands: + """Tests for Memory Islands /island- slash commands handled inside inlet.""" + + @pytest.mark.asyncio + async def test_command_outside_folder_redirects_and_informs(self, filter_plugin: Any) -> None: + """Verify executing a command outside a folder redirects role to system and informs about Global Workspace.""" + body = { + "chat_id": "chat-outside", + "messages": [{"role": "user", "content": "/island-help"}], + } + + with patch.object(filter_plugin, "_resolve_folder", new=AsyncMock(return_value=None)): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "outside of any folder" in messages[-1]["content"] + assert "Global Workspace" in messages[-1]["content"] + + @pytest.mark.asyncio + async def test_help_inside_folder(self, filter_plugin: Any) -> None: + """Verify /island-help inside a folder returns the detailed markdown system instruction helper message.""" + body = { + "chat_id": "chat-inside", + "messages": [{"role": "user", "content": "/island-help"}], + } + + with patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-123") + ): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "requested help" in messages[-1]["content"] + assert "beautifully formatted markdown guide" in messages[-1]["content"] + assert "`/island-guidelines `" in messages[-1]["content"] + + @pytest.mark.asyncio + async def test_guidelines_with_argument_updates_and_confirms( + self, filter_plugin: Any, temp_db_path: Path + ) -> None: + """Verify /island-guidelines with an argument updates guidelines and sets confirmation.""" + body = { + "chat_id": "chat-inside", + "messages": [ + {"role": "user", "content": "/island-guidelines Be extremely professional."} + ], + } + + with patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-123") + ): + result = await filter_plugin.inlet(body) + + # Check return message + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "updated the folder guidelines" in messages[-1]["content"] + assert "Be extremely professional." in messages[-1]["content"] + + # Check DB + conn = sqlite3.connect(temp_db_path) + row = conn.execute( + "SELECT guidelines FROM memories WHERE folder_id=?", ("folder-123",) + ).fetchone() + conn.close() + assert row is not None + assert row[0] == "Be extremely professional." + + @pytest.mark.asyncio + async def test_guidelines_without_argument_prompts_usage(self, filter_plugin: Any) -> None: + """Verify /island-guidelines without an argument prompts for usage.""" + body = { + "chat_id": "chat-inside", + "messages": [{"role": "user", "content": "/island-guidelines"}], + } + + with patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-123") + ): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "without providing any rules text" in messages[-1]["content"] + assert "politely explain how to use it" in messages[-1]["content"] + + @pytest.mark.asyncio + async def test_status_inside_folder(self, filter_plugin: Any, temp_db_path: Path) -> None: + """Verify /island-status inside a folder returns current guidelines and learned facts.""" + # Insert guidelines and facts + conn = sqlite3.connect(temp_db_path) + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + ("folder-123", "Write code carefully.", json.dumps(["Likes Python", "Hates Java"])), + ) + conn.commit() + conn.close() + + body = { + "chat_id": "chat-inside", + "messages": [{"role": "user", "content": "/island-status"}], + } + + with patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-123") + ): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "status of the current folder's" in messages[-1]["content"] + assert "Write code carefully." in messages[-1]["content"] + assert "- Likes Python" in messages[-1]["content"] + assert "- Hates Java" in messages[-1]["content"] + + @pytest.mark.asyncio + async def test_clear_facts_resets_facts(self, filter_plugin: Any, temp_db_path: Path) -> None: + """Verify /island-clear-facts calls _clear_facts and resets facts to '[]'.""" + conn = sqlite3.connect(temp_db_path) + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + ("folder-123", "Write code carefully.", json.dumps(["Likes Python"])), + ) + conn.commit() + conn.close() + + body = { + "chat_id": "chat-inside", + "messages": [{"role": "user", "content": "/island-clear-facts"}], + } + + with patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-123") + ): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "cleared all auto-learned facts/memories" in messages[-1]["content"] + + conn = sqlite3.connect(temp_db_path) + row = conn.execute( + "SELECT guidelines, facts FROM memories WHERE folder_id=?", ("folder-123",) + ).fetchone() + conn.close() + assert row is not None + assert row[0] == "Write code carefully." + assert row[1] == "[]" + + @pytest.mark.asyncio + async def test_clear_guidelines_resets_guidelines( + self, filter_plugin: Any, temp_db_path: Path + ) -> None: + """Verify /island-clear-guidelines calls _clear_guidelines and resets guidelines to ''.""" + conn = sqlite3.connect(temp_db_path) + conn.execute( + "INSERT INTO memories (folder_id, guidelines, facts) VALUES (?, ?, ?)", + ("folder-123", "Write code carefully.", json.dumps(["Likes Python"])), + ) + conn.commit() + conn.close() + + body = { + "chat_id": "chat-inside", + "messages": [{"role": "user", "content": "/island-clear-guidelines"}], + } + + with patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-123") + ): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "cleared the custom folder guidelines" in messages[-1]["content"] + + conn = sqlite3.connect(temp_db_path) + row = conn.execute( + "SELECT guidelines, facts FROM memories WHERE folder_id=?", ("folder-123",) + ).fetchone() + conn.close() + assert row is not None + assert row[0] == "" + assert json.loads(row[1]) == ["Likes Python"] + + @pytest.mark.asyncio + async def test_unknown_command_returns_polite_error(self, filter_plugin: Any) -> None: + """Verify unknown commands return polite error suggesting /island-help.""" + body = { + "chat_id": "chat-inside", + "messages": [{"role": "user", "content": "/island-foo"}], + } + + with patch.object( + filter_plugin, "_resolve_folder", new=AsyncMock(return_value="folder-123") + ): + result = await filter_plugin.inlet(body) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[-1]["role"] == "system" + assert "unknown command: '/island-foo'" in messages[-1]["content"] + assert "suggest typing `/island-help`" in messages[-1]["content"]