From bc07811601d5c07856d626dae0655adca38e629b Mon Sep 17 00:00:00 2001 From: MikeFalcon77 <3686170+MikeFalcon77@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:48:50 +0300 Subject: [PATCH] feat(mini-chat): exa.ai web search via custom function tools + robustness improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement a generic custom FunctionTool trait framework enabling mini-chat models to call external APIs (exa.ai web search, future tools) without hardcoding each tool in the agentic loop. Includes vLLM adapter fixes, graceful tool error handling, and improved response quality. - Add `domain/ports/function_tool.rs` — generic FunctionTool trait with lifecycle hooks (name, definition, system prompt guard, max calls, execute, result handling). - `StreamService::resolve_function_tools()` filters and assembles tool registry per-request. - Generic agentic-loop dispatch in `provider_task.rs` mirrors `search_knowledge` pattern: replay function_call, execute, inject function_call_output, continue loop. - Per-tool call limits + graceful degradation (return "unavailable" message instead of failing). - Add `infra/llm/providers/exa_search_tool.rs` — implement exa_search function tool. Makes HTTP POST to exa.ai via OAGW, returns web results (title, URL, highlights). - Model opt-in via `enabled_function_tools: ["exa_search"]` in model catalog. - Config: `exa_search` section with host, search_type, max results, per-message call limit. - Enhanced system-prompt guard: explicit instruction to cite sources as Markdown links, no harmony citation markers【1†…】; keep reasoning out of final answer. Signed-off-by: MikeFalcon77 <3686170+MikeFalcon77@users.noreply.github.com> --- config/mini-chat.yaml | 119 +++++- ...-mini-chat-adr-pluggable-function-tools.md | 87 +++++ gears/mini-chat/docs/DESIGN.md | 57 +++ gears/mini-chat/mini-chat-sdk/src/models.rs | 6 + gears/mini-chat/mini-chat/src/config.rs | 141 ++++++++ gears/mini-chat/mini-chat/src/domain/llm.rs | 14 + .../mini-chat/mini-chat/src/domain/models.rs | 4 + .../src/domain/ports/function_tool.rs | 75 ++++ .../mini-chat/src/domain/ports/mod.rs | 2 + .../src/domain/service/context_assembly.rs | 112 ++++++ .../mini-chat/src/domain/service/mod.rs | 5 + .../src/domain/service/stream_service/mod.rs | 178 ++++++++- .../service/stream_service/provider_task.rs | 128 ++++++- .../src/domain/service/test_helpers.rs | 1 + gears/mini-chat/mini-chat/src/gear.rs | 51 +++ .../infra/llm/providers/exa_search_tool.rs | 339 ++++++++++++++++++ .../mini-chat/src/infra/llm/providers/mod.rs | 1 + .../src/infra/llm/providers/vllm_responses.rs | 133 ++++++- .../llm/providers/vllm_responses_tests.rs | 158 +++++++- .../mini-chat/src/infra/oagw_provisioning.rs | 71 +++- .../plugins/static_model_policy/tests.rs | 1 + 21 files changed, 1659 insertions(+), 24 deletions(-) create mode 100644 gears/mini-chat/docs/ADR/0004-cpt-cf-mini-chat-adr-pluggable-function-tools.md create mode 100644 gears/mini-chat/mini-chat/src/domain/ports/function_tool.rs create mode 100644 gears/mini-chat/mini-chat/src/infra/llm/providers/exa_search_tool.rs diff --git a/config/mini-chat.yaml b/config/mini-chat.yaml index e4147e691..e0ca69354 100644 --- a/config/mini-chat.yaml +++ b/config/mini-chat.yaml @@ -105,7 +105,13 @@ gears: config: secrets: - key: "azure-openai-key" - value: "${AZURE_OPENAI_API_KEY}" + value: "${AZURE_OPENAI_API_KEY:-}" + # Local H200 vLLM (gpt-oss). Add the value yourself. + - key: "vllm-h200-key" + value: "${VLLM_H200_API_KEY}" + # exa.ai web search. Add the value yourself. + - key: "exa-api-key" + value: "${EXA_API_KEY}" file-parser: config: @@ -135,7 +141,7 @@ gears: # `upstream_alias` is optional — defaults to `host` when omitted. thread_summary_worker: enabled: true - summary_model_id: "gpt-4.1-mini" + summary_model_id: "gpt-oss-120b" compression_threshold_pct: 80 summary_system_prompt: "You are a conversation summarizer. Given a conversation (and optionally an existing summary), produce a detailed structured summary. Respond with an block (your reasoning) followed by a block (the final summary). Only the content will be stored. Do not invent information not present in the conversation." providers: @@ -152,6 +158,38 @@ gears: prefix: "" secret_ref: "azure-openai-key" + # ── Local H200 vLLM (gpt-oss via the Responses API) ─────────────── + # https://local-llm.adc.corp.acronis.com → POST /v1/responses + # (If this endpoint speaks Chat Completions instead, switch to + # kind: openai_chat_completions and api_path: /v1/chat/completions.) + vllm_h200: + kind: vllm_responses # vLLM omits the SSE `event:` field; its adapter compensates + storage_kind: openai # required; no RAG used for this model + host: "local-llm.adc.corp.acronis.com" + api_path: "/v1/responses" + supports_file_search_filters: false + auth_plugin_type: "gts.cf.core.oagw.auth_plugin.v1~cf.core.oagw.apikey.v1" + auth_config: + header: "Authorization" + prefix: "Bearer " + secret_ref: "vllm-h200-key" + + # exa.ai web-search function tool. Models opt in via + # `enabled_function_tools: ["exa_search"]` in the model catalog below. + # Egress goes through OAGW (apikey auth injects the x-api-key header). + exa_search: + enabled: true + host: "api.exa.ai" + search_type: "auto" + num_results: 10 + max_calls_per_message: 3 + max_chars: 2000 + auth_plugin_type: "gts.cf.core.oagw.auth_plugin.v1~cf.core.oagw.apikey.v1" + auth_config: + header: "x-api-key" + prefix: "" + secret_ref: "exa-api-key" + # ── Azure OpenAI example (uncomment and set your endpoint) ────────── # azure_openai: # kind: openai_responses @@ -210,8 +248,8 @@ gears: provider_model_id: "gpt-4.1" display_name: "GPT-4.1" description: "Most capable model" - provider_id: "azure_openai" - provider_display_name: "Azure OpenAI" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" icon: "" tier: Premium enabled: true @@ -272,8 +310,8 @@ gears: provider_model_id: "gpt-4.1-mini" display_name: "GPT-4.1 Mini" description: "Fast and efficient model" - provider_id: "azure_openai" - provider_display_name: "Azure OpenAI" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" icon: "" tier: Standard enabled: true @@ -336,8 +374,8 @@ gears: provider_model_id: "gpt-4.1-mini" display_name: "GPT-4.1 Mini (Tiny Context - E2E)" description: "E2E testing model with small context for thread summary trigger" - provider_id: "azure_openai" - provider_display_name: "Azure OpenAI" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" icon: "" tier: Standard enabled: true @@ -393,6 +431,71 @@ gears: is_default: false sort_order: 99 + # ── GPT-OSS 120B on local H200 (vLLM) with exa.ai web search ── + - id: "gpt-oss-120b" + provider_model_id: "h200/gpt-oss-120b" + display_name: "GPT-OSS 120B (H200)" + description: "Local gpt-oss-120b served by vLLM on the H200, with web search via exa.ai" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" + icon: "" + tier: Standard + enabled: true + system_prompt: "You are a helpful, concise assistant. Answer clearly and directly." + thread_summary_prompt: "" + multimodal_capabilities: [] + context_window: 131072 + max_output_tokens: 32768 + max_input_tokens: 131072 + input_tokens_credit_multiplier_micro: 1000000 + output_tokens_credit_multiplier_micro: 3000000 + multiplier_display: "1x" + # Custom function tools enabled for this model (resolved against the + # gear's function-tool registry; `exa_search` is configured above). + enabled_function_tools: ["exa_search"] + estimation_budgets: + bytes_per_token_conservative: 4 + fixed_overhead_tokens: 100 + safety_margin_pct: 10 + image_token_budget: 1000 + tool_surcharge_tokens: 500 + web_search_surcharge_tokens: 500 + code_interpreter_surcharge_tokens: 1000 + minimal_generation_floor: 50 + max_num_results: 5 + web_search_context_size: low + max_tool_calls: 10 + general_config: + type: "" + available_from: "1970-01-01T00:00:00Z" + max_file_size_mb: 25 + api_params: + temperature: 0.7 + top_p: 1.0 + frequency_penalty: 0.0 + presence_penalty: 0.0 + stop: [] + features: + streaming: true + structured_output: true + tool_support: + web_search: false + file_search: false + image_generation: false + code_interpreter: false + mcp: false + supported_endpoints: + chat_completions: true + responses: true + embeddings: false + image_generation: false + audio_speech_generation: false + audio_transcription: false + audio_translation: false + preference: + is_default: false + sort_order: 2 + oagw: config: proxy_timeout_secs: 60 diff --git a/gears/mini-chat/docs/ADR/0004-cpt-cf-mini-chat-adr-pluggable-function-tools.md b/gears/mini-chat/docs/ADR/0004-cpt-cf-mini-chat-adr-pluggable-function-tools.md new file mode 100644 index 000000000..a63d7b813 --- /dev/null +++ b/gears/mini-chat/docs/ADR/0004-cpt-cf-mini-chat-adr-pluggable-function-tools.md @@ -0,0 +1,87 @@ +--- +status: accepted +date: 2026-06-25 +--- +# Pluggable Function Tools via a Generic Registry (exa.ai Web Search) + +**ID**: `cpt-cf-mini-chat-adr-pluggable-function-tools` + +## Context and Problem Statement + +mini-chat already supports one custom LLM function tool — `search_knowledge` (RAG). Its +handling is hard-coded in the agentic provider loop: the loop matches the tool name literally +(`if name == "search_knowledge" { … } else { unexpected_tool_use → fail }`). Adding a second +tool (web search via exa.ai, requested for specific models) by copying that branch would +duplicate the per-tool limit, replay, and graceful-degradation logic for every future tool. + +We need a way to (a) add new function tools without touching the loop, (b) enable them +per-model from configuration, and (c) call third-party tool providers (exa.ai) safely. + +## Decision Drivers + +* Extensibility — new tools should not require new branches in the agentic loop. +* Per-model control — only models that opt in (config) should advertise a given tool. +* Egress & credential management — third-party calls must be auditable and key-managed. +* Blast radius — avoid destabilising the working `search_knowledge` RAG path. + +## Considered Options + +* **A. Generic `FunctionTool` registry** — a domain trait + a `HashMap>`; the loop dispatches `function_call`s by name. +* **B. Hard-coded branch per tool** — add an `exa_search` branch beside `search_knowledge`. +* **C. Direct HTTP from the tool** vs. **egress through OAGW** (orthogonal sub-decision). + +## Decision Outcome + +Chosen: **A (generic registry)** with **egress through OAGW**. + +* A domain port `FunctionTool { name, definition() -> LlmFunctionDef, system_prompt_guard(), + max_calls(), execute() }` decouples the loop from concrete tools. The loop keeps the existing + `search_knowledge` branch and adds **one** generic fall-through that looks the tool up in the + per-request registry, enforces `max_calls()` with the same soft-limit notice, executes, and + injects the `function_call_output`. +* Per-model enablement is a new `enabled_function_tools: Vec` on `ModelCatalogEntry` + (CCM/policy-snapshot driven, `#[serde(default)]`). `StreamService` resolves the model's list + against the registry per request. +* `exa_search` is the first registered tool. It reaches exa.ai **through OAGW** (host + apikey + auth plugin injecting `x-api-key`), reusing the existing `RagHttpClient` JSON-POST primitive — + consistent with how LLM providers and RAG egress already work + (`gears/model-registry/docs/ADR/0003-…-oagw-provider-access`). +* Token accounting reuses the existing flat `tool_surcharge_tokens`: when a model has any + enabled function tool, preflight sets `tools_enabled = true`. No per-tool surcharge for now. +* The function descriptor authoring type is a new function-only `LlmFunctionDef` + (`definition()` returns it; `assemble_context` wraps it into `LlmTool::Function`). + +### Consequences + +* Good — new tools are added by implementing `FunctionTool` and registering; the loop is closed + to modification. +* Good — per-model gating lives in the policy catalog alongside other model capabilities. +* Good — exa credentials stay in OAGW; calls are audited via the existing egress path. +* Good — `search_knowledge` is untouched, so the RAG path is not destabilised. +* Bad / accepted trade-off — two mechanisms temporarily coexist: `search_knowledge` keeps its + bespoke branch (per-tool metrics, DB increments, `search_result` blocks) while new tools use + the registry. **Follow-up (phase 2):** migrate `search_knowledge` into the registry and + collapse `LlmTool::Function { … }` into `Function(LlmFunctionDef)` so there is one tool type + and one dispatch path. +* Bad / accepted — token cost of a tool's *output* is bounded by config (`max_chars`), not + preflighted (same as `search_knowledge` today). + +### Confirmation + +* Unit tests: `assemble_context` appends the tool descriptor + guard; `resolve_function_tools_from` + filters by the model's `enabled_function_tools`; `ExaSearchTool::format_results` against a + recorded exa response. +* Code review: the agentic loop has exactly one generic dispatch branch; unknown tool names + still fail via `unexpected_tool_use`. +* E2E: a model with `enabled_function_tools: ["exa_search"]` triggers a web-search tool call; + a model without the flag does not advertise the tool. + +## Pros and Cons of the Options + +* **A. Generic registry** — Good: extensible, testable, single dispatch path. Bad: one + indirection layer; interim coexistence with the `search_knowledge` branch. +* **B. Hard-coded branch** — Good: trivial for one tool. Bad: duplicates limit/replay/ + degradation logic per tool; the loop grows with every tool. +* **C. Direct HTTP** — Good: fewer moving parts. Bad: bypasses centralised egress, credential + management, and audit; inconsistent with the rest of the stack. Rejected in favour of OAGW. diff --git a/gears/mini-chat/docs/DESIGN.md b/gears/mini-chat/docs/DESIGN.md index 350272fbb..f15e624bf 100644 --- a/gears/mini-chat/docs/DESIGN.md +++ b/gears/mini-chat/docs/DESIGN.md @@ -5027,6 +5027,53 @@ The surcharge model is **deterministic and independent of provider runtime behav - P1 does **NOT** implement proportional infrastructure cost modelling. There is no per-invocation, per-query, or per-retrieval-pass cost tracking. - Accurate backend cost accounting for tools, web search, and RAG operations is explicitly **out of scope** for P1. Future phases MAY introduce metered surcharges; any such change MUST be reflected via a new `policy_version`. +##### Custom Function Tools (Generic Registry + exa.ai Web Search) + +Beyond the provider-native tools (`file_search`, `web_search`, `code_interpreter`) and the +built-in `search_knowledge` RAG function, mini-chat supports **pluggable custom function tools** +through a generic registry. This is how `exa_search` (web search via exa.ai) is added without +new branches in the agentic loop. + +**Mechanism** + +- Domain port `FunctionTool` (`domain/ports/function_tool.rs`): + `name()`, `definition() -> LlmFunctionDef`, `system_prompt_guard() -> Option`, + `max_calls() -> u32`, `async execute(ctx, input) -> Result`. +- The gear builds a registry `HashMap>` at init and injects it into + `StreamService`. Per request, `resolve_function_tools` intersects the model's + `enabled_function_tools` with the registry, yielding the tool descriptors (appended to the LLM + request), their system-prompt guards (appended to the system instructions), and a dispatch map. +- The agentic loop in `provider_task.rs` keeps the bespoke `search_knowledge` branch and adds one + **generic** dispatch: on a `function_call` whose name is in the dispatch map, it enforces the + tool's `max_calls()` per-message limit (soft-limit notice on overflow, same graceful degradation + as `search_knowledge`), calls `execute`, and injects the result as `function_call_output`. + Unknown names still fail via `unexpected_tool_use`. + +**Per-model enablement** + +`ModelCatalogEntry.enabled_function_tools: Vec` (policy-snapshot / CCM driven) lists the +tool names a model may call, e.g. `["exa_search"]`. Models without the entry never see the tool. + +**exa_search tool** + +- Descriptor: `{ name: "exa_search", description: "Search the public web …", + parameters: { query: string (required) } }`. +- Egress through OAGW: `POST /{alias}/search` with body + `{ "query", "type": , "numResults": , "contents": { "highlights": true } }`. + The OAGW apikey auth plugin injects the `x-api-key` header — the key lives in the credstore, + never in mini-chat. Reuses the `RagHttpClient` JSON-POST primitive. +- Response `results[]{ title, url, highlights[] }` is formatted into a compact text block + (bounded by `exa_search.max_chars`) and returned as the `function_call_output`. +- Reference: . + +**Token cost.** Custom function tools reuse the existing flat `tool_surcharge_tokens`: when a +model has any enabled function tool, preflight sets `tools_enabled = true` (no per-tool +surcharge). The tool's output is bounded by `max_chars`, not separately preflighted — consistent +with `search_knowledge`. + +> See ADR `cpt-cf-mini-chat-adr-pluggable-function-tools`. Phase-2 follow-up: migrate +> `search_knowledge` into the registry and collapse `LlmTool::Function` onto `LlmFunctionDef`. + #### 5.5.7 Reserved Credits Calculation After estimation (canonical form — identical to section 5.4.1): @@ -6544,6 +6591,15 @@ Legend: | Parameter | Type | Default | Source | Notes | |-----------|------|---------|--------|-------| | `mini-chat.url_prefix` | `string` | `/mini-chat` | ConfigMap | ToolKit gear config, no CCM API equivalent | +| `mini-chat.exa_search.enabled` | `bool` | `false` | ConfigMap | Enables the `exa_search` web-search function tool | +| `mini-chat.exa_search.host` | `string` | `api.exa.ai` | ConfigMap | exa upstream host (OAGW registers it + a `POST /search` route) | +| `mini-chat.exa_search.upstream_alias` | `string?` | host | ConfigMap | OAGW alias; defaults to `host` | +| `mini-chat.exa_search.auth_plugin_type` / `auth_config` | object | — | ConfigMap | OAGW apikey auth (`header: x-api-key`, `secret_ref`); key lives in the credstore | +| `mini-chat.exa_search.search_type` | `string` | `auto` | ConfigMap | exa search type (`auto`/`fast`/`deep`/…) | +| `mini-chat.exa_search.num_results` | `integer` | `10` | ConfigMap | `numResults` per call | +| `mini-chat.exa_search.max_calls_per_message` | `integer` | `3` | ConfigMap | Soft per-message call limit (graceful degradation) | +| `mini-chat.exa_search.max_chars` | `integer` | `2000` | ConfigMap | Max characters kept per formatted result | +| `mini-chat.exa_search.guard` | `string` | built-in | ConfigMap | System-prompt guard appended when the tool is enabled | ## B.2 Policy / Models / Limits (via `mini-chat-model-policy-plugin`) @@ -6575,6 +6631,7 @@ All fields below are per-model entries inside the catalog. | `api_params.*` | object | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].general_config.api_params` | | `features.*` | object | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].general_config.features` | | `tool_support.*` | object | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].general_config.tool_support` | +| `enabled_function_tools` | `string[]` | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].enabled_function_tools` — custom function tools the model may call (e.g. `["exa_search"]`); resolved against the gear's function-tool registry, defaults to `[]` | | `sort_order` | `integer` | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].preference.sort_order` | ### B.2.3 Kill switches / emergency flags diff --git a/gears/mini-chat/mini-chat-sdk/src/models.rs b/gears/mini-chat/mini-chat-sdk/src/models.rs index 2ea5556b1..26e32bfbb 100644 --- a/gears/mini-chat/mini-chat-sdk/src/models.rs +++ b/gears/mini-chat/mini-chat-sdk/src/models.rs @@ -85,6 +85,11 @@ pub struct ModelCatalogEntry { /// Maximum tool calls the provider may make per request. #[serde(default = "default_max_tool_calls")] pub max_tool_calls: u32, + /// Names of custom function tools enabled for this model (e.g. + /// `["exa_search"]`). Resolved against the gear's function-tool registry at + /// request time; unknown names are ignored. Empty = no custom tools. + #[serde(default)] + pub enabled_function_tools: Vec, /// Full general config captured at snapshot time. pub general_config: ModelGeneralConfig, /// Tenant preference settings captured at snapshot time. @@ -387,6 +392,7 @@ mod tests { }), system_prompt: String::new(), thread_summary_prompt: String::new(), + enabled_function_tools: Vec::new(), } } diff --git a/gears/mini-chat/mini-chat/src/config.rs b/gears/mini-chat/mini-chat/src/config.rs index d849d65dc..7e153acd2 100644 --- a/gears/mini-chat/mini-chat/src/config.rs +++ b/gears/mini-chat/mini-chat/src/config.rs @@ -57,6 +57,10 @@ pub struct MiniChatConfig { /// Knowledge search (RAG) feature configuration. #[serde(default)] pub knowledge_search: KnowledgeSearchConfig, + /// Exa web-search function-tool configuration. + #[expand_vars] + #[serde(default)] + pub exa_search: ExaSearchConfig, } /// Which file/vector-store implementation to use for RAG operations. @@ -441,6 +445,7 @@ impl Default for MiniChatConfig { cleanup_worker: CleanupWorkerConfig::default(), thumbnail: ThumbnailConfig::default(), knowledge_search: KnowledgeSearchConfig::default(), + exa_search: ExaSearchConfig::default(), } } } @@ -1044,6 +1049,142 @@ impl KnowledgeSearchConfig { } } +/// Configuration for the `exa_search` web-search function tool (exa.ai). +/// +/// When `enabled` is `true`, an [`crate::domain::ports::FunctionTool`] named +/// `exa_search` is registered. Models that list `"exa_search"` in their +/// `enabled_function_tools` may call it; the agentic loop dispatches the call, +/// issues `POST /{alias}/search` through OAGW, and injects the formatted +/// results as a `function_call_output`. +/// +/// Egress goes through OAGW (host + apikey auth plugin), mirroring the LLM +/// providers — the API key is held by OAGW, never by mini-chat. +#[derive(Debug, Clone, Serialize, Deserialize, toolkit_macros::ExpandVars)] +#[serde(deny_unknown_fields)] +pub struct ExaSearchConfig { + /// Enable the `exa_search` function tool. Default: `false`. + #[serde(default)] + pub enabled: bool, + + /// Upstream hostname for exa. Default: `"api.exa.ai"`. + #[serde(default = "default_exa_host")] + pub host: String, + + /// OAGW upstream alias. When `None`, falls back to `host` (OAGW derives + /// the alias from the hostname). Set explicitly only for IP-based hosts. + #[serde(default)] + pub upstream_alias: Option, + + /// OAGW auth plugin type. Example: + /// `gts.cf.core.oagw.auth_plugin.v1~cf.core.oagw.apikey.v1`. + #[serde(default)] + pub auth_plugin_type: Option, + + /// Auth plugin config (e.g. `header=x-api-key`, `secret_ref=${EXA_API_KEY}`). + /// Values support `${VAR}` env expansion via [`config_expanded()`]. + #[expand_vars] + #[serde(default)] + pub auth_config: Option>, + + /// Exa search `type` (`auto`, `fast`, `instant`, `deep`, …). Default: `"auto"`. + #[serde(default = "default_exa_search_type")] + pub search_type: String, + + /// Number of results requested per call (`numResults`). Default: 10. + #[serde(default = "default_exa_num_results")] + pub num_results: u32, + + /// Maximum `exa_search` calls per message in the agentic loop. Default: 3. + #[serde(default = "default_exa_max_calls")] + pub max_calls_per_message: u32, + + /// Maximum characters kept per result after formatting (bounds context cost). + /// Default: 2000. + #[serde(default = "default_exa_max_chars")] + pub max_chars: usize, + + /// Guard instruction appended to the system prompt when enabled. + #[serde(default = "default_exa_guard")] + pub guard: String, +} + +impl Default for ExaSearchConfig { + fn default() -> Self { + Self { + enabled: false, + host: default_exa_host(), + upstream_alias: None, + auth_plugin_type: None, + auth_config: None, + search_type: default_exa_search_type(), + num_results: default_exa_num_results(), + max_calls_per_message: default_exa_max_calls(), + max_chars: default_exa_max_chars(), + guard: default_exa_guard(), + } + } +} + +impl ExaSearchConfig { + pub fn validate(&self) -> Result<(), String> { + if self.enabled { + if self.host.trim().is_empty() { + return Err("exa_search.host must not be empty when enabled".to_owned()); + } + if self.search_type.trim().is_empty() { + return Err("exa_search.search_type must not be empty when enabled".to_owned()); + } + } + if self.num_results == 0 { + return Err("exa_search.num_results must be > 0".to_owned()); + } + if self.max_calls_per_message == 0 { + return Err("exa_search.max_calls_per_message must be > 0".to_owned()); + } + if self.max_chars == 0 { + return Err("exa_search.max_chars must be > 0".to_owned()); + } + Ok(()) + } +} + +fn default_exa_host() -> String { + "api.exa.ai".to_owned() +} + +fn default_exa_search_type() -> String { + "auto".to_owned() +} + +const fn default_exa_num_results() -> u32 { + 10 +} + +const fn default_exa_max_calls() -> u32 { + 3 +} + +const fn default_exa_max_chars() -> usize { + 2000 +} + +fn default_exa_guard() -> String { + "You have access to exactly one web tool: exa_search, which searches the public web \ + for current information. Use it when the question depends on recent events or facts \ + that may not be in your training data. You do NOT have a browser: never call `open`, \ + `find`, `click`, or any other browsing tool. The exa_search results (titles, URLs, \ + and highlights) are all you get — answer directly from them; do not attempt to open \ + or fetch pages.\n\ + \n\ + Your final answer must contain ONLY the user-facing response. Do not include planning, \ + reasoning, or meta-commentary in it (e.g. phrases like \"Provide temperature…\", \ + \"Use the search results\", \"Cite sources\") — keep all of that out of the answer.\n\ + Cite sources as inline Markdown links using the URLs from the exa_search results, e.g. \ + [AccuWeather](https://example.com/page). Do NOT use citation markers such as 【1†…】, \ + footnotes, or bare numbers — only Markdown links." + .to_owned() +} + fn default_url_prefix() -> String { DEFAULT_URL_PREFIX.to_owned() } diff --git a/gears/mini-chat/mini-chat/src/domain/llm.rs b/gears/mini-chat/mini-chat/src/domain/llm.rs index 8e799db60..76ecf68bd 100644 --- a/gears/mini-chat/mini-chat/src/domain/llm.rs +++ b/gears/mini-chat/mini-chat/src/domain/llm.rs @@ -123,6 +123,20 @@ impl FileSearchFilter { } } +/// A provider-agnostic function-tool descriptor (name + description + JSON-schema). +/// +/// This is the function-only authoring type returned by +/// [`crate::domain::ports::FunctionTool::definition`]. `assemble_context` wraps +/// it into [`LlmTool::Function`] for the request. (Phase 2 will collapse the +/// enum variant to hold this struct directly.) +#[domain_model] +#[derive(Debug, Clone)] +pub struct LlmFunctionDef { + pub name: String, + pub description: String, + pub parameters: serde_json::Value, +} + /// A provider-agnostic tool descriptor. /// /// Each adapter maps supported tools to its wire format and silently drops diff --git a/gears/mini-chat/mini-chat/src/domain/models.rs b/gears/mini-chat/mini-chat/src/domain/models.rs index b5d668566..26046f474 100644 --- a/gears/mini-chat/mini-chat/src/domain/models.rs +++ b/gears/mini-chat/mini-chat/src/domain/models.rs @@ -176,6 +176,9 @@ pub struct ResolvedModel { pub thread_summary_prompt: String, /// Maximum output tokens for this model. pub max_output_tokens: u32, + /// Names of custom function tools enabled for this model (e.g. + /// `["exa_search"]`). Resolved against the gear's function-tool registry. + pub enabled_function_tools: Vec, } impl From<&mini_chat_sdk::ModelCatalogEntry> for ResolvedModel { @@ -202,6 +205,7 @@ impl From<&mini_chat_sdk::ModelCatalogEntry> for ResolvedModel { tool_support: e.general_config.tool_support.clone(), thread_summary_prompt: e.thread_summary_prompt.clone(), max_output_tokens: e.max_output_tokens, + enabled_function_tools: e.enabled_function_tools.clone(), } } } diff --git a/gears/mini-chat/mini-chat/src/domain/ports/function_tool.rs b/gears/mini-chat/mini-chat/src/domain/ports/function_tool.rs new file mode 100644 index 000000000..6779f8a92 --- /dev/null +++ b/gears/mini-chat/mini-chat/src/domain/ports/function_tool.rs @@ -0,0 +1,75 @@ +//! Domain port for pluggable LLM function tools. +//! +//! The [`FunctionTool`] trait decouples the agentic provider loop from any +//! specific custom-tool implementation. Each tool advertises its provider- +//! agnostic [`LlmFunctionDef`] descriptor and executes a single tool call, +//! returning the text injected back as a `function_call_output`. +//! +//! The agentic loop in `stream_service/provider_task.rs` dispatches incoming +//! `function_call`s by name through a registry of these trait objects, so a +//! new tool (e.g. web search via Exa) is added by implementing this trait and +//! registering it — no new branch in the loop. +//! +//! Concrete implementations live in `infra::llm::providers` (e.g. +//! `exa_search_tool.rs`). + +use async_trait::async_trait; +use toolkit_macros::domain_model; +use toolkit_security::SecurityContext; + +use crate::domain::llm::LlmFunctionDef; + +/// Errors from a function-tool invocation. +/// +/// The agentic loop never fails the turn on these — it injects a textual +/// fallback as the `function_call_output` so the model can keep going +/// (same graceful-degradation policy as `search_knowledge`). +#[domain_model] +#[derive(Debug, thiserror::Error)] +pub enum FunctionToolError { + /// Upstream explicitly rejected the request (4xx) or the arguments were invalid. + #[error("function tool rejected: {0}")] + Rejected(String), + /// Upstream unavailable or transient failure (5xx, timeout). + #[error("function tool unavailable: {0}")] + Unavailable(String), + /// Configuration error (missing alias, bad credentials, etc.). + #[error("function tool configuration error: {0}")] + Configuration(String), +} + +/// Port for a pluggable LLM function tool. +/// +/// Implementations are registered in a `HashMap>` +/// keyed by [`FunctionTool::name`]. Per-model enablement is resolved upstream; +/// the loop only ever sees the tools enabled for the current request. +#[async_trait] +pub trait FunctionTool: Send + Sync { + /// Tool name the model calls and the loop dispatches on. MUST equal the + /// `name` of the [`LlmFunctionDef`] returned by [`Self::definition`]. + fn name(&self) -> &str; + + /// Provider-agnostic function descriptor (name, description, JSON-schema + /// params) advertised to the model. + fn definition(&self) -> LlmFunctionDef; + + /// Guard instruction appended to the system prompt when this tool is + /// enabled (steers the model on when/how to call it). `None` = no guard. + fn system_prompt_guard(&self) -> Option { + None + } + + /// Maximum calls allowed per message before the loop injects a soft + /// limit notice instead of executing (graceful degradation). + fn max_calls(&self) -> u32; + + /// Execute a single tool call. + /// + /// `input` is the parsed arguments JSON the model produced. The returned + /// string is injected verbatim as the `function_call_output` text. + async fn execute( + &self, + ctx: SecurityContext, + input: serde_json::Value, + ) -> Result; +} diff --git a/gears/mini-chat/mini-chat/src/domain/ports/mod.rs b/gears/mini-chat/mini-chat/src/domain/ports/mod.rs index 0496c696c..9a72da822 100644 --- a/gears/mini-chat/mini-chat/src/domain/ports/mod.rs +++ b/gears/mini-chat/mini-chat/src/domain/ports/mod.rs @@ -25,10 +25,12 @@ use super::error::DomainError; pub type FileStream = Pin>> + Send>>; +pub(crate) mod function_tool; pub(crate) mod knowledge_retriever; pub(crate) mod metric_labels; pub(crate) mod metrics; +pub(crate) use function_tool::{FunctionTool, FunctionToolError}; pub(crate) use knowledge_retriever::KnowledgeRetriever; pub(crate) use metrics::MiniChatMetricsPort; diff --git a/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs b/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs index 0f2c85085..9261addb2 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs @@ -79,6 +79,13 @@ pub struct ContextInput<'a> { pub token_budget: Option, /// Provider file IDs for image attachments on the current user message. pub image_file_ids: &'a [String], + /// Extra custom function-tool descriptors (e.g. `exa_search`) enabled for + /// this request. Appended verbatim to the tool list. Each is a + /// [`LlmTool::Function`] resolved from the gear's function-tool registry. + pub extra_function_tools: Vec, + /// System-prompt guards for the enabled custom function tools, appended to + /// the system instructions (parallel to `extra_function_tools`). + pub extra_function_guards: Vec, } /// Output of context assembly — ready to feed into `LlmRequestBuilder`. @@ -219,6 +226,7 @@ pub fn assemble_context( input.file_search_guard, input.knowledge_search_enabled, input.knowledge_search_guard, + &input.extra_function_guards, ); // ── Tools ── @@ -266,6 +274,9 @@ pub fn assemble_context( }); } + // Custom function tools (e.g. exa_search) resolved per-model from the registry. + tools.extend(input.extra_function_tools.iter().cloned()); + // ── Truncation ── if let Some(ref budget) = input.token_budget { let available = compute_available_budget(budget)?; @@ -380,6 +391,7 @@ pub fn assemble_context( /// Build system instructions from base prompt + conditional guard strings. /// Returns `None` if the result would be empty. +#[allow(clippy::too_many_arguments)] fn build_system_instructions( system_prompt: &str, web_search_enabled: bool, @@ -388,6 +400,7 @@ fn build_system_instructions( file_search_guard: &str, knowledge_search_enabled: bool, knowledge_search_guard: &str, + extra_function_guards: &[String], ) -> Option { let mut parts: Vec<&str> = Vec::new(); @@ -403,6 +416,12 @@ fn build_system_instructions( if knowledge_search_enabled && !knowledge_search_guard.is_empty() { parts.push(knowledge_search_guard); } + // Guards for enabled custom function tools (e.g. exa_search). + for guard in extra_function_guards { + if !guard.is_empty() { + parts.push(guard); + } + } if parts.is_empty() { None @@ -443,6 +462,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert!(result.system_instructions.is_none()); @@ -450,6 +471,49 @@ mod tests { assert_eq!(result.messages.len(), 1); } + // Custom function tools are appended to the tool list, and their guards + // are appended to the system instructions. + #[test] + fn extra_function_tools_and_guards_are_appended() { + let exa_tool = LlmTool::Function { + name: "exa_search".to_owned(), + description: "Search the web.".to_owned(), + parameters: serde_json::json!({"type": "object"}), + }; + let result = assemble_context(&ContextInput { + system_prompt: "You are helpful.", + web_search_guard: "", + file_search_guard: "", + thread_summary: None, + recent_messages: &[], + user_message: "hello", + web_search_enabled: false, + file_search_enabled: false, + vector_store_ids: &[], + file_search_filters: None, + web_search_context_size: crate::domain::llm::WebSearchContextSize::Low, + file_search_max_num_results: 5, + code_interpreter_file_ids: vec![], + token_budget: None, + knowledge_search_enabled: false, + knowledge_search_guard: "", + image_file_ids: &[], + extra_function_tools: vec![exa_tool], + extra_function_guards: vec!["Use exa_search for fresh facts.".to_owned()], + }) + .unwrap(); + + assert!( + result + .tools + .iter() + .any(|t| matches!(t, LlmTool::Function { name, .. } if name == "exa_search")), + "exa_search tool should be present" + ); + let sys = result.system_instructions.expect("system instructions"); + assert!(sys.contains("Use exa_search for fresh facts.")); + } + // 5.7: system prompt + web_search enabled → guard appended #[test] fn system_prompt_with_web_search_guard() { @@ -471,6 +535,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -499,6 +565,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -527,6 +595,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -556,6 +626,8 @@ mod tests { knowledge_search_enabled: true, knowledge_search_guard: "Use search_knowledge for internal docs.", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -595,6 +667,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "Use search_knowledge for internal docs.", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -631,6 +705,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); // First message should be the thread summary @@ -672,6 +748,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.messages.len(), 3); // 2 recent + current @@ -703,6 +781,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); // system message skipped: 2 recent (user+assistant) + 1 current = 3 @@ -731,6 +811,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let last = result.messages.last().unwrap(); @@ -766,6 +848,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.tools.len(), 2); @@ -802,6 +886,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert!(result.tools.is_empty()); @@ -825,6 +911,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.tools.len(), 1); @@ -943,6 +1031,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -985,6 +1075,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -1037,6 +1129,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -1066,6 +1160,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }); assert!(matches!( @@ -1100,6 +1196,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -1142,6 +1240,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.tools.len(), 1); @@ -1172,6 +1272,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert!(result.tools.is_empty()); @@ -1216,6 +1318,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.messages.len(), 1); @@ -1246,6 +1350,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let msg = &result.messages[0]; @@ -1274,6 +1380,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let msg = &result.messages[0]; @@ -1302,6 +1410,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }); assert!(result.is_ok()); } @@ -1327,6 +1437,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }); assert!(matches!( result, diff --git a/gears/mini-chat/mini-chat/src/domain/service/mod.rs b/gears/mini-chat/mini-chat/src/domain/service/mod.rs index 62d70c10e..43a044fc4 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/mod.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/mod.rs @@ -210,6 +210,10 @@ impl< summary_config: crate::config::background::ThreadSummaryWorkerConfig, knowledge_search_config: crate::config::KnowledgeSearchConfig, knowledge_retriever: Option>, + function_tools: std::collections::HashMap< + String, + Arc, + >, anthropic_files_client: Option< Arc, >, @@ -287,6 +291,7 @@ impl< Arc::clone(&metrics), knowledge_search_config, knowledge_retriever, + function_tools, ), turns, reactions: ReactionService::new( diff --git a/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs b/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs index 30b9173b8..15a695c95 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs @@ -39,6 +39,42 @@ use types::{ // StreamService // ════════════════════════════════════════════════════════════════════════════ +/// Resolved custom function tools for a request: `(dispatch map keyed by name, +/// tool descriptors for the LLM request, system-prompt guards)`. +type ResolvedFunctionTools = ( + std::collections::HashMap>, + Vec, + Vec, +); + +/// Pure resolution of enabled custom function tools against a registry. +/// +/// Unknown tool names are silently ignored. Extracted as a free function so it +/// can be unit-tested without constructing a `StreamService`. +fn resolve_function_tools_from( + registry: &std::collections::HashMap>, + enabled: &[String], +) -> ResolvedFunctionTools { + let mut map = std::collections::HashMap::new(); + let mut tools = Vec::new(); + let mut guards = Vec::new(); + for name in enabled { + if let Some(tool) = registry.get(name) { + let def = tool.definition(); + tools.push(crate::domain::llm::LlmTool::Function { + name: def.name, + description: def.description, + parameters: def.parameters, + }); + if let Some(guard) = tool.system_prompt_guard() { + guards.push(guard); + } + map.insert(name.clone(), Arc::clone(tool)); + } + } + (map, tools, guards) +} + /// Service handling SSE streaming and turn orchestration. /// /// In P1 this is a stateless proxy: it builds an LLM request, streams @@ -74,6 +110,10 @@ pub struct StreamService< metrics: Arc, knowledge_search_config: crate::config::KnowledgeSearchConfig, knowledge_retriever: Option>, + /// Custom function-tool registry, keyed by tool name. Resolved per-request + /// against the model's `enabled_function_tools`. + function_tools: + std::collections::HashMap>, } impl< @@ -109,6 +149,10 @@ impl< metrics: Arc, knowledge_search_config: crate::config::KnowledgeSearchConfig, knowledge_retriever: Option>, + function_tools: std::collections::HashMap< + String, + Arc, + >, ) -> Self { Self { db, @@ -129,9 +173,17 @@ impl< metrics, knowledge_search_config, knowledge_retriever, + function_tools, } } + /// Resolve the custom function tools enabled for a model against the + /// registry. Returns `(dispatch map, tool descriptors, system-prompt guards)`. + /// Unknown tool names are silently ignored. + fn resolve_function_tools(&self, enabled: &[String]) -> ResolvedFunctionTools { + resolve_function_tools_from(&self.function_tools, enabled) + } + /// The configured channel capacity for the provider->writer mpsc channel. pub(crate) fn channel_capacity(&self) -> usize { usize::from(self.streaming_config.sse_channel_capacity) @@ -279,6 +331,9 @@ impl< .multimodal_capabilities .iter() .any(|c| c == "VISION_INPUT"); + // Resolve custom function tools enabled for this model against the registry. + let (function_tools, extra_function_tools, extra_function_guards) = + self.resolve_function_tools(&resolved_model.enabled_function_tools); let ResolvedModel { model_id: model, provider_id, @@ -613,7 +668,8 @@ impl< context_window: pf.context_window, max_output_tokens_applied: pf.max_output_tokens_applied, budgets: pf.estimation_budgets, - tools_enabled: file_search_enabled, + // Custom function tools also incur the flat tool surcharge (Q1). + tools_enabled: file_search_enabled || !function_tools.is_empty(), web_search_enabled, code_interpreter_enabled, }); @@ -641,6 +697,8 @@ impl< ci_file_ids, token_budget, &image_file_ids, + extra_function_tools, + extra_function_guards, ) .await?; @@ -686,6 +744,7 @@ impl< provider_file_id_map, anthropic_file_ids, knowledge_search: self.build_knowledge_search_params(&tenant_id_str), + function_tools, }, cancel, tx, @@ -930,6 +989,8 @@ impl< code_interpreter_file_ids: Vec, token_budget: Option, image_file_ids: &[String], + extra_function_tools: Vec, + extra_function_guards: Vec, ) -> Result< ( super::context_assembly::AssembledContext, @@ -1021,6 +1082,8 @@ impl< code_interpreter_file_ids, token_budget, image_file_ids, + extra_function_tools, + extra_function_guards, }) .map_err(|e| StreamError::ContextBudgetExceeded { required_tokens: match &e { @@ -1065,6 +1128,9 @@ impl< cancel: CancellationToken, tx: mpsc::Sender, ) -> Result, StreamError> { + // Resolve custom function tools enabled for this model against the registry. + let (function_tools, extra_function_tools, extra_function_guards) = + self.resolve_function_tools(&resolved_model.enabled_function_tools); let model = resolved_model.model_id; let provider_id = resolved_model.provider_id; let tenant_id = ctx.subject_tenant_id(); @@ -1356,7 +1422,8 @@ impl< context_window: pf.context_window, max_output_tokens_applied: pf.max_output_tokens_applied, budgets: pf.estimation_budgets, - tools_enabled: file_search_enabled, + // Custom function tools also incur the flat tool surcharge (Q1). + tools_enabled: file_search_enabled || !function_tools.is_empty(), web_search_enabled, code_interpreter_enabled, }); @@ -1381,6 +1448,8 @@ impl< ci_file_ids, token_budget, &[], // retry/edit: no new image attachments + extra_function_tools, + extra_function_guards, ) .await?; @@ -1420,6 +1489,7 @@ impl< provider_file_id_map, anthropic_file_ids, knowledge_search: self.build_knowledge_search_params(&tenant_id_str), + function_tools, }, cancel, tx, @@ -1848,6 +1918,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -1913,6 +1984,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -1971,6 +2043,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -2078,6 +2151,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel.clone(), tx, @@ -2255,6 +2329,7 @@ mod tests { metrics, crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -2305,6 +2380,7 @@ mod tests { }, thread_summary_prompt: String::new(), max_output_tokens: 16_384, + enabled_function_tools: Vec::new(), } } @@ -2998,6 +3074,7 @@ mod tests { metrics, crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -3431,6 +3508,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -3611,6 +3689,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -3790,6 +3869,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -3951,6 +4031,7 @@ mod tests { Arc::new(crate::domain::ports::metrics::NoopMetrics), crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -4173,6 +4254,7 @@ mod tests { }, thread_summary_prompt: String::new(), max_output_tokens: 16_384, + enabled_function_tools: Vec::new(), }, false, Vec::new(), @@ -4337,6 +4419,7 @@ mod tests { }, thread_summary_prompt: String::new(), max_output_tokens: 16_384, + enabled_function_tools: Vec::new(), }, false, Vec::new(), @@ -4561,6 +4644,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4620,6 +4704,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4689,6 +4774,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4745,6 +4831,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4803,6 +4890,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4872,6 +4960,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -5614,6 +5703,7 @@ mod tests { Arc::new(crate::domain::ports::metrics::NoopMetrics), crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -6280,3 +6370,87 @@ mod tests { ); } } + +#[cfg(test)] +mod function_tool_resolution_tests { + use std::sync::Arc; + + use async_trait::async_trait; + + use super::resolve_function_tools_from; + use crate::domain::llm::{LlmFunctionDef, LlmTool}; + use crate::domain::ports::{FunctionTool, FunctionToolError}; + + struct MockTool { + name: &'static str, + guard: Option<&'static str>, + } + + #[async_trait] + impl FunctionTool for MockTool { + fn name(&self) -> &str { + self.name + } + fn definition(&self) -> LlmFunctionDef { + LlmFunctionDef { + name: self.name.to_owned(), + description: "mock".to_owned(), + parameters: serde_json::json!({"type": "object"}), + } + } + fn system_prompt_guard(&self) -> Option { + self.guard.map(ToOwned::to_owned) + } + fn max_calls(&self) -> u32 { + 3 + } + async fn execute( + &self, + _ctx: toolkit_security::SecurityContext, + _input: serde_json::Value, + ) -> Result { + Ok("ok".to_owned()) + } + } + + fn registry() -> std::collections::HashMap> { + let mut m: std::collections::HashMap> = + std::collections::HashMap::new(); + m.insert( + "exa_search".to_owned(), + Arc::new(MockTool { + name: "exa_search", + guard: Some("exa guard"), + }), + ); + m + } + + #[test] + fn resolves_enabled_tool_with_def_and_guard() { + let (map, tools, guards) = + resolve_function_tools_from(®istry(), &["exa_search".to_owned()]); + assert_eq!(map.len(), 1); + assert!(map.contains_key("exa_search")); + assert!( + matches!(&tools[..], [LlmTool::Function { name, .. }] if name == "exa_search"), + "should produce one exa_search Function tool" + ); + assert_eq!(guards, vec!["exa guard".to_owned()]); + } + + #[test] + fn ignores_unknown_tool_names() { + let (map, tools, guards) = + resolve_function_tools_from(®istry(), &["nonexistent".to_owned()]); + assert!(map.is_empty()); + assert!(tools.is_empty()); + assert!(guards.is_empty()); + } + + #[test] + fn empty_enabled_resolves_to_empty() { + let (map, tools, guards) = resolve_function_tools_from(®istry(), &[]); + assert!(map.is_empty() && tools.is_empty() && guards.is_empty()); + } +} diff --git a/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs b/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs index 1288418be..e72c2da80 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs @@ -15,6 +15,7 @@ use toolkit_security::SecurityContext; use tracing::{Instrument, debug, info, warn}; use crate::domain::llm::ToolPhase; +use crate::domain::ports::FunctionTool; use crate::domain::ports::knowledge_retriever::{ KnowledgeRetriever, RetrievalRequest, RetrievedChunk, }; @@ -74,6 +75,10 @@ pub(super) struct ProviderTaskConfig { pub anthropic_file_ids: std::collections::HashMap, /// Knowledge search parameters; `None` when the feature is disabled. pub knowledge_search: Option, + /// Custom function tools enabled for this request, keyed by tool name. + /// The agentic loop dispatches `function_call`s whose name matches a key + /// here. Empty when no custom tools are enabled for the model. + pub function_tools: std::collections::HashMap>, } /// All five terminal paths (provider done, incomplete, provider error, @@ -109,6 +114,7 @@ pub(super) fn spawn_provider_task = Vec::new(); let mut knowledge_call_count: u32 = 0; + // Per-custom-tool call counts (keyed by tool name) for generic + // function-tool dispatch. Mirrors knowledge_call_count's soft-limit policy. + let mut function_tool_calls: std::collections::HashMap = + std::collections::HashMap::new(); // Hard cap on agentic-loop iterations. Without it, a model that keeps // emitting `search_knowledge` after the soft per-message limit fires @@ -158,12 +168,23 @@ pub(super) fn spawn_provider_task= tool.max_calls() { + warn!( + tool = %name, + limit = tool.max_calls(), + "function tool per-message limit reached, injecting soft limit response" + ); + raw_input_items.push(serde_json::json!({ + "type": "function_call", + "call_id": tool_use_id, + "name": name, + "arguments": raw_arguments, + })); + raw_input_items.push(serde_json::json!({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": "Tool call limit reached for this message. \ + Please answer based on the information already gathered.", + })); + continue 'agentic; + } + *count += 1; + + // Append the model's function_call item to replay history. + raw_input_items.push(serde_json::json!({ + "type": "function_call", + "call_id": tool_use_id, + "name": name, + "arguments": raw_arguments, + })); + + let exec_start = std::time::Instant::now(); + let output_text = match tool.execute(ctx.clone(), input).await { + Ok(text) => text, + Err(e) => { + // Distinct from a legitimate empty result so the model + // can tell a tool failure from a zero-hit query. + warn!(tool = %name, error = %e, "function tool execution failed"); + "Tool call failed; answer without the tool result.".to_owned() + } + }; + debug!( + tool = %name, + elapsed_ms = exec_start.elapsed().as_millis() as u64, + "function tool executed" + ); + + raw_input_items.push(serde_json::json!({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output_text, + })); + + continue 'agentic; + } + + // Unrecognised tool while custom function tools ARE enabled. + // Models like gpt-oss emit their built-in browser actions + // (`open`, `find`, `search`) that we don't provide. Don't fail + // the turn — replay the call with an "unavailable" output so the + // model answers from what it already gathered (e.g. exa_search + // results). The agentic-iteration cap bounds a model that keeps + // ignoring the notice. + if !function_tools.is_empty() { + warn!( + tool = %name, + "unsupported tool requested; injecting unavailable notice and continuing" + ); + let raw_arguments = + serde_json::to_string(&input).unwrap_or_else(|_| "{}".to_owned()); + let notice = format!( + "The tool \"{name}\" is not available. Do not call it again. \ + Answer the user directly using the information already gathered." + ); + raw_input_items.push(serde_json::json!({ + "type": "function_call", + "call_id": tool_use_id, + "name": name, + "arguments": raw_arguments, + })); + raw_input_items.push(serde_json::json!({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": notice, + })); + continue 'agentic; + } + + // No custom tools enabled — genuinely unsupported. Fail the turn. warn!(tool = %name, "unexpected ToolUse outcome; finalizing as failed"); let code = "unexpected_tool_use".to_owned(); let message = "Provider requested an unsupported function tool".to_owned(); diff --git a/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs b/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs index b2c9cd134..15d24257f 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs @@ -502,6 +502,7 @@ pub fn test_catalog_entry(params: TestCatalogEntryParams) -> ModelCatalogEntry { }), system_prompt: String::new(), thread_summary_prompt: String::new(), + enabled_function_tools: Vec::new(), } } diff --git a/gears/mini-chat/mini-chat/src/gear.rs b/gears/mini-chat/mini-chat/src/gear.rs index 1eb977e75..d9ae157f2 100644 --- a/gears/mini-chat/mini-chat/src/gear.rs +++ b/gears/mini-chat/mini-chat/src/gear.rs @@ -77,6 +77,9 @@ struct OagwDeferred { authn: Arc, client_credentials: crate::config::ClientCredentialsConfig, providers: std::collections::HashMap, + /// Exa search config — registers its own OAGW upstream + `/search` route + /// in `start()` when enabled. + exa_search: crate::config::ExaSearchConfig, } /// State needed to build + start the outbox pipeline in `start()`. @@ -166,6 +169,9 @@ impl Gear for MiniChatGear { cfg.knowledge_search .validate() .map_err(|e| anyhow::anyhow!("knowledge_search config: {e}"))?; + cfg.exa_search + .validate() + .map_err(|e| anyhow::anyhow!("exa_search config: {e}"))?; let vendor = cfg.vendor.trim().to_owned(); if vendor.is_empty() { @@ -226,6 +232,11 @@ impl Gear for MiniChatGear { } } } + // Same host-as-alias fallback for the exa upstream (OAGW derives the + // alias from the hostname; real registration is deferred to start()). + if cfg.exa_search.enabled && cfg.exa_search.upstream_alias.is_none() { + cfg.exa_search.upstream_alias = Some(cfg.exa_search.host.clone()); + } // Save a copy for deferred OAGW registration in start(). // Ignore the result: if already set, we keep the first value. @@ -234,6 +245,7 @@ impl Gear for MiniChatGear { authn: Arc::clone(&authn_client), client_credentials: cfg.client_credentials.clone(), providers: cfg.providers.clone(), + exa_search: cfg.exa_search.clone(), })); let provider_resolver = Arc::new(ProviderResolver::new(&gateway, cfg.providers)); @@ -411,6 +423,34 @@ impl Gear for MiniChatGear { anthropic_files_client: anthropic_files_client.clone(), })); + // ── Custom function-tool registry ─────────────────────────────────── + // + // Each enabled tool is registered by name; per-model enablement is + // resolved per-request in StreamService against `enabled_function_tools`. + let mut function_tools: std::collections::HashMap< + String, + Arc, + > = std::collections::HashMap::new(); + if cfg.exa_search.enabled { + let alias = cfg + .exa_search + .upstream_alias + .clone() + .unwrap_or_else(|| cfg.exa_search.host.clone()); + let exa: Arc = Arc::new( + crate::infra::llm::providers::exa_search_tool::ExaSearchTool::new( + Arc::clone(&rag_client), + alias, + cfg.exa_search.search_type.clone(), + cfg.exa_search.num_results, + cfg.exa_search.max_calls_per_message, + cfg.exa_search.max_chars, + cfg.exa_search.guard.clone(), + ), + ); + function_tools.insert(exa.name().to_owned(), exa); + } + // ── Services ──────────────────────────────────────────────────────── let services = Arc::new(AppServices::new( @@ -434,6 +474,7 @@ impl Gear for MiniChatGear { cfg.thread_summary_worker, cfg.knowledge_search, knowledge_retriever, + function_tools, anthropic_files_client, )); @@ -511,6 +552,16 @@ impl RunnableCapability for MiniChatGear { &mut providers, ) .await?; + + // Register the exa upstream + /search route when enabled. + if deferred.exa_search.enabled { + crate::infra::oagw_provisioning::register_exa_upstream( + &deferred.gateway, + &ctx, + &deferred.exa_search, + ) + .await?; + } } // Start the outbox pipeline now that OAGW upstreams are registered. diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/exa_search_tool.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/exa_search_tool.rs new file mode 100644 index 000000000..4c7e0685d --- /dev/null +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/exa_search_tool.rs @@ -0,0 +1,339 @@ +//! Exa web-search function tool (exa.ai `/search`). +//! +//! Implements [`FunctionTool`] as `exa_search`: a generic web-search tool the +//! model can call when it needs current information. Egress goes through OAGW +//! (the apikey auth plugin injects `x-api-key`), reusing [`RagHttpClient`] as a +//! generic "JSON POST through OAGW" primitive. +//! +//! Request: `POST /{alias}/search` +//! `{ "query", "type", "numResults", "contents": { "highlights": true } }` +//! Response: `{ "results": [ { "title", "url", "highlights": [..] } ] }` +//! +//! Reference: + +use std::sync::Arc; + +use async_trait::async_trait; +use toolkit_security::SecurityContext; +use tracing::warn; + +use crate::domain::llm::LlmFunctionDef; +use crate::domain::ports::{FileStorageError, FunctionTool, FunctionToolError}; +use crate::infra::llm::providers::rag_http_client::RagHttpClient; + +/// `exa.ai` `/search` response envelope (subset we consume). +#[derive(serde::Deserialize)] +struct ExaSearchResponse { + #[serde(default)] + results: Vec, +} + +#[derive(serde::Deserialize)] +struct ExaResult { + #[serde(default)] + title: Option, + #[serde(default)] + url: Option, + #[serde(default)] + highlights: Vec, +} + +/// Web search via exa.ai, dispatched by the agentic loop as `exa_search`. +pub struct ExaSearchTool { + client: Arc, + /// Pre-resolved OAGW upstream alias for exa. + upstream_alias: String, + search_type: String, + num_results: u32, + max_calls: u32, + /// Maximum characters kept per formatted result (bounds context cost). + max_chars: usize, + guard: String, +} + +impl ExaSearchTool { + #[must_use] + pub fn new( + client: Arc, + upstream_alias: String, + search_type: String, + num_results: u32, + max_calls: u32, + max_chars: usize, + guard: String, + ) -> Self { + Self { + client, + upstream_alias, + search_type, + num_results, + max_calls, + max_chars, + guard, + } + } + + /// Format the exa results into a compact text block for `function_call_output`. + fn format_results(&self, resp: &ExaSearchResponse) -> String { + use std::fmt::Write as _; + + if resp.results.is_empty() { + return "No web results found.".to_owned(); + } + let mut out = String::new(); + for (i, r) in resp.results.iter().enumerate() { + let title = r.title.as_deref().unwrap_or("(untitled)"); + let url = r.url.as_deref().unwrap_or(""); + write!(out, "[{}] {title}\n{url}\n", i + 1).ok(); + if !r.highlights.is_empty() { + out.push_str(&r.highlights.join(" ... ")); + out.push('\n'); + } + out.push('\n'); + if out.len() >= self.max_chars { + break; + } + } + out.truncate(self.max_chars); + out + } +} + +fn map_err(e: FileStorageError) -> FunctionToolError { + match e { + FileStorageError::Rejected { message, .. } => FunctionToolError::Rejected(message), + FileStorageError::Unavailable { message } => FunctionToolError::Unavailable(message), + other => FunctionToolError::Configuration(other.to_string()), + } +} + +#[async_trait] +impl FunctionTool for ExaSearchTool { + // Trait signature ties the return to `&self`; the literal is `'static` but + // the impl must match the trait, so the suggested `&'static str` is moot. + #[allow(clippy::unnecessary_literal_bound)] + fn name(&self) -> &str { + "exa_search" + } + + fn definition(&self) -> LlmFunctionDef { + LlmFunctionDef { + name: "exa_search".to_owned(), + description: "Search the public web for current information. Use this when the \ + answer depends on recent events or facts that may not be in your \ + training data." + .to_owned(), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "A focused web search query." + } + }, + "required": ["query"] + }), + } + } + + fn system_prompt_guard(&self) -> Option { + if self.guard.is_empty() { + None + } else { + Some(self.guard.clone()) + } + } + + fn max_calls(&self) -> u32 { + self.max_calls + } + + async fn execute( + &self, + ctx: SecurityContext, + input: serde_json::Value, + ) -> Result { + let query = input.get("query").and_then(|v| v.as_str()).unwrap_or_default(); + if query.is_empty() { + return Err(FunctionToolError::Rejected( + "exa_search called without a 'query'".to_owned(), + )); + } + + let uri = format!("/{}/search", self.upstream_alias); + let body = serde_json::json!({ + "query": query, + "type": self.search_type, + "numResults": self.num_results, + "contents": { "highlights": true }, + }); + + let resp: ExaSearchResponse = + self.client.json_post(ctx, &uri, &body).await.map_err(|e| { + warn!(error = %e, "exa_search request failed"); + map_err(e) + })?; + + Ok(self.format_results(&resp)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tool() -> ExaSearchTool { + ExaSearchTool::new( + Arc::new(RagHttpClient::new(Arc::new(NoopOagw))), + "api.exa.ai".to_owned(), + "auto".to_owned(), + 10, + 3, + 2000, + "guard text".to_owned(), + ) + } + + // Minimal OAGW stub (never called — format_results is pure). + struct NoopOagw; + #[async_trait] + impl oagw_sdk::ServiceGatewayClientV1 for NoopOagw { + async fn create_upstream( + &self, + _: SecurityContext, + _: oagw_sdk::CreateUpstreamRequest, + ) -> Result { + unimplemented!() + } + async fn get_upstream( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result { + unimplemented!() + } + async fn list_upstreams( + &self, + _: SecurityContext, + _: &oagw_sdk::ListQuery, + ) -> Result, toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn update_upstream( + &self, + _: SecurityContext, + _: uuid::Uuid, + _: oagw_sdk::UpdateUpstreamRequest, + ) -> Result { + unimplemented!() + } + async fn delete_upstream( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result<(), toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn create_route( + &self, + _: SecurityContext, + _: oagw_sdk::CreateRouteRequest, + ) -> Result { + unimplemented!() + } + async fn get_route( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result { + unimplemented!() + } + async fn list_routes( + &self, + _: SecurityContext, + _: Option, + _: &oagw_sdk::ListQuery, + ) -> Result, toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn update_route( + &self, + _: SecurityContext, + _: uuid::Uuid, + _: oagw_sdk::UpdateRouteRequest, + ) -> Result { + unimplemented!() + } + async fn delete_route( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result<(), toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn resolve_proxy_target( + &self, + _: SecurityContext, + _: &str, + _: &str, + _: &str, + ) -> Result<(oagw_sdk::Upstream, oagw_sdk::Route), toolkit_canonical_errors::CanonicalError> + { + unimplemented!() + } + async fn proxy_request( + &self, + _: SecurityContext, + _: http::Request, + ) -> Result, toolkit_canonical_errors::CanonicalError> + { + unimplemented!() + } + } + + #[test] + fn definition_is_exa_search() { + let d = tool().definition(); + assert_eq!(d.name, "exa_search"); + assert_eq!(tool().name(), "exa_search"); + } + + #[test] + fn guard_exposed_from_config() { + assert_eq!(tool().system_prompt_guard().as_deref(), Some("guard text")); + } + + #[test] + fn format_empty_results() { + let resp = ExaSearchResponse { results: vec![] }; + assert_eq!(tool().format_results(&resp), "No web results found."); + } + + #[test] + fn format_results_includes_title_url_highlights() { + let resp = ExaSearchResponse { + results: vec![ExaResult { + title: Some("Rust 2.0 released".to_owned()), + url: Some("https://example.com/rust".to_owned()), + highlights: vec!["A big release".to_owned()], + }], + }; + let out = tool().format_results(&resp); + assert!(out.contains("Rust 2.0 released")); + assert!(out.contains("https://example.com/rust")); + assert!(out.contains("A big release")); + } + + #[test] + fn format_results_truncates_to_max_chars() { + let big = "x".repeat(5000); + let resp = ExaSearchResponse { + results: vec![ExaResult { + title: Some(big), + url: Some("https://e.com".to_owned()), + highlights: vec![], + }], + }; + assert!(tool().format_results(&resp).len() <= 2000); + } +} diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs index d8a506326..a6bc2407a 100644 --- a/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs @@ -10,6 +10,7 @@ pub mod azure_file_storage; pub mod azure_knowledge_retriever; pub mod azure_vector_store; pub mod dispatching_storage; +pub mod exa_search_tool; pub mod openai_chat; pub mod openai_file_storage; pub mod openai_responses; diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs index eaefa06dc..29a53095d 100644 --- a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs @@ -18,13 +18,14 @@ use std::sync::Arc; use bytes::Bytes; use futures::StreamExt; use oagw_sdk::error::StreamingError; -use oagw_sdk::sse::{ServerEventsResponse, ServerEventsStream}; +use oagw_sdk::sse::{FromServerEvent, ServerEvent, ServerEventsResponse, ServerEventsStream}; use oagw_sdk::{Body, ServiceGatewayClientV1}; +use serde::Deserialize; use tokio_util::sync::CancellationToken; use toolkit_security::SecurityContext; use tracing::debug; -use crate::infra::llm::request::ContentPart as MessageContentPart; +use crate::infra::llm::request::{ContentPart as MessageContentPart, LlmTool}; use crate::infra::llm::{ ClientSseEvent, LlmProviderError, LlmRequest, NonStreaming, ProviderStream, ResponseResult, Streaming, TranslatedEvent, @@ -35,6 +36,77 @@ use super::openai_responses::{ translate_provider_event, }; +// ════════════════════════════════════════════════════════════════════════════ +// Event-type resolution +// ════════════════════════════════════════════════════════════════════════════ + +/// Minimal projection used to recover the event type from the JSON payload. +#[derive(Deserialize)] +struct TypeTag { + r#type: String, +} + +/// `response.reasoning_text.delta` payload (gpt-oss chain-of-thought). +#[derive(Deserialize)] +struct ReasoningTextDeltaData { + #[serde(default)] + delta: String, +} + +/// vLLM-specific event, parsed from the upstream SSE stream. +/// +/// Two divergences from `OpenAI`'s Responses API are handled here so the shared +/// [`ProviderEvent`]/[`OpenAiResponsesProvider`](super::OpenAiResponsesProvider) +/// path stays spec-clean: +/// +/// 1. **No `event:` field.** vLLM omits the SSE `event:` line and carries the +/// type only inside the JSON payload's `type` field. Without compensating, +/// every frame would resolve to `"message"` → [`ProviderEvent::Unknown`], +/// so no deltas would be emitted and no terminal event captured — surfacing +/// to the client as "stream ended without terminal event". +/// 2. **Dedicated reasoning events.** gpt-oss streams chain-of-thought as +/// `response.reasoning_text.delta` events (not the `…` tags +/// the [`ThinkState`] machine handles for Qwen). The shared parser doesn't +/// model these, so they are captured here and surfaced as `"reasoning"` +/// deltas. +enum VllmProviderEvent { + /// A gpt-oss reasoning delta to surface as a `"reasoning"` SSE delta. + ReasoningDelta { delta: String }, + /// Any event understood by the shared Responses parser. + Shared(ProviderEvent), +} + +impl FromServerEvent for VllmProviderEvent { + fn from_server_event(event: ServerEvent) -> Result { + // Recover the missing `event:` field from the JSON payload `type`. + let event = if event.event.is_none() { + let r#type = serde_json::from_str::(&event.data) + .ok() + .map(|t| t.r#type); + ServerEvent { + event: r#type, + ..event + } + } else { + event + }; + + // Reasoning deltas aren't modelled by the shared parser — capture the + // text here so it can be rendered in the "Thinking" panel. + if event.event.as_deref() == Some("response.reasoning_text.delta") { + let data: ReasoningTextDeltaData = + serde_json::from_str(&event.data).map_err(|e| { + StreamingError::ServerEventsParse { + detail: format!("failed to parse reasoning delta: {e}"), + } + })?; + return Ok(VllmProviderEvent::ReasoningDelta { delta: data.delta }); + } + + ProviderEvent::from_server_event(event).map(VllmProviderEvent::Shared) + } +} + // ════════════════════════════════════════════════════════════════════════════ // Think-tag state machine // ════════════════════════════════════════════════════════════════════════════ @@ -232,7 +304,9 @@ fn strip_think_tags(text: &str) -> String { /// Compared to the `OpenAI` variant, this: /// - Uses plain string `content` for assistant messages (vLLM rejects the /// `output_text` array format). -/// - Omits tool definitions (`file_search`, `web_search`, `code_interpreter`). +/// - Serializes function tools (`LlmTool::Function`) so the model can call +/// custom tools like `exa_search`. Provider-hosted tools (`file_search`, +/// `web_search`, `code_interpreter`) are omitted — vLLM doesn't host them. /// - Omits `metadata` and `max_tool_calls`. fn build_request_body(request: &LlmRequest, stream: bool) -> serde_json::Value { let mut body = serde_json::json!({ @@ -294,8 +368,17 @@ fn build_request_body(request: &LlmRequest, stream: bool) -> serde_json::V }) .collect(); - if !input.is_empty() { - body["input"] = serde_json::Value::Array(input); + // Append raw input items (function_call + function_call_output) so the + // agentic loop can replay prior tool calls back to the model. + let combined_input = if request.raw_input_items.is_empty() { + input + } else { + let mut combined = input; + combined.extend(request.raw_input_items.iter().cloned()); + combined + }; + if !combined_input.is_empty() { + body["input"] = serde_json::Value::Array(combined_input); } if let Some(ref instructions) = request.system_instructions { @@ -333,6 +416,30 @@ fn build_request_body(request: &LlmRequest, stream: bool) -> serde_json::V } } + // Serialize function tools so the model can call them (e.g. `exa_search`). + // Provider-hosted tools (file/web/code) are skipped — vLLM doesn't host + // them, and sending unknown tool types risks an upstream error. + let tools: Vec = request + .tools + .iter() + .filter_map(|tool| match tool { + LlmTool::Function { + name, + description, + parameters, + } => Some(serde_json::json!({ + "type": "function", + "name": name, + "description": description, + "parameters": parameters + })), + _ => None, + }) + .collect(); + if !tools.is_empty() { + body["tools"] = serde_json::Value::Array(tools); + } + // Adapter-specific extras outside the typed surface. if let Some(ref extra) = request.additional_params && let (Some(body_obj), Some(extra_obj)) = (body.as_object_mut(), extra.as_object()) @@ -402,7 +509,7 @@ impl crate::infra::llm::LlmProvider for VllmResponsesProvider { let response = self.gateway.proxy_request(ctx, http_request).await?; - match ServerEventsStream::from_response::(response) { + match ServerEventsStream::from_response::(response) { ServerEventsResponse::Events(event_stream) => { // Scan state: (accumulated_text, think_state_machine). let translated = event_stream @@ -411,7 +518,19 @@ impl crate::infra::llm::LlmProvider for VllmResponsesProvider { |(accumulated, think), result| { let output: Vec> = match result { - Ok(event) => { + Ok(VllmProviderEvent::ReasoningDelta { delta }) => { + // Surfaced as a reasoning delta; not added to + // `accumulated` (which holds visible text only). + if delta.is_empty() { + vec![] + } else { + vec![Ok(TranslatedEvent::Sse(ClientSseEvent::Delta { + r#type: "reasoning", + content: delta, + }))] + } + } + Ok(VllmProviderEvent::Shared(event)) => { if let ProviderEvent::ResponseOutputTextDelta { ref delta } = event { diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs index 30b67f318..52114d8a4 100644 --- a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs @@ -174,9 +174,11 @@ fn assistant_content_is_plain_string() { } #[test] -fn tools_are_omitted_even_when_set() { +fn provider_hosted_tools_are_omitted() { use crate::domain::llm::{LlmTool, WebSearchContextSize}; + // vLLM doesn't host file_search/web_search/code_interpreter — these must + // be dropped so the upstream doesn't reject unknown tool types. let request = llm_request("test-model") .message(LlmMessage::user("Search")) .tool(LlmTool::WebSearch { @@ -190,9 +192,73 @@ fn tools_are_omitted_even_when_set() { .build_streaming(); let body = build_request_body(&request, true); + // No function tools present → `tools` omitted entirely. assert!(body.get("tools").is_none()); } +#[test] +fn function_tools_are_serialized() { + use crate::domain::llm::LlmTool; + + // Regression guard: function tools (e.g. `exa_search`) MUST reach the + // model, otherwise it can never call them. + let request = llm_request("test-model") + .message(LlmMessage::user("weather in Istanbul")) + .tool(LlmTool::Function { + name: "exa_search".into(), + description: "Search the web.".into(), + parameters: serde_json::json!({ + "type": "object", + "properties": { "query": { "type": "string" } }, + "required": ["query"] + }), + }) + // Provider-hosted tool alongside — must be filtered out. + .tool(LlmTool::WebSearch { + search_context_size: crate::domain::llm::WebSearchContextSize::Low, + }) + .build_streaming(); + + let body = build_request_body(&request, true); + let tools = body["tools"].as_array().expect("tools array present"); + assert_eq!(tools.len(), 1, "only the function tool should be serialized"); + assert_eq!(tools[0]["type"], "function"); + assert_eq!(tools[0]["name"], "exa_search"); + assert_eq!(tools[0]["description"], "Search the web."); + assert_eq!(tools[0]["parameters"]["required"][0], "query"); +} + +#[test] +fn raw_input_items_are_appended_to_input() { + // Agentic-loop replay: function_call / function_call_output items must be + // forwarded so the model sees its prior tool call and the result. + let request = llm_request("test-model") + .message(LlmMessage::user("weather in Istanbul")) + .raw_input_items(vec![ + serde_json::json!({ + "type": "function_call", + "name": "exa_search", + "arguments": "{\"query\":\"weather Istanbul\"}", + "call_id": "call_1" + }), + serde_json::json!({ + "type": "function_call_output", + "call_id": "call_1", + "output": "Sunny, 28°C" + }), + ]) + .build_streaming(); + + let body = build_request_body(&request, true); + let input = body["input"].as_array().unwrap(); + // 1 user message + 2 replay items. + assert_eq!(input.len(), 3); + assert_eq!(input[1]["type"], "function_call"); + assert_eq!(input[1]["name"], "exa_search"); + assert_eq!(input[2]["type"], "function_call_output"); + assert_eq!(input[2]["output"], "Sunny, 28°C"); +} + #[test] fn metadata_and_max_tool_calls_omitted_even_when_set() { use crate::infra::llm::request::{RequestMetadata, RequestType}; @@ -244,3 +310,93 @@ fn additional_params_are_merged() { assert_eq!(body["temperature"], 0.5); assert_eq!(body["top_p"], 0.9); } + +// ── VllmProviderEvent: event type carried in JSON `type` ────────────── +// vLLM omits the SSE `event:` line and puts the type only inside the JSON +// payload. The wrapper must recover it so events are recognised; otherwise +// the stream ends without a terminal event. + +#[test] +fn vllm_event_resolves_text_delta_from_payload_type() { + let event = ServerEvent { + event: None, + data: r#"{"type":"response.output_text.delta","delta":"Hello","item_id":"msg_1","sequence_number":4}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!( + matches!(result, ProviderEvent::ResponseOutputTextDelta { delta } if delta == "Hello"), + "payload `type` should be honoured when the SSE `event:` field is absent" + ); +} + +#[test] +fn vllm_event_resolves_completed_from_payload_type() { + let event = ServerEvent { + event: None, + data: r#"{"type":"response.completed","sequence_number":10,"response":{"id":"resp_1","output":[],"usage":{"input_tokens":1,"output_tokens":1}}}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!( + matches!(result, ProviderEvent::ResponseCompleted { response } if response.id == "resp_1"), + "terminal event must be recognised from the JSON `type` field" + ); +} + +#[test] +fn vllm_event_prefers_sse_event_field_when_present() { + // If the upstream does send `event:` (OpenAI shape), it wins. + let event = ServerEvent { + event: Some("response.output_text.delta".to_string()), + data: r#"{"type":"response.completed","delta":"Hi"}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!(matches!(result, ProviderEvent::ResponseOutputTextDelta { delta } if delta == "Hi")); +} + +#[test] +fn vllm_event_captures_reasoning_delta() { + // gpt-oss streams chain-of-thought as `response.reasoning_text.delta`. + let event = ServerEvent { + event: None, + data: r#"{"type":"response.reasoning_text.delta","content_index":0,"delta":"thinking…","item_id":"msg_1"}"#.to_string(), + id: None, + retry: None, + }; + let result = VllmProviderEvent::from_server_event(event).unwrap(); + assert!( + matches!(result, VllmProviderEvent::ReasoningDelta { delta } if delta == "thinking…"), + "reasoning deltas must be captured as a dedicated variant" + ); +} + +#[test] +fn vllm_event_unmodelled_lifecycle_resolves_to_unknown() { + // A lifecycle event vLLM emits but the adapter doesn't model must parse + // cleanly as Unknown, not error the stream. + let event = ServerEvent { + event: None, + data: r#"{"type":"response.reasoning_part.added","content_index":0}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!(matches!(result, ProviderEvent::Unknown { .. })); +} diff --git a/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs b/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs index 44dd40651..86543b268 100644 --- a/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs +++ b/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use oagw_sdk::ServiceGatewayClientV1; use tracing::{info, warn}; -use crate::config::ProviderEntry; +use crate::config::{ExaSearchConfig, ProviderEntry}; /// Register OAGW upstreams and routes for each configured provider. /// @@ -73,6 +73,75 @@ pub async fn register_oagw_upstreams( Ok(()) } +/// Register the OAGW upstream + `POST /search` route for the exa web-search tool. +/// +/// Mirrors [`create_upstream`]/[`register_route`] but for a standalone service +/// (no `ProviderEntry`): host + apikey auth from [`ExaSearchConfig`], a single +/// route for `POST {alias}/search`. +#[allow(clippy::cognitive_complexity)] +pub async fn register_exa_upstream( + gateway: &Arc, + ctx: &toolkit_security::SecurityContext, + cfg: &ExaSearchConfig, +) -> anyhow::Result<()> { + use oagw_sdk::{ + AuthConfig, CreateRouteRequest, CreateUpstreamRequest, Endpoint, HttpMatch, HttpMethod, + MatchRules, PathSuffixMode, Scheme, Server, + }; + + let server = Server { + endpoints: vec![Endpoint { + scheme: Scheme::Https, + host: cfg.host.clone(), + port: 443, + }], + }; + + let mut builder = + CreateUpstreamRequest::builder(server, "gts.cf.core.oagw.protocol.v1~cf.core.oagw.http.v1") + .enabled(true); + if let Some(alias) = &cfg.upstream_alias { + builder = builder.alias(alias); + } + if let (Some(plugin_type), Some(config)) = (&cfg.auth_plugin_type, &cfg.auth_config) { + builder = builder.auth(AuthConfig { + plugin_type: plugin_type.clone(), + sharing: oagw_sdk::SharingMode::Inherit, + config: Some(config.clone()), + }); + } + + let upstream = gateway + .create_upstream(ctx.clone(), builder.build()) + .await + .map_err(|e| anyhow::anyhow!("OAGW exa upstream registration failed: {e}"))?; + info!(alias = %upstream.alias, upstream_id = %upstream.id, "OAGW exa upstream registered"); + + let match_rules = MatchRules { + http: Some(HttpMatch { + methods: vec![HttpMethod::Post], + path: "/search".to_owned(), + query_allowlist: vec![], + path_suffix_mode: PathSuffixMode::Disabled, + }), + grpc: None, + }; + match gateway + .create_route( + ctx.clone(), + CreateRouteRequest::builder(upstream.id, match_rules) + .enabled(true) + .build(), + ) + .await + { + Ok(route) => info!(route_id = %route.id, "OAGW exa /search route registered"), + Err(e) => warn!(error = %e, "OAGW exa route registration failed (may already exist)"), + } + + Ok(()) +} + /// Create an OAGW upstream for a single provider entry. /// /// Only passes `upstream_alias` to OAGW when explicitly configured diff --git a/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs b/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs index 6de969187..1cf4b34fb 100644 --- a/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs +++ b/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs @@ -74,6 +74,7 @@ fn make_entry(model_id: &str, tier: ModelTier) -> ModelCatalogEntry { }), system_prompt: String::new(), thread_summary_prompt: String::new(), + enabled_function_tools: Vec::new(), } }