diff --git a/config/mini-chat.yaml b/config/mini-chat.yaml index e4147e691..e0ca69354 100644 --- a/config/mini-chat.yaml +++ b/config/mini-chat.yaml @@ -105,7 +105,13 @@ gears: config: secrets: - key: "azure-openai-key" - value: "${AZURE_OPENAI_API_KEY}" + value: "${AZURE_OPENAI_API_KEY:-}" + # Local H200 vLLM (gpt-oss). Add the value yourself. + - key: "vllm-h200-key" + value: "${VLLM_H200_API_KEY}" + # exa.ai web search. Add the value yourself. + - key: "exa-api-key" + value: "${EXA_API_KEY}" file-parser: config: @@ -135,7 +141,7 @@ gears: # `upstream_alias` is optional — defaults to `host` when omitted. thread_summary_worker: enabled: true - summary_model_id: "gpt-4.1-mini" + summary_model_id: "gpt-oss-120b" compression_threshold_pct: 80 summary_system_prompt: "You are a conversation summarizer. Given a conversation (and optionally an existing summary), produce a detailed structured summary. Respond with an block (your reasoning) followed by a block (the final summary). Only the content will be stored. Do not invent information not present in the conversation." providers: @@ -152,6 +158,38 @@ gears: prefix: "" secret_ref: "azure-openai-key" + # ── Local H200 vLLM (gpt-oss via the Responses API) ─────────────── + # https://local-llm.adc.corp.acronis.com → POST /v1/responses + # (If this endpoint speaks Chat Completions instead, switch to + # kind: openai_chat_completions and api_path: /v1/chat/completions.) + vllm_h200: + kind: vllm_responses # vLLM omits the SSE `event:` field; its adapter compensates + storage_kind: openai # required; no RAG used for this model + host: "local-llm.adc.corp.acronis.com" + api_path: "/v1/responses" + supports_file_search_filters: false + auth_plugin_type: "gts.cf.core.oagw.auth_plugin.v1~cf.core.oagw.apikey.v1" + auth_config: + header: "Authorization" + prefix: "Bearer " + secret_ref: "vllm-h200-key" + + # exa.ai web-search function tool. Models opt in via + # `enabled_function_tools: ["exa_search"]` in the model catalog below. + # Egress goes through OAGW (apikey auth injects the x-api-key header). + exa_search: + enabled: true + host: "api.exa.ai" + search_type: "auto" + num_results: 10 + max_calls_per_message: 3 + max_chars: 2000 + auth_plugin_type: "gts.cf.core.oagw.auth_plugin.v1~cf.core.oagw.apikey.v1" + auth_config: + header: "x-api-key" + prefix: "" + secret_ref: "exa-api-key" + # ── Azure OpenAI example (uncomment and set your endpoint) ────────── # azure_openai: # kind: openai_responses @@ -210,8 +248,8 @@ gears: provider_model_id: "gpt-4.1" display_name: "GPT-4.1" description: "Most capable model" - provider_id: "azure_openai" - provider_display_name: "Azure OpenAI" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" icon: "" tier: Premium enabled: true @@ -272,8 +310,8 @@ gears: provider_model_id: "gpt-4.1-mini" display_name: "GPT-4.1 Mini" description: "Fast and efficient model" - provider_id: "azure_openai" - provider_display_name: "Azure OpenAI" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" icon: "" tier: Standard enabled: true @@ -336,8 +374,8 @@ gears: provider_model_id: "gpt-4.1-mini" display_name: "GPT-4.1 Mini (Tiny Context - E2E)" description: "E2E testing model with small context for thread summary trigger" - provider_id: "azure_openai" - provider_display_name: "Azure OpenAI" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" icon: "" tier: Standard enabled: true @@ -393,6 +431,71 @@ gears: is_default: false sort_order: 99 + # ── GPT-OSS 120B on local H200 (vLLM) with exa.ai web search ── + - id: "gpt-oss-120b" + provider_model_id: "h200/gpt-oss-120b" + display_name: "GPT-OSS 120B (H200)" + description: "Local gpt-oss-120b served by vLLM on the H200, with web search via exa.ai" + provider_id: "vllm_h200" + provider_display_name: "Local H200 (vLLM)" + icon: "" + tier: Standard + enabled: true + system_prompt: "You are a helpful, concise assistant. Answer clearly and directly." + thread_summary_prompt: "" + multimodal_capabilities: [] + context_window: 131072 + max_output_tokens: 32768 + max_input_tokens: 131072 + input_tokens_credit_multiplier_micro: 1000000 + output_tokens_credit_multiplier_micro: 3000000 + multiplier_display: "1x" + # Custom function tools enabled for this model (resolved against the + # gear's function-tool registry; `exa_search` is configured above). + enabled_function_tools: ["exa_search"] + estimation_budgets: + bytes_per_token_conservative: 4 + fixed_overhead_tokens: 100 + safety_margin_pct: 10 + image_token_budget: 1000 + tool_surcharge_tokens: 500 + web_search_surcharge_tokens: 500 + code_interpreter_surcharge_tokens: 1000 + minimal_generation_floor: 50 + max_num_results: 5 + web_search_context_size: low + max_tool_calls: 10 + general_config: + type: "" + available_from: "1970-01-01T00:00:00Z" + max_file_size_mb: 25 + api_params: + temperature: 0.7 + top_p: 1.0 + frequency_penalty: 0.0 + presence_penalty: 0.0 + stop: [] + features: + streaming: true + structured_output: true + tool_support: + web_search: false + file_search: false + image_generation: false + code_interpreter: false + mcp: false + supported_endpoints: + chat_completions: true + responses: true + embeddings: false + image_generation: false + audio_speech_generation: false + audio_transcription: false + audio_translation: false + preference: + is_default: false + sort_order: 2 + oagw: config: proxy_timeout_secs: 60 diff --git a/gears/mini-chat/docs/ADR/0004-cpt-cf-mini-chat-adr-pluggable-function-tools.md b/gears/mini-chat/docs/ADR/0004-cpt-cf-mini-chat-adr-pluggable-function-tools.md new file mode 100644 index 000000000..a63d7b813 --- /dev/null +++ b/gears/mini-chat/docs/ADR/0004-cpt-cf-mini-chat-adr-pluggable-function-tools.md @@ -0,0 +1,87 @@ +--- +status: accepted +date: 2026-06-25 +--- +# Pluggable Function Tools via a Generic Registry (exa.ai Web Search) + +**ID**: `cpt-cf-mini-chat-adr-pluggable-function-tools` + +## Context and Problem Statement + +mini-chat already supports one custom LLM function tool — `search_knowledge` (RAG). Its +handling is hard-coded in the agentic provider loop: the loop matches the tool name literally +(`if name == "search_knowledge" { … } else { unexpected_tool_use → fail }`). Adding a second +tool (web search via exa.ai, requested for specific models) by copying that branch would +duplicate the per-tool limit, replay, and graceful-degradation logic for every future tool. + +We need a way to (a) add new function tools without touching the loop, (b) enable them +per-model from configuration, and (c) call third-party tool providers (exa.ai) safely. + +## Decision Drivers + +* Extensibility — new tools should not require new branches in the agentic loop. +* Per-model control — only models that opt in (config) should advertise a given tool. +* Egress & credential management — third-party calls must be auditable and key-managed. +* Blast radius — avoid destabilising the working `search_knowledge` RAG path. + +## Considered Options + +* **A. Generic `FunctionTool` registry** — a domain trait + a `HashMap>`; the loop dispatches `function_call`s by name. +* **B. Hard-coded branch per tool** — add an `exa_search` branch beside `search_knowledge`. +* **C. Direct HTTP from the tool** vs. **egress through OAGW** (orthogonal sub-decision). + +## Decision Outcome + +Chosen: **A (generic registry)** with **egress through OAGW**. + +* A domain port `FunctionTool { name, definition() -> LlmFunctionDef, system_prompt_guard(), + max_calls(), execute() }` decouples the loop from concrete tools. The loop keeps the existing + `search_knowledge` branch and adds **one** generic fall-through that looks the tool up in the + per-request registry, enforces `max_calls()` with the same soft-limit notice, executes, and + injects the `function_call_output`. +* Per-model enablement is a new `enabled_function_tools: Vec` on `ModelCatalogEntry` + (CCM/policy-snapshot driven, `#[serde(default)]`). `StreamService` resolves the model's list + against the registry per request. +* `exa_search` is the first registered tool. It reaches exa.ai **through OAGW** (host + apikey + auth plugin injecting `x-api-key`), reusing the existing `RagHttpClient` JSON-POST primitive — + consistent with how LLM providers and RAG egress already work + (`gears/model-registry/docs/ADR/0003-…-oagw-provider-access`). +* Token accounting reuses the existing flat `tool_surcharge_tokens`: when a model has any + enabled function tool, preflight sets `tools_enabled = true`. No per-tool surcharge for now. +* The function descriptor authoring type is a new function-only `LlmFunctionDef` + (`definition()` returns it; `assemble_context` wraps it into `LlmTool::Function`). + +### Consequences + +* Good — new tools are added by implementing `FunctionTool` and registering; the loop is closed + to modification. +* Good — per-model gating lives in the policy catalog alongside other model capabilities. +* Good — exa credentials stay in OAGW; calls are audited via the existing egress path. +* Good — `search_knowledge` is untouched, so the RAG path is not destabilised. +* Bad / accepted trade-off — two mechanisms temporarily coexist: `search_knowledge` keeps its + bespoke branch (per-tool metrics, DB increments, `search_result` blocks) while new tools use + the registry. **Follow-up (phase 2):** migrate `search_knowledge` into the registry and + collapse `LlmTool::Function { … }` into `Function(LlmFunctionDef)` so there is one tool type + and one dispatch path. +* Bad / accepted — token cost of a tool's *output* is bounded by config (`max_chars`), not + preflighted (same as `search_knowledge` today). + +### Confirmation + +* Unit tests: `assemble_context` appends the tool descriptor + guard; `resolve_function_tools_from` + filters by the model's `enabled_function_tools`; `ExaSearchTool::format_results` against a + recorded exa response. +* Code review: the agentic loop has exactly one generic dispatch branch; unknown tool names + still fail via `unexpected_tool_use`. +* E2E: a model with `enabled_function_tools: ["exa_search"]` triggers a web-search tool call; + a model without the flag does not advertise the tool. + +## Pros and Cons of the Options + +* **A. Generic registry** — Good: extensible, testable, single dispatch path. Bad: one + indirection layer; interim coexistence with the `search_knowledge` branch. +* **B. Hard-coded branch** — Good: trivial for one tool. Bad: duplicates limit/replay/ + degradation logic per tool; the loop grows with every tool. +* **C. Direct HTTP** — Good: fewer moving parts. Bad: bypasses centralised egress, credential + management, and audit; inconsistent with the rest of the stack. Rejected in favour of OAGW. diff --git a/gears/mini-chat/docs/DESIGN.md b/gears/mini-chat/docs/DESIGN.md index 350272fbb..f15e624bf 100644 --- a/gears/mini-chat/docs/DESIGN.md +++ b/gears/mini-chat/docs/DESIGN.md @@ -5027,6 +5027,53 @@ The surcharge model is **deterministic and independent of provider runtime behav - P1 does **NOT** implement proportional infrastructure cost modelling. There is no per-invocation, per-query, or per-retrieval-pass cost tracking. - Accurate backend cost accounting for tools, web search, and RAG operations is explicitly **out of scope** for P1. Future phases MAY introduce metered surcharges; any such change MUST be reflected via a new `policy_version`. +##### Custom Function Tools (Generic Registry + exa.ai Web Search) + +Beyond the provider-native tools (`file_search`, `web_search`, `code_interpreter`) and the +built-in `search_knowledge` RAG function, mini-chat supports **pluggable custom function tools** +through a generic registry. This is how `exa_search` (web search via exa.ai) is added without +new branches in the agentic loop. + +**Mechanism** + +- Domain port `FunctionTool` (`domain/ports/function_tool.rs`): + `name()`, `definition() -> LlmFunctionDef`, `system_prompt_guard() -> Option`, + `max_calls() -> u32`, `async execute(ctx, input) -> Result`. +- The gear builds a registry `HashMap>` at init and injects it into + `StreamService`. Per request, `resolve_function_tools` intersects the model's + `enabled_function_tools` with the registry, yielding the tool descriptors (appended to the LLM + request), their system-prompt guards (appended to the system instructions), and a dispatch map. +- The agentic loop in `provider_task.rs` keeps the bespoke `search_knowledge` branch and adds one + **generic** dispatch: on a `function_call` whose name is in the dispatch map, it enforces the + tool's `max_calls()` per-message limit (soft-limit notice on overflow, same graceful degradation + as `search_knowledge`), calls `execute`, and injects the result as `function_call_output`. + Unknown names still fail via `unexpected_tool_use`. + +**Per-model enablement** + +`ModelCatalogEntry.enabled_function_tools: Vec` (policy-snapshot / CCM driven) lists the +tool names a model may call, e.g. `["exa_search"]`. Models without the entry never see the tool. + +**exa_search tool** + +- Descriptor: `{ name: "exa_search", description: "Search the public web …", + parameters: { query: string (required) } }`. +- Egress through OAGW: `POST /{alias}/search` with body + `{ "query", "type": , "numResults": , "contents": { "highlights": true } }`. + The OAGW apikey auth plugin injects the `x-api-key` header — the key lives in the credstore, + never in mini-chat. Reuses the `RagHttpClient` JSON-POST primitive. +- Response `results[]{ title, url, highlights[] }` is formatted into a compact text block + (bounded by `exa_search.max_chars`) and returned as the `function_call_output`. +- Reference: . + +**Token cost.** Custom function tools reuse the existing flat `tool_surcharge_tokens`: when a +model has any enabled function tool, preflight sets `tools_enabled = true` (no per-tool +surcharge). The tool's output is bounded by `max_chars`, not separately preflighted — consistent +with `search_knowledge`. + +> See ADR `cpt-cf-mini-chat-adr-pluggable-function-tools`. Phase-2 follow-up: migrate +> `search_knowledge` into the registry and collapse `LlmTool::Function` onto `LlmFunctionDef`. + #### 5.5.7 Reserved Credits Calculation After estimation (canonical form — identical to section 5.4.1): @@ -6544,6 +6591,15 @@ Legend: | Parameter | Type | Default | Source | Notes | |-----------|------|---------|--------|-------| | `mini-chat.url_prefix` | `string` | `/mini-chat` | ConfigMap | ToolKit gear config, no CCM API equivalent | +| `mini-chat.exa_search.enabled` | `bool` | `false` | ConfigMap | Enables the `exa_search` web-search function tool | +| `mini-chat.exa_search.host` | `string` | `api.exa.ai` | ConfigMap | exa upstream host (OAGW registers it + a `POST /search` route) | +| `mini-chat.exa_search.upstream_alias` | `string?` | host | ConfigMap | OAGW alias; defaults to `host` | +| `mini-chat.exa_search.auth_plugin_type` / `auth_config` | object | — | ConfigMap | OAGW apikey auth (`header: x-api-key`, `secret_ref`); key lives in the credstore | +| `mini-chat.exa_search.search_type` | `string` | `auto` | ConfigMap | exa search type (`auto`/`fast`/`deep`/…) | +| `mini-chat.exa_search.num_results` | `integer` | `10` | ConfigMap | `numResults` per call | +| `mini-chat.exa_search.max_calls_per_message` | `integer` | `3` | ConfigMap | Soft per-message call limit (graceful degradation) | +| `mini-chat.exa_search.max_chars` | `integer` | `2000` | ConfigMap | Max characters kept per formatted result | +| `mini-chat.exa_search.guard` | `string` | built-in | ConfigMap | System-prompt guard appended when the tool is enabled | ## B.2 Policy / Models / Limits (via `mini-chat-model-policy-plugin`) @@ -6575,6 +6631,7 @@ All fields below are per-model entries inside the catalog. | `api_params.*` | object | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].general_config.api_params` | | `features.*` | object | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].general_config.features` | | `tool_support.*` | object | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].general_config.tool_support` | +| `enabled_function_tools` | `string[]` | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].enabled_function_tools` — custom function tools the model may call (e.g. `["exa_search"]`); resolved against the gear's function-tool registry, defaults to `[]` | | `sort_order` | `integer` | **CCM API**: `GET /policies/{v}` | `snapshot.model_catalog[].preference.sort_order` | ### B.2.3 Kill switches / emergency flags diff --git a/gears/mini-chat/mini-chat-sdk/src/models.rs b/gears/mini-chat/mini-chat-sdk/src/models.rs index 2ea5556b1..26e32bfbb 100644 --- a/gears/mini-chat/mini-chat-sdk/src/models.rs +++ b/gears/mini-chat/mini-chat-sdk/src/models.rs @@ -85,6 +85,11 @@ pub struct ModelCatalogEntry { /// Maximum tool calls the provider may make per request. #[serde(default = "default_max_tool_calls")] pub max_tool_calls: u32, + /// Names of custom function tools enabled for this model (e.g. + /// `["exa_search"]`). Resolved against the gear's function-tool registry at + /// request time; unknown names are ignored. Empty = no custom tools. + #[serde(default)] + pub enabled_function_tools: Vec, /// Full general config captured at snapshot time. pub general_config: ModelGeneralConfig, /// Tenant preference settings captured at snapshot time. @@ -387,6 +392,7 @@ mod tests { }), system_prompt: String::new(), thread_summary_prompt: String::new(), + enabled_function_tools: Vec::new(), } } diff --git a/gears/mini-chat/mini-chat/src/config.rs b/gears/mini-chat/mini-chat/src/config.rs index d849d65dc..7e153acd2 100644 --- a/gears/mini-chat/mini-chat/src/config.rs +++ b/gears/mini-chat/mini-chat/src/config.rs @@ -57,6 +57,10 @@ pub struct MiniChatConfig { /// Knowledge search (RAG) feature configuration. #[serde(default)] pub knowledge_search: KnowledgeSearchConfig, + /// Exa web-search function-tool configuration. + #[expand_vars] + #[serde(default)] + pub exa_search: ExaSearchConfig, } /// Which file/vector-store implementation to use for RAG operations. @@ -441,6 +445,7 @@ impl Default for MiniChatConfig { cleanup_worker: CleanupWorkerConfig::default(), thumbnail: ThumbnailConfig::default(), knowledge_search: KnowledgeSearchConfig::default(), + exa_search: ExaSearchConfig::default(), } } } @@ -1044,6 +1049,142 @@ impl KnowledgeSearchConfig { } } +/// Configuration for the `exa_search` web-search function tool (exa.ai). +/// +/// When `enabled` is `true`, an [`crate::domain::ports::FunctionTool`] named +/// `exa_search` is registered. Models that list `"exa_search"` in their +/// `enabled_function_tools` may call it; the agentic loop dispatches the call, +/// issues `POST /{alias}/search` through OAGW, and injects the formatted +/// results as a `function_call_output`. +/// +/// Egress goes through OAGW (host + apikey auth plugin), mirroring the LLM +/// providers — the API key is held by OAGW, never by mini-chat. +#[derive(Debug, Clone, Serialize, Deserialize, toolkit_macros::ExpandVars)] +#[serde(deny_unknown_fields)] +pub struct ExaSearchConfig { + /// Enable the `exa_search` function tool. Default: `false`. + #[serde(default)] + pub enabled: bool, + + /// Upstream hostname for exa. Default: `"api.exa.ai"`. + #[serde(default = "default_exa_host")] + pub host: String, + + /// OAGW upstream alias. When `None`, falls back to `host` (OAGW derives + /// the alias from the hostname). Set explicitly only for IP-based hosts. + #[serde(default)] + pub upstream_alias: Option, + + /// OAGW auth plugin type. Example: + /// `gts.cf.core.oagw.auth_plugin.v1~cf.core.oagw.apikey.v1`. + #[serde(default)] + pub auth_plugin_type: Option, + + /// Auth plugin config (e.g. `header=x-api-key`, `secret_ref=${EXA_API_KEY}`). + /// Values support `${VAR}` env expansion via [`config_expanded()`]. + #[expand_vars] + #[serde(default)] + pub auth_config: Option>, + + /// Exa search `type` (`auto`, `fast`, `instant`, `deep`, …). Default: `"auto"`. + #[serde(default = "default_exa_search_type")] + pub search_type: String, + + /// Number of results requested per call (`numResults`). Default: 10. + #[serde(default = "default_exa_num_results")] + pub num_results: u32, + + /// Maximum `exa_search` calls per message in the agentic loop. Default: 3. + #[serde(default = "default_exa_max_calls")] + pub max_calls_per_message: u32, + + /// Maximum characters kept per result after formatting (bounds context cost). + /// Default: 2000. + #[serde(default = "default_exa_max_chars")] + pub max_chars: usize, + + /// Guard instruction appended to the system prompt when enabled. + #[serde(default = "default_exa_guard")] + pub guard: String, +} + +impl Default for ExaSearchConfig { + fn default() -> Self { + Self { + enabled: false, + host: default_exa_host(), + upstream_alias: None, + auth_plugin_type: None, + auth_config: None, + search_type: default_exa_search_type(), + num_results: default_exa_num_results(), + max_calls_per_message: default_exa_max_calls(), + max_chars: default_exa_max_chars(), + guard: default_exa_guard(), + } + } +} + +impl ExaSearchConfig { + pub fn validate(&self) -> Result<(), String> { + if self.enabled { + if self.host.trim().is_empty() { + return Err("exa_search.host must not be empty when enabled".to_owned()); + } + if self.search_type.trim().is_empty() { + return Err("exa_search.search_type must not be empty when enabled".to_owned()); + } + } + if self.num_results == 0 { + return Err("exa_search.num_results must be > 0".to_owned()); + } + if self.max_calls_per_message == 0 { + return Err("exa_search.max_calls_per_message must be > 0".to_owned()); + } + if self.max_chars == 0 { + return Err("exa_search.max_chars must be > 0".to_owned()); + } + Ok(()) + } +} + +fn default_exa_host() -> String { + "api.exa.ai".to_owned() +} + +fn default_exa_search_type() -> String { + "auto".to_owned() +} + +const fn default_exa_num_results() -> u32 { + 10 +} + +const fn default_exa_max_calls() -> u32 { + 3 +} + +const fn default_exa_max_chars() -> usize { + 2000 +} + +fn default_exa_guard() -> String { + "You have access to exactly one web tool: exa_search, which searches the public web \ + for current information. Use it when the question depends on recent events or facts \ + that may not be in your training data. You do NOT have a browser: never call `open`, \ + `find`, `click`, or any other browsing tool. The exa_search results (titles, URLs, \ + and highlights) are all you get — answer directly from them; do not attempt to open \ + or fetch pages.\n\ + \n\ + Your final answer must contain ONLY the user-facing response. Do not include planning, \ + reasoning, or meta-commentary in it (e.g. phrases like \"Provide temperature…\", \ + \"Use the search results\", \"Cite sources\") — keep all of that out of the answer.\n\ + Cite sources as inline Markdown links using the URLs from the exa_search results, e.g. \ + [AccuWeather](https://example.com/page). Do NOT use citation markers such as 【1†…】, \ + footnotes, or bare numbers — only Markdown links." + .to_owned() +} + fn default_url_prefix() -> String { DEFAULT_URL_PREFIX.to_owned() } diff --git a/gears/mini-chat/mini-chat/src/domain/llm.rs b/gears/mini-chat/mini-chat/src/domain/llm.rs index 8e799db60..76ecf68bd 100644 --- a/gears/mini-chat/mini-chat/src/domain/llm.rs +++ b/gears/mini-chat/mini-chat/src/domain/llm.rs @@ -123,6 +123,20 @@ impl FileSearchFilter { } } +/// A provider-agnostic function-tool descriptor (name + description + JSON-schema). +/// +/// This is the function-only authoring type returned by +/// [`crate::domain::ports::FunctionTool::definition`]. `assemble_context` wraps +/// it into [`LlmTool::Function`] for the request. (Phase 2 will collapse the +/// enum variant to hold this struct directly.) +#[domain_model] +#[derive(Debug, Clone)] +pub struct LlmFunctionDef { + pub name: String, + pub description: String, + pub parameters: serde_json::Value, +} + /// A provider-agnostic tool descriptor. /// /// Each adapter maps supported tools to its wire format and silently drops diff --git a/gears/mini-chat/mini-chat/src/domain/models.rs b/gears/mini-chat/mini-chat/src/domain/models.rs index b5d668566..26046f474 100644 --- a/gears/mini-chat/mini-chat/src/domain/models.rs +++ b/gears/mini-chat/mini-chat/src/domain/models.rs @@ -176,6 +176,9 @@ pub struct ResolvedModel { pub thread_summary_prompt: String, /// Maximum output tokens for this model. pub max_output_tokens: u32, + /// Names of custom function tools enabled for this model (e.g. + /// `["exa_search"]`). Resolved against the gear's function-tool registry. + pub enabled_function_tools: Vec, } impl From<&mini_chat_sdk::ModelCatalogEntry> for ResolvedModel { @@ -202,6 +205,7 @@ impl From<&mini_chat_sdk::ModelCatalogEntry> for ResolvedModel { tool_support: e.general_config.tool_support.clone(), thread_summary_prompt: e.thread_summary_prompt.clone(), max_output_tokens: e.max_output_tokens, + enabled_function_tools: e.enabled_function_tools.clone(), } } } diff --git a/gears/mini-chat/mini-chat/src/domain/ports/function_tool.rs b/gears/mini-chat/mini-chat/src/domain/ports/function_tool.rs new file mode 100644 index 000000000..6779f8a92 --- /dev/null +++ b/gears/mini-chat/mini-chat/src/domain/ports/function_tool.rs @@ -0,0 +1,75 @@ +//! Domain port for pluggable LLM function tools. +//! +//! The [`FunctionTool`] trait decouples the agentic provider loop from any +//! specific custom-tool implementation. Each tool advertises its provider- +//! agnostic [`LlmFunctionDef`] descriptor and executes a single tool call, +//! returning the text injected back as a `function_call_output`. +//! +//! The agentic loop in `stream_service/provider_task.rs` dispatches incoming +//! `function_call`s by name through a registry of these trait objects, so a +//! new tool (e.g. web search via Exa) is added by implementing this trait and +//! registering it — no new branch in the loop. +//! +//! Concrete implementations live in `infra::llm::providers` (e.g. +//! `exa_search_tool.rs`). + +use async_trait::async_trait; +use toolkit_macros::domain_model; +use toolkit_security::SecurityContext; + +use crate::domain::llm::LlmFunctionDef; + +/// Errors from a function-tool invocation. +/// +/// The agentic loop never fails the turn on these — it injects a textual +/// fallback as the `function_call_output` so the model can keep going +/// (same graceful-degradation policy as `search_knowledge`). +#[domain_model] +#[derive(Debug, thiserror::Error)] +pub enum FunctionToolError { + /// Upstream explicitly rejected the request (4xx) or the arguments were invalid. + #[error("function tool rejected: {0}")] + Rejected(String), + /// Upstream unavailable or transient failure (5xx, timeout). + #[error("function tool unavailable: {0}")] + Unavailable(String), + /// Configuration error (missing alias, bad credentials, etc.). + #[error("function tool configuration error: {0}")] + Configuration(String), +} + +/// Port for a pluggable LLM function tool. +/// +/// Implementations are registered in a `HashMap>` +/// keyed by [`FunctionTool::name`]. Per-model enablement is resolved upstream; +/// the loop only ever sees the tools enabled for the current request. +#[async_trait] +pub trait FunctionTool: Send + Sync { + /// Tool name the model calls and the loop dispatches on. MUST equal the + /// `name` of the [`LlmFunctionDef`] returned by [`Self::definition`]. + fn name(&self) -> &str; + + /// Provider-agnostic function descriptor (name, description, JSON-schema + /// params) advertised to the model. + fn definition(&self) -> LlmFunctionDef; + + /// Guard instruction appended to the system prompt when this tool is + /// enabled (steers the model on when/how to call it). `None` = no guard. + fn system_prompt_guard(&self) -> Option { + None + } + + /// Maximum calls allowed per message before the loop injects a soft + /// limit notice instead of executing (graceful degradation). + fn max_calls(&self) -> u32; + + /// Execute a single tool call. + /// + /// `input` is the parsed arguments JSON the model produced. The returned + /// string is injected verbatim as the `function_call_output` text. + async fn execute( + &self, + ctx: SecurityContext, + input: serde_json::Value, + ) -> Result; +} diff --git a/gears/mini-chat/mini-chat/src/domain/ports/mod.rs b/gears/mini-chat/mini-chat/src/domain/ports/mod.rs index 0496c696c..9a72da822 100644 --- a/gears/mini-chat/mini-chat/src/domain/ports/mod.rs +++ b/gears/mini-chat/mini-chat/src/domain/ports/mod.rs @@ -25,10 +25,12 @@ use super::error::DomainError; pub type FileStream = Pin>> + Send>>; +pub(crate) mod function_tool; pub(crate) mod knowledge_retriever; pub(crate) mod metric_labels; pub(crate) mod metrics; +pub(crate) use function_tool::{FunctionTool, FunctionToolError}; pub(crate) use knowledge_retriever::KnowledgeRetriever; pub(crate) use metrics::MiniChatMetricsPort; diff --git a/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs b/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs index 0f2c85085..9261addb2 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/context_assembly.rs @@ -79,6 +79,13 @@ pub struct ContextInput<'a> { pub token_budget: Option, /// Provider file IDs for image attachments on the current user message. pub image_file_ids: &'a [String], + /// Extra custom function-tool descriptors (e.g. `exa_search`) enabled for + /// this request. Appended verbatim to the tool list. Each is a + /// [`LlmTool::Function`] resolved from the gear's function-tool registry. + pub extra_function_tools: Vec, + /// System-prompt guards for the enabled custom function tools, appended to + /// the system instructions (parallel to `extra_function_tools`). + pub extra_function_guards: Vec, } /// Output of context assembly — ready to feed into `LlmRequestBuilder`. @@ -219,6 +226,7 @@ pub fn assemble_context( input.file_search_guard, input.knowledge_search_enabled, input.knowledge_search_guard, + &input.extra_function_guards, ); // ── Tools ── @@ -266,6 +274,9 @@ pub fn assemble_context( }); } + // Custom function tools (e.g. exa_search) resolved per-model from the registry. + tools.extend(input.extra_function_tools.iter().cloned()); + // ── Truncation ── if let Some(ref budget) = input.token_budget { let available = compute_available_budget(budget)?; @@ -380,6 +391,7 @@ pub fn assemble_context( /// Build system instructions from base prompt + conditional guard strings. /// Returns `None` if the result would be empty. +#[allow(clippy::too_many_arguments)] fn build_system_instructions( system_prompt: &str, web_search_enabled: bool, @@ -388,6 +400,7 @@ fn build_system_instructions( file_search_guard: &str, knowledge_search_enabled: bool, knowledge_search_guard: &str, + extra_function_guards: &[String], ) -> Option { let mut parts: Vec<&str> = Vec::new(); @@ -403,6 +416,12 @@ fn build_system_instructions( if knowledge_search_enabled && !knowledge_search_guard.is_empty() { parts.push(knowledge_search_guard); } + // Guards for enabled custom function tools (e.g. exa_search). + for guard in extra_function_guards { + if !guard.is_empty() { + parts.push(guard); + } + } if parts.is_empty() { None @@ -443,6 +462,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert!(result.system_instructions.is_none()); @@ -450,6 +471,49 @@ mod tests { assert_eq!(result.messages.len(), 1); } + // Custom function tools are appended to the tool list, and their guards + // are appended to the system instructions. + #[test] + fn extra_function_tools_and_guards_are_appended() { + let exa_tool = LlmTool::Function { + name: "exa_search".to_owned(), + description: "Search the web.".to_owned(), + parameters: serde_json::json!({"type": "object"}), + }; + let result = assemble_context(&ContextInput { + system_prompt: "You are helpful.", + web_search_guard: "", + file_search_guard: "", + thread_summary: None, + recent_messages: &[], + user_message: "hello", + web_search_enabled: false, + file_search_enabled: false, + vector_store_ids: &[], + file_search_filters: None, + web_search_context_size: crate::domain::llm::WebSearchContextSize::Low, + file_search_max_num_results: 5, + code_interpreter_file_ids: vec![], + token_budget: None, + knowledge_search_enabled: false, + knowledge_search_guard: "", + image_file_ids: &[], + extra_function_tools: vec![exa_tool], + extra_function_guards: vec!["Use exa_search for fresh facts.".to_owned()], + }) + .unwrap(); + + assert!( + result + .tools + .iter() + .any(|t| matches!(t, LlmTool::Function { name, .. } if name == "exa_search")), + "exa_search tool should be present" + ); + let sys = result.system_instructions.expect("system instructions"); + assert!(sys.contains("Use exa_search for fresh facts.")); + } + // 5.7: system prompt + web_search enabled → guard appended #[test] fn system_prompt_with_web_search_guard() { @@ -471,6 +535,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -499,6 +565,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -527,6 +595,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -556,6 +626,8 @@ mod tests { knowledge_search_enabled: true, knowledge_search_guard: "Use search_knowledge for internal docs.", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -595,6 +667,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "Use search_knowledge for internal docs.", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let instructions = result.system_instructions.unwrap(); @@ -631,6 +705,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); // First message should be the thread summary @@ -672,6 +748,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.messages.len(), 3); // 2 recent + current @@ -703,6 +781,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); // system message skipped: 2 recent (user+assistant) + 1 current = 3 @@ -731,6 +811,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let last = result.messages.last().unwrap(); @@ -766,6 +848,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.tools.len(), 2); @@ -802,6 +886,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert!(result.tools.is_empty()); @@ -825,6 +911,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.tools.len(), 1); @@ -943,6 +1031,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -985,6 +1075,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -1037,6 +1129,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -1066,6 +1160,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }); assert!(matches!( @@ -1100,6 +1196,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); @@ -1142,6 +1240,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.tools.len(), 1); @@ -1172,6 +1272,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert!(result.tools.is_empty()); @@ -1216,6 +1318,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); assert_eq!(result.messages.len(), 1); @@ -1246,6 +1350,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let msg = &result.messages[0]; @@ -1274,6 +1380,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &[], + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }) .unwrap(); let msg = &result.messages[0]; @@ -1302,6 +1410,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }); assert!(result.is_ok()); } @@ -1327,6 +1437,8 @@ mod tests { knowledge_search_enabled: false, knowledge_search_guard: "", image_file_ids: &images, + extra_function_tools: Vec::new(), + extra_function_guards: Vec::new(), }); assert!(matches!( result, diff --git a/gears/mini-chat/mini-chat/src/domain/service/mod.rs b/gears/mini-chat/mini-chat/src/domain/service/mod.rs index 62d70c10e..43a044fc4 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/mod.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/mod.rs @@ -210,6 +210,10 @@ impl< summary_config: crate::config::background::ThreadSummaryWorkerConfig, knowledge_search_config: crate::config::KnowledgeSearchConfig, knowledge_retriever: Option>, + function_tools: std::collections::HashMap< + String, + Arc, + >, anthropic_files_client: Option< Arc, >, @@ -287,6 +291,7 @@ impl< Arc::clone(&metrics), knowledge_search_config, knowledge_retriever, + function_tools, ), turns, reactions: ReactionService::new( diff --git a/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs b/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs index 30b9173b8..15a695c95 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/stream_service/mod.rs @@ -39,6 +39,42 @@ use types::{ // StreamService // ════════════════════════════════════════════════════════════════════════════ +/// Resolved custom function tools for a request: `(dispatch map keyed by name, +/// tool descriptors for the LLM request, system-prompt guards)`. +type ResolvedFunctionTools = ( + std::collections::HashMap>, + Vec, + Vec, +); + +/// Pure resolution of enabled custom function tools against a registry. +/// +/// Unknown tool names are silently ignored. Extracted as a free function so it +/// can be unit-tested without constructing a `StreamService`. +fn resolve_function_tools_from( + registry: &std::collections::HashMap>, + enabled: &[String], +) -> ResolvedFunctionTools { + let mut map = std::collections::HashMap::new(); + let mut tools = Vec::new(); + let mut guards = Vec::new(); + for name in enabled { + if let Some(tool) = registry.get(name) { + let def = tool.definition(); + tools.push(crate::domain::llm::LlmTool::Function { + name: def.name, + description: def.description, + parameters: def.parameters, + }); + if let Some(guard) = tool.system_prompt_guard() { + guards.push(guard); + } + map.insert(name.clone(), Arc::clone(tool)); + } + } + (map, tools, guards) +} + /// Service handling SSE streaming and turn orchestration. /// /// In P1 this is a stateless proxy: it builds an LLM request, streams @@ -74,6 +110,10 @@ pub struct StreamService< metrics: Arc, knowledge_search_config: crate::config::KnowledgeSearchConfig, knowledge_retriever: Option>, + /// Custom function-tool registry, keyed by tool name. Resolved per-request + /// against the model's `enabled_function_tools`. + function_tools: + std::collections::HashMap>, } impl< @@ -109,6 +149,10 @@ impl< metrics: Arc, knowledge_search_config: crate::config::KnowledgeSearchConfig, knowledge_retriever: Option>, + function_tools: std::collections::HashMap< + String, + Arc, + >, ) -> Self { Self { db, @@ -129,9 +173,17 @@ impl< metrics, knowledge_search_config, knowledge_retriever, + function_tools, } } + /// Resolve the custom function tools enabled for a model against the + /// registry. Returns `(dispatch map, tool descriptors, system-prompt guards)`. + /// Unknown tool names are silently ignored. + fn resolve_function_tools(&self, enabled: &[String]) -> ResolvedFunctionTools { + resolve_function_tools_from(&self.function_tools, enabled) + } + /// The configured channel capacity for the provider->writer mpsc channel. pub(crate) fn channel_capacity(&self) -> usize { usize::from(self.streaming_config.sse_channel_capacity) @@ -279,6 +331,9 @@ impl< .multimodal_capabilities .iter() .any(|c| c == "VISION_INPUT"); + // Resolve custom function tools enabled for this model against the registry. + let (function_tools, extra_function_tools, extra_function_guards) = + self.resolve_function_tools(&resolved_model.enabled_function_tools); let ResolvedModel { model_id: model, provider_id, @@ -613,7 +668,8 @@ impl< context_window: pf.context_window, max_output_tokens_applied: pf.max_output_tokens_applied, budgets: pf.estimation_budgets, - tools_enabled: file_search_enabled, + // Custom function tools also incur the flat tool surcharge (Q1). + tools_enabled: file_search_enabled || !function_tools.is_empty(), web_search_enabled, code_interpreter_enabled, }); @@ -641,6 +697,8 @@ impl< ci_file_ids, token_budget, &image_file_ids, + extra_function_tools, + extra_function_guards, ) .await?; @@ -686,6 +744,7 @@ impl< provider_file_id_map, anthropic_file_ids, knowledge_search: self.build_knowledge_search_params(&tenant_id_str), + function_tools, }, cancel, tx, @@ -930,6 +989,8 @@ impl< code_interpreter_file_ids: Vec, token_budget: Option, image_file_ids: &[String], + extra_function_tools: Vec, + extra_function_guards: Vec, ) -> Result< ( super::context_assembly::AssembledContext, @@ -1021,6 +1082,8 @@ impl< code_interpreter_file_ids, token_budget, image_file_ids, + extra_function_tools, + extra_function_guards, }) .map_err(|e| StreamError::ContextBudgetExceeded { required_tokens: match &e { @@ -1065,6 +1128,9 @@ impl< cancel: CancellationToken, tx: mpsc::Sender, ) -> Result, StreamError> { + // Resolve custom function tools enabled for this model against the registry. + let (function_tools, extra_function_tools, extra_function_guards) = + self.resolve_function_tools(&resolved_model.enabled_function_tools); let model = resolved_model.model_id; let provider_id = resolved_model.provider_id; let tenant_id = ctx.subject_tenant_id(); @@ -1356,7 +1422,8 @@ impl< context_window: pf.context_window, max_output_tokens_applied: pf.max_output_tokens_applied, budgets: pf.estimation_budgets, - tools_enabled: file_search_enabled, + // Custom function tools also incur the flat tool surcharge (Q1). + tools_enabled: file_search_enabled || !function_tools.is_empty(), web_search_enabled, code_interpreter_enabled, }); @@ -1381,6 +1448,8 @@ impl< ci_file_ids, token_budget, &[], // retry/edit: no new image attachments + extra_function_tools, + extra_function_guards, ) .await?; @@ -1420,6 +1489,7 @@ impl< provider_file_id_map, anthropic_file_ids, knowledge_search: self.build_knowledge_search_params(&tenant_id_str), + function_tools, }, cancel, tx, @@ -1848,6 +1918,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -1913,6 +1984,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -1971,6 +2043,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -2078,6 +2151,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel.clone(), tx, @@ -2255,6 +2329,7 @@ mod tests { metrics, crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -2305,6 +2380,7 @@ mod tests { }, thread_summary_prompt: String::new(), max_output_tokens: 16_384, + enabled_function_tools: Vec::new(), } } @@ -2998,6 +3074,7 @@ mod tests { metrics, crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -3431,6 +3508,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -3611,6 +3689,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -3790,6 +3869,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -3951,6 +4031,7 @@ mod tests { Arc::new(crate::domain::ports::metrics::NoopMetrics), crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -4173,6 +4254,7 @@ mod tests { }, thread_summary_prompt: String::new(), max_output_tokens: 16_384, + enabled_function_tools: Vec::new(), }, false, Vec::new(), @@ -4337,6 +4419,7 @@ mod tests { }, thread_summary_prompt: String::new(), max_output_tokens: 16_384, + enabled_function_tools: Vec::new(), }, false, Vec::new(), @@ -4561,6 +4644,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4620,6 +4704,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4689,6 +4774,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4745,6 +4831,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4803,6 +4890,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -4872,6 +4960,7 @@ mod tests { provider_file_id_map: std::collections::HashMap::new(), anthropic_file_ids: std::collections::HashMap::new(), knowledge_search: None, + function_tools: std::collections::HashMap::new(), }, cancel, tx, @@ -5614,6 +5703,7 @@ mod tests { Arc::new(crate::domain::ports::metrics::NoopMetrics), crate::config::KnowledgeSearchConfig::default(), None, + std::collections::HashMap::new(), ) } @@ -6280,3 +6370,87 @@ mod tests { ); } } + +#[cfg(test)] +mod function_tool_resolution_tests { + use std::sync::Arc; + + use async_trait::async_trait; + + use super::resolve_function_tools_from; + use crate::domain::llm::{LlmFunctionDef, LlmTool}; + use crate::domain::ports::{FunctionTool, FunctionToolError}; + + struct MockTool { + name: &'static str, + guard: Option<&'static str>, + } + + #[async_trait] + impl FunctionTool for MockTool { + fn name(&self) -> &str { + self.name + } + fn definition(&self) -> LlmFunctionDef { + LlmFunctionDef { + name: self.name.to_owned(), + description: "mock".to_owned(), + parameters: serde_json::json!({"type": "object"}), + } + } + fn system_prompt_guard(&self) -> Option { + self.guard.map(ToOwned::to_owned) + } + fn max_calls(&self) -> u32 { + 3 + } + async fn execute( + &self, + _ctx: toolkit_security::SecurityContext, + _input: serde_json::Value, + ) -> Result { + Ok("ok".to_owned()) + } + } + + fn registry() -> std::collections::HashMap> { + let mut m: std::collections::HashMap> = + std::collections::HashMap::new(); + m.insert( + "exa_search".to_owned(), + Arc::new(MockTool { + name: "exa_search", + guard: Some("exa guard"), + }), + ); + m + } + + #[test] + fn resolves_enabled_tool_with_def_and_guard() { + let (map, tools, guards) = + resolve_function_tools_from(®istry(), &["exa_search".to_owned()]); + assert_eq!(map.len(), 1); + assert!(map.contains_key("exa_search")); + assert!( + matches!(&tools[..], [LlmTool::Function { name, .. }] if name == "exa_search"), + "should produce one exa_search Function tool" + ); + assert_eq!(guards, vec!["exa guard".to_owned()]); + } + + #[test] + fn ignores_unknown_tool_names() { + let (map, tools, guards) = + resolve_function_tools_from(®istry(), &["nonexistent".to_owned()]); + assert!(map.is_empty()); + assert!(tools.is_empty()); + assert!(guards.is_empty()); + } + + #[test] + fn empty_enabled_resolves_to_empty() { + let (map, tools, guards) = resolve_function_tools_from(®istry(), &[]); + assert!(map.is_empty() && tools.is_empty() && guards.is_empty()); + } +} diff --git a/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs b/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs index 1288418be..e72c2da80 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/stream_service/provider_task.rs @@ -15,6 +15,7 @@ use toolkit_security::SecurityContext; use tracing::{Instrument, debug, info, warn}; use crate::domain::llm::ToolPhase; +use crate::domain::ports::FunctionTool; use crate::domain::ports::knowledge_retriever::{ KnowledgeRetriever, RetrievalRequest, RetrievedChunk, }; @@ -74,6 +75,10 @@ pub(super) struct ProviderTaskConfig { pub anthropic_file_ids: std::collections::HashMap, /// Knowledge search parameters; `None` when the feature is disabled. pub knowledge_search: Option, + /// Custom function tools enabled for this request, keyed by tool name. + /// The agentic loop dispatches `function_call`s whose name matches a key + /// here. Empty when no custom tools are enabled for the model. + pub function_tools: std::collections::HashMap>, } /// All five terminal paths (provider done, incomplete, provider error, @@ -109,6 +114,7 @@ pub(super) fn spawn_provider_task = Vec::new(); let mut knowledge_call_count: u32 = 0; + // Per-custom-tool call counts (keyed by tool name) for generic + // function-tool dispatch. Mirrors knowledge_call_count's soft-limit policy. + let mut function_tool_calls: std::collections::HashMap = + std::collections::HashMap::new(); // Hard cap on agentic-loop iterations. Without it, a model that keeps // emitting `search_knowledge` after the soft per-message limit fires @@ -158,12 +168,23 @@ pub(super) fn spawn_provider_task= tool.max_calls() { + warn!( + tool = %name, + limit = tool.max_calls(), + "function tool per-message limit reached, injecting soft limit response" + ); + raw_input_items.push(serde_json::json!({ + "type": "function_call", + "call_id": tool_use_id, + "name": name, + "arguments": raw_arguments, + })); + raw_input_items.push(serde_json::json!({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": "Tool call limit reached for this message. \ + Please answer based on the information already gathered.", + })); + continue 'agentic; + } + *count += 1; + + // Append the model's function_call item to replay history. + raw_input_items.push(serde_json::json!({ + "type": "function_call", + "call_id": tool_use_id, + "name": name, + "arguments": raw_arguments, + })); + + let exec_start = std::time::Instant::now(); + let output_text = match tool.execute(ctx.clone(), input).await { + Ok(text) => text, + Err(e) => { + // Distinct from a legitimate empty result so the model + // can tell a tool failure from a zero-hit query. + warn!(tool = %name, error = %e, "function tool execution failed"); + "Tool call failed; answer without the tool result.".to_owned() + } + }; + debug!( + tool = %name, + elapsed_ms = exec_start.elapsed().as_millis() as u64, + "function tool executed" + ); + + raw_input_items.push(serde_json::json!({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output_text, + })); + + continue 'agentic; + } + + // Unrecognised tool while custom function tools ARE enabled. + // Models like gpt-oss emit their built-in browser actions + // (`open`, `find`, `search`) that we don't provide. Don't fail + // the turn — replay the call with an "unavailable" output so the + // model answers from what it already gathered (e.g. exa_search + // results). The agentic-iteration cap bounds a model that keeps + // ignoring the notice. + if !function_tools.is_empty() { + warn!( + tool = %name, + "unsupported tool requested; injecting unavailable notice and continuing" + ); + let raw_arguments = + serde_json::to_string(&input).unwrap_or_else(|_| "{}".to_owned()); + let notice = format!( + "The tool \"{name}\" is not available. Do not call it again. \ + Answer the user directly using the information already gathered." + ); + raw_input_items.push(serde_json::json!({ + "type": "function_call", + "call_id": tool_use_id, + "name": name, + "arguments": raw_arguments, + })); + raw_input_items.push(serde_json::json!({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": notice, + })); + continue 'agentic; + } + + // No custom tools enabled — genuinely unsupported. Fail the turn. warn!(tool = %name, "unexpected ToolUse outcome; finalizing as failed"); let code = "unexpected_tool_use".to_owned(); let message = "Provider requested an unsupported function tool".to_owned(); diff --git a/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs b/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs index b2c9cd134..15d24257f 100644 --- a/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs +++ b/gears/mini-chat/mini-chat/src/domain/service/test_helpers.rs @@ -502,6 +502,7 @@ pub fn test_catalog_entry(params: TestCatalogEntryParams) -> ModelCatalogEntry { }), system_prompt: String::new(), thread_summary_prompt: String::new(), + enabled_function_tools: Vec::new(), } } diff --git a/gears/mini-chat/mini-chat/src/gear.rs b/gears/mini-chat/mini-chat/src/gear.rs index 1eb977e75..d9ae157f2 100644 --- a/gears/mini-chat/mini-chat/src/gear.rs +++ b/gears/mini-chat/mini-chat/src/gear.rs @@ -77,6 +77,9 @@ struct OagwDeferred { authn: Arc, client_credentials: crate::config::ClientCredentialsConfig, providers: std::collections::HashMap, + /// Exa search config — registers its own OAGW upstream + `/search` route + /// in `start()` when enabled. + exa_search: crate::config::ExaSearchConfig, } /// State needed to build + start the outbox pipeline in `start()`. @@ -166,6 +169,9 @@ impl Gear for MiniChatGear { cfg.knowledge_search .validate() .map_err(|e| anyhow::anyhow!("knowledge_search config: {e}"))?; + cfg.exa_search + .validate() + .map_err(|e| anyhow::anyhow!("exa_search config: {e}"))?; let vendor = cfg.vendor.trim().to_owned(); if vendor.is_empty() { @@ -226,6 +232,11 @@ impl Gear for MiniChatGear { } } } + // Same host-as-alias fallback for the exa upstream (OAGW derives the + // alias from the hostname; real registration is deferred to start()). + if cfg.exa_search.enabled && cfg.exa_search.upstream_alias.is_none() { + cfg.exa_search.upstream_alias = Some(cfg.exa_search.host.clone()); + } // Save a copy for deferred OAGW registration in start(). // Ignore the result: if already set, we keep the first value. @@ -234,6 +245,7 @@ impl Gear for MiniChatGear { authn: Arc::clone(&authn_client), client_credentials: cfg.client_credentials.clone(), providers: cfg.providers.clone(), + exa_search: cfg.exa_search.clone(), })); let provider_resolver = Arc::new(ProviderResolver::new(&gateway, cfg.providers)); @@ -411,6 +423,34 @@ impl Gear for MiniChatGear { anthropic_files_client: anthropic_files_client.clone(), })); + // ── Custom function-tool registry ─────────────────────────────────── + // + // Each enabled tool is registered by name; per-model enablement is + // resolved per-request in StreamService against `enabled_function_tools`. + let mut function_tools: std::collections::HashMap< + String, + Arc, + > = std::collections::HashMap::new(); + if cfg.exa_search.enabled { + let alias = cfg + .exa_search + .upstream_alias + .clone() + .unwrap_or_else(|| cfg.exa_search.host.clone()); + let exa: Arc = Arc::new( + crate::infra::llm::providers::exa_search_tool::ExaSearchTool::new( + Arc::clone(&rag_client), + alias, + cfg.exa_search.search_type.clone(), + cfg.exa_search.num_results, + cfg.exa_search.max_calls_per_message, + cfg.exa_search.max_chars, + cfg.exa_search.guard.clone(), + ), + ); + function_tools.insert(exa.name().to_owned(), exa); + } + // ── Services ──────────────────────────────────────────────────────── let services = Arc::new(AppServices::new( @@ -434,6 +474,7 @@ impl Gear for MiniChatGear { cfg.thread_summary_worker, cfg.knowledge_search, knowledge_retriever, + function_tools, anthropic_files_client, )); @@ -511,6 +552,16 @@ impl RunnableCapability for MiniChatGear { &mut providers, ) .await?; + + // Register the exa upstream + /search route when enabled. + if deferred.exa_search.enabled { + crate::infra::oagw_provisioning::register_exa_upstream( + &deferred.gateway, + &ctx, + &deferred.exa_search, + ) + .await?; + } } // Start the outbox pipeline now that OAGW upstreams are registered. diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/exa_search_tool.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/exa_search_tool.rs new file mode 100644 index 000000000..4c7e0685d --- /dev/null +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/exa_search_tool.rs @@ -0,0 +1,339 @@ +//! Exa web-search function tool (exa.ai `/search`). +//! +//! Implements [`FunctionTool`] as `exa_search`: a generic web-search tool the +//! model can call when it needs current information. Egress goes through OAGW +//! (the apikey auth plugin injects `x-api-key`), reusing [`RagHttpClient`] as a +//! generic "JSON POST through OAGW" primitive. +//! +//! Request: `POST /{alias}/search` +//! `{ "query", "type", "numResults", "contents": { "highlights": true } }` +//! Response: `{ "results": [ { "title", "url", "highlights": [..] } ] }` +//! +//! Reference: + +use std::sync::Arc; + +use async_trait::async_trait; +use toolkit_security::SecurityContext; +use tracing::warn; + +use crate::domain::llm::LlmFunctionDef; +use crate::domain::ports::{FileStorageError, FunctionTool, FunctionToolError}; +use crate::infra::llm::providers::rag_http_client::RagHttpClient; + +/// `exa.ai` `/search` response envelope (subset we consume). +#[derive(serde::Deserialize)] +struct ExaSearchResponse { + #[serde(default)] + results: Vec, +} + +#[derive(serde::Deserialize)] +struct ExaResult { + #[serde(default)] + title: Option, + #[serde(default)] + url: Option, + #[serde(default)] + highlights: Vec, +} + +/// Web search via exa.ai, dispatched by the agentic loop as `exa_search`. +pub struct ExaSearchTool { + client: Arc, + /// Pre-resolved OAGW upstream alias for exa. + upstream_alias: String, + search_type: String, + num_results: u32, + max_calls: u32, + /// Maximum characters kept per formatted result (bounds context cost). + max_chars: usize, + guard: String, +} + +impl ExaSearchTool { + #[must_use] + pub fn new( + client: Arc, + upstream_alias: String, + search_type: String, + num_results: u32, + max_calls: u32, + max_chars: usize, + guard: String, + ) -> Self { + Self { + client, + upstream_alias, + search_type, + num_results, + max_calls, + max_chars, + guard, + } + } + + /// Format the exa results into a compact text block for `function_call_output`. + fn format_results(&self, resp: &ExaSearchResponse) -> String { + use std::fmt::Write as _; + + if resp.results.is_empty() { + return "No web results found.".to_owned(); + } + let mut out = String::new(); + for (i, r) in resp.results.iter().enumerate() { + let title = r.title.as_deref().unwrap_or("(untitled)"); + let url = r.url.as_deref().unwrap_or(""); + write!(out, "[{}] {title}\n{url}\n", i + 1).ok(); + if !r.highlights.is_empty() { + out.push_str(&r.highlights.join(" ... ")); + out.push('\n'); + } + out.push('\n'); + if out.len() >= self.max_chars { + break; + } + } + out.truncate(self.max_chars); + out + } +} + +fn map_err(e: FileStorageError) -> FunctionToolError { + match e { + FileStorageError::Rejected { message, .. } => FunctionToolError::Rejected(message), + FileStorageError::Unavailable { message } => FunctionToolError::Unavailable(message), + other => FunctionToolError::Configuration(other.to_string()), + } +} + +#[async_trait] +impl FunctionTool for ExaSearchTool { + // Trait signature ties the return to `&self`; the literal is `'static` but + // the impl must match the trait, so the suggested `&'static str` is moot. + #[allow(clippy::unnecessary_literal_bound)] + fn name(&self) -> &str { + "exa_search" + } + + fn definition(&self) -> LlmFunctionDef { + LlmFunctionDef { + name: "exa_search".to_owned(), + description: "Search the public web for current information. Use this when the \ + answer depends on recent events or facts that may not be in your \ + training data." + .to_owned(), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "A focused web search query." + } + }, + "required": ["query"] + }), + } + } + + fn system_prompt_guard(&self) -> Option { + if self.guard.is_empty() { + None + } else { + Some(self.guard.clone()) + } + } + + fn max_calls(&self) -> u32 { + self.max_calls + } + + async fn execute( + &self, + ctx: SecurityContext, + input: serde_json::Value, + ) -> Result { + let query = input.get("query").and_then(|v| v.as_str()).unwrap_or_default(); + if query.is_empty() { + return Err(FunctionToolError::Rejected( + "exa_search called without a 'query'".to_owned(), + )); + } + + let uri = format!("/{}/search", self.upstream_alias); + let body = serde_json::json!({ + "query": query, + "type": self.search_type, + "numResults": self.num_results, + "contents": { "highlights": true }, + }); + + let resp: ExaSearchResponse = + self.client.json_post(ctx, &uri, &body).await.map_err(|e| { + warn!(error = %e, "exa_search request failed"); + map_err(e) + })?; + + Ok(self.format_results(&resp)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tool() -> ExaSearchTool { + ExaSearchTool::new( + Arc::new(RagHttpClient::new(Arc::new(NoopOagw))), + "api.exa.ai".to_owned(), + "auto".to_owned(), + 10, + 3, + 2000, + "guard text".to_owned(), + ) + } + + // Minimal OAGW stub (never called — format_results is pure). + struct NoopOagw; + #[async_trait] + impl oagw_sdk::ServiceGatewayClientV1 for NoopOagw { + async fn create_upstream( + &self, + _: SecurityContext, + _: oagw_sdk::CreateUpstreamRequest, + ) -> Result { + unimplemented!() + } + async fn get_upstream( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result { + unimplemented!() + } + async fn list_upstreams( + &self, + _: SecurityContext, + _: &oagw_sdk::ListQuery, + ) -> Result, toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn update_upstream( + &self, + _: SecurityContext, + _: uuid::Uuid, + _: oagw_sdk::UpdateUpstreamRequest, + ) -> Result { + unimplemented!() + } + async fn delete_upstream( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result<(), toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn create_route( + &self, + _: SecurityContext, + _: oagw_sdk::CreateRouteRequest, + ) -> Result { + unimplemented!() + } + async fn get_route( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result { + unimplemented!() + } + async fn list_routes( + &self, + _: SecurityContext, + _: Option, + _: &oagw_sdk::ListQuery, + ) -> Result, toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn update_route( + &self, + _: SecurityContext, + _: uuid::Uuid, + _: oagw_sdk::UpdateRouteRequest, + ) -> Result { + unimplemented!() + } + async fn delete_route( + &self, + _: SecurityContext, + _: uuid::Uuid, + ) -> Result<(), toolkit_canonical_errors::CanonicalError> { + unimplemented!() + } + async fn resolve_proxy_target( + &self, + _: SecurityContext, + _: &str, + _: &str, + _: &str, + ) -> Result<(oagw_sdk::Upstream, oagw_sdk::Route), toolkit_canonical_errors::CanonicalError> + { + unimplemented!() + } + async fn proxy_request( + &self, + _: SecurityContext, + _: http::Request, + ) -> Result, toolkit_canonical_errors::CanonicalError> + { + unimplemented!() + } + } + + #[test] + fn definition_is_exa_search() { + let d = tool().definition(); + assert_eq!(d.name, "exa_search"); + assert_eq!(tool().name(), "exa_search"); + } + + #[test] + fn guard_exposed_from_config() { + assert_eq!(tool().system_prompt_guard().as_deref(), Some("guard text")); + } + + #[test] + fn format_empty_results() { + let resp = ExaSearchResponse { results: vec![] }; + assert_eq!(tool().format_results(&resp), "No web results found."); + } + + #[test] + fn format_results_includes_title_url_highlights() { + let resp = ExaSearchResponse { + results: vec![ExaResult { + title: Some("Rust 2.0 released".to_owned()), + url: Some("https://example.com/rust".to_owned()), + highlights: vec!["A big release".to_owned()], + }], + }; + let out = tool().format_results(&resp); + assert!(out.contains("Rust 2.0 released")); + assert!(out.contains("https://example.com/rust")); + assert!(out.contains("A big release")); + } + + #[test] + fn format_results_truncates_to_max_chars() { + let big = "x".repeat(5000); + let resp = ExaSearchResponse { + results: vec![ExaResult { + title: Some(big), + url: Some("https://e.com".to_owned()), + highlights: vec![], + }], + }; + assert!(tool().format_results(&resp).len() <= 2000); + } +} diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs index d8a506326..a6bc2407a 100644 --- a/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/mod.rs @@ -10,6 +10,7 @@ pub mod azure_file_storage; pub mod azure_knowledge_retriever; pub mod azure_vector_store; pub mod dispatching_storage; +pub mod exa_search_tool; pub mod openai_chat; pub mod openai_file_storage; pub mod openai_responses; diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs index eaefa06dc..29a53095d 100644 --- a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses.rs @@ -18,13 +18,14 @@ use std::sync::Arc; use bytes::Bytes; use futures::StreamExt; use oagw_sdk::error::StreamingError; -use oagw_sdk::sse::{ServerEventsResponse, ServerEventsStream}; +use oagw_sdk::sse::{FromServerEvent, ServerEvent, ServerEventsResponse, ServerEventsStream}; use oagw_sdk::{Body, ServiceGatewayClientV1}; +use serde::Deserialize; use tokio_util::sync::CancellationToken; use toolkit_security::SecurityContext; use tracing::debug; -use crate::infra::llm::request::ContentPart as MessageContentPart; +use crate::infra::llm::request::{ContentPart as MessageContentPart, LlmTool}; use crate::infra::llm::{ ClientSseEvent, LlmProviderError, LlmRequest, NonStreaming, ProviderStream, ResponseResult, Streaming, TranslatedEvent, @@ -35,6 +36,77 @@ use super::openai_responses::{ translate_provider_event, }; +// ════════════════════════════════════════════════════════════════════════════ +// Event-type resolution +// ════════════════════════════════════════════════════════════════════════════ + +/// Minimal projection used to recover the event type from the JSON payload. +#[derive(Deserialize)] +struct TypeTag { + r#type: String, +} + +/// `response.reasoning_text.delta` payload (gpt-oss chain-of-thought). +#[derive(Deserialize)] +struct ReasoningTextDeltaData { + #[serde(default)] + delta: String, +} + +/// vLLM-specific event, parsed from the upstream SSE stream. +/// +/// Two divergences from `OpenAI`'s Responses API are handled here so the shared +/// [`ProviderEvent`]/[`OpenAiResponsesProvider`](super::OpenAiResponsesProvider) +/// path stays spec-clean: +/// +/// 1. **No `event:` field.** vLLM omits the SSE `event:` line and carries the +/// type only inside the JSON payload's `type` field. Without compensating, +/// every frame would resolve to `"message"` → [`ProviderEvent::Unknown`], +/// so no deltas would be emitted and no terminal event captured — surfacing +/// to the client as "stream ended without terminal event". +/// 2. **Dedicated reasoning events.** gpt-oss streams chain-of-thought as +/// `response.reasoning_text.delta` events (not the `…` tags +/// the [`ThinkState`] machine handles for Qwen). The shared parser doesn't +/// model these, so they are captured here and surfaced as `"reasoning"` +/// deltas. +enum VllmProviderEvent { + /// A gpt-oss reasoning delta to surface as a `"reasoning"` SSE delta. + ReasoningDelta { delta: String }, + /// Any event understood by the shared Responses parser. + Shared(ProviderEvent), +} + +impl FromServerEvent for VllmProviderEvent { + fn from_server_event(event: ServerEvent) -> Result { + // Recover the missing `event:` field from the JSON payload `type`. + let event = if event.event.is_none() { + let r#type = serde_json::from_str::(&event.data) + .ok() + .map(|t| t.r#type); + ServerEvent { + event: r#type, + ..event + } + } else { + event + }; + + // Reasoning deltas aren't modelled by the shared parser — capture the + // text here so it can be rendered in the "Thinking" panel. + if event.event.as_deref() == Some("response.reasoning_text.delta") { + let data: ReasoningTextDeltaData = + serde_json::from_str(&event.data).map_err(|e| { + StreamingError::ServerEventsParse { + detail: format!("failed to parse reasoning delta: {e}"), + } + })?; + return Ok(VllmProviderEvent::ReasoningDelta { delta: data.delta }); + } + + ProviderEvent::from_server_event(event).map(VllmProviderEvent::Shared) + } +} + // ════════════════════════════════════════════════════════════════════════════ // Think-tag state machine // ════════════════════════════════════════════════════════════════════════════ @@ -232,7 +304,9 @@ fn strip_think_tags(text: &str) -> String { /// Compared to the `OpenAI` variant, this: /// - Uses plain string `content` for assistant messages (vLLM rejects the /// `output_text` array format). -/// - Omits tool definitions (`file_search`, `web_search`, `code_interpreter`). +/// - Serializes function tools (`LlmTool::Function`) so the model can call +/// custom tools like `exa_search`. Provider-hosted tools (`file_search`, +/// `web_search`, `code_interpreter`) are omitted — vLLM doesn't host them. /// - Omits `metadata` and `max_tool_calls`. fn build_request_body(request: &LlmRequest, stream: bool) -> serde_json::Value { let mut body = serde_json::json!({ @@ -294,8 +368,17 @@ fn build_request_body(request: &LlmRequest, stream: bool) -> serde_json::V }) .collect(); - if !input.is_empty() { - body["input"] = serde_json::Value::Array(input); + // Append raw input items (function_call + function_call_output) so the + // agentic loop can replay prior tool calls back to the model. + let combined_input = if request.raw_input_items.is_empty() { + input + } else { + let mut combined = input; + combined.extend(request.raw_input_items.iter().cloned()); + combined + }; + if !combined_input.is_empty() { + body["input"] = serde_json::Value::Array(combined_input); } if let Some(ref instructions) = request.system_instructions { @@ -333,6 +416,30 @@ fn build_request_body(request: &LlmRequest, stream: bool) -> serde_json::V } } + // Serialize function tools so the model can call them (e.g. `exa_search`). + // Provider-hosted tools (file/web/code) are skipped — vLLM doesn't host + // them, and sending unknown tool types risks an upstream error. + let tools: Vec = request + .tools + .iter() + .filter_map(|tool| match tool { + LlmTool::Function { + name, + description, + parameters, + } => Some(serde_json::json!({ + "type": "function", + "name": name, + "description": description, + "parameters": parameters + })), + _ => None, + }) + .collect(); + if !tools.is_empty() { + body["tools"] = serde_json::Value::Array(tools); + } + // Adapter-specific extras outside the typed surface. if let Some(ref extra) = request.additional_params && let (Some(body_obj), Some(extra_obj)) = (body.as_object_mut(), extra.as_object()) @@ -402,7 +509,7 @@ impl crate::infra::llm::LlmProvider for VllmResponsesProvider { let response = self.gateway.proxy_request(ctx, http_request).await?; - match ServerEventsStream::from_response::(response) { + match ServerEventsStream::from_response::(response) { ServerEventsResponse::Events(event_stream) => { // Scan state: (accumulated_text, think_state_machine). let translated = event_stream @@ -411,7 +518,19 @@ impl crate::infra::llm::LlmProvider for VllmResponsesProvider { |(accumulated, think), result| { let output: Vec> = match result { - Ok(event) => { + Ok(VllmProviderEvent::ReasoningDelta { delta }) => { + // Surfaced as a reasoning delta; not added to + // `accumulated` (which holds visible text only). + if delta.is_empty() { + vec![] + } else { + vec![Ok(TranslatedEvent::Sse(ClientSseEvent::Delta { + r#type: "reasoning", + content: delta, + }))] + } + } + Ok(VllmProviderEvent::Shared(event)) => { if let ProviderEvent::ResponseOutputTextDelta { ref delta } = event { diff --git a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs index 30b67f318..52114d8a4 100644 --- a/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs +++ b/gears/mini-chat/mini-chat/src/infra/llm/providers/vllm_responses_tests.rs @@ -174,9 +174,11 @@ fn assistant_content_is_plain_string() { } #[test] -fn tools_are_omitted_even_when_set() { +fn provider_hosted_tools_are_omitted() { use crate::domain::llm::{LlmTool, WebSearchContextSize}; + // vLLM doesn't host file_search/web_search/code_interpreter — these must + // be dropped so the upstream doesn't reject unknown tool types. let request = llm_request("test-model") .message(LlmMessage::user("Search")) .tool(LlmTool::WebSearch { @@ -190,9 +192,73 @@ fn tools_are_omitted_even_when_set() { .build_streaming(); let body = build_request_body(&request, true); + // No function tools present → `tools` omitted entirely. assert!(body.get("tools").is_none()); } +#[test] +fn function_tools_are_serialized() { + use crate::domain::llm::LlmTool; + + // Regression guard: function tools (e.g. `exa_search`) MUST reach the + // model, otherwise it can never call them. + let request = llm_request("test-model") + .message(LlmMessage::user("weather in Istanbul")) + .tool(LlmTool::Function { + name: "exa_search".into(), + description: "Search the web.".into(), + parameters: serde_json::json!({ + "type": "object", + "properties": { "query": { "type": "string" } }, + "required": ["query"] + }), + }) + // Provider-hosted tool alongside — must be filtered out. + .tool(LlmTool::WebSearch { + search_context_size: crate::domain::llm::WebSearchContextSize::Low, + }) + .build_streaming(); + + let body = build_request_body(&request, true); + let tools = body["tools"].as_array().expect("tools array present"); + assert_eq!(tools.len(), 1, "only the function tool should be serialized"); + assert_eq!(tools[0]["type"], "function"); + assert_eq!(tools[0]["name"], "exa_search"); + assert_eq!(tools[0]["description"], "Search the web."); + assert_eq!(tools[0]["parameters"]["required"][0], "query"); +} + +#[test] +fn raw_input_items_are_appended_to_input() { + // Agentic-loop replay: function_call / function_call_output items must be + // forwarded so the model sees its prior tool call and the result. + let request = llm_request("test-model") + .message(LlmMessage::user("weather in Istanbul")) + .raw_input_items(vec![ + serde_json::json!({ + "type": "function_call", + "name": "exa_search", + "arguments": "{\"query\":\"weather Istanbul\"}", + "call_id": "call_1" + }), + serde_json::json!({ + "type": "function_call_output", + "call_id": "call_1", + "output": "Sunny, 28°C" + }), + ]) + .build_streaming(); + + let body = build_request_body(&request, true); + let input = body["input"].as_array().unwrap(); + // 1 user message + 2 replay items. + assert_eq!(input.len(), 3); + assert_eq!(input[1]["type"], "function_call"); + assert_eq!(input[1]["name"], "exa_search"); + assert_eq!(input[2]["type"], "function_call_output"); + assert_eq!(input[2]["output"], "Sunny, 28°C"); +} + #[test] fn metadata_and_max_tool_calls_omitted_even_when_set() { use crate::infra::llm::request::{RequestMetadata, RequestType}; @@ -244,3 +310,93 @@ fn additional_params_are_merged() { assert_eq!(body["temperature"], 0.5); assert_eq!(body["top_p"], 0.9); } + +// ── VllmProviderEvent: event type carried in JSON `type` ────────────── +// vLLM omits the SSE `event:` line and puts the type only inside the JSON +// payload. The wrapper must recover it so events are recognised; otherwise +// the stream ends without a terminal event. + +#[test] +fn vllm_event_resolves_text_delta_from_payload_type() { + let event = ServerEvent { + event: None, + data: r#"{"type":"response.output_text.delta","delta":"Hello","item_id":"msg_1","sequence_number":4}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!( + matches!(result, ProviderEvent::ResponseOutputTextDelta { delta } if delta == "Hello"), + "payload `type` should be honoured when the SSE `event:` field is absent" + ); +} + +#[test] +fn vllm_event_resolves_completed_from_payload_type() { + let event = ServerEvent { + event: None, + data: r#"{"type":"response.completed","sequence_number":10,"response":{"id":"resp_1","output":[],"usage":{"input_tokens":1,"output_tokens":1}}}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!( + matches!(result, ProviderEvent::ResponseCompleted { response } if response.id == "resp_1"), + "terminal event must be recognised from the JSON `type` field" + ); +} + +#[test] +fn vllm_event_prefers_sse_event_field_when_present() { + // If the upstream does send `event:` (OpenAI shape), it wins. + let event = ServerEvent { + event: Some("response.output_text.delta".to_string()), + data: r#"{"type":"response.completed","delta":"Hi"}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!(matches!(result, ProviderEvent::ResponseOutputTextDelta { delta } if delta == "Hi")); +} + +#[test] +fn vllm_event_captures_reasoning_delta() { + // gpt-oss streams chain-of-thought as `response.reasoning_text.delta`. + let event = ServerEvent { + event: None, + data: r#"{"type":"response.reasoning_text.delta","content_index":0,"delta":"thinking…","item_id":"msg_1"}"#.to_string(), + id: None, + retry: None, + }; + let result = VllmProviderEvent::from_server_event(event).unwrap(); + assert!( + matches!(result, VllmProviderEvent::ReasoningDelta { delta } if delta == "thinking…"), + "reasoning deltas must be captured as a dedicated variant" + ); +} + +#[test] +fn vllm_event_unmodelled_lifecycle_resolves_to_unknown() { + // A lifecycle event vLLM emits but the adapter doesn't model must parse + // cleanly as Unknown, not error the stream. + let event = ServerEvent { + event: None, + data: r#"{"type":"response.reasoning_part.added","content_index":0}"#.to_string(), + id: None, + retry: None, + }; + let VllmProviderEvent::Shared(result) = VllmProviderEvent::from_server_event(event).unwrap() + else { + panic!("expected a shared ProviderEvent"); + }; + assert!(matches!(result, ProviderEvent::Unknown { .. })); +} diff --git a/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs b/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs index 44dd40651..86543b268 100644 --- a/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs +++ b/gears/mini-chat/mini-chat/src/infra/oagw_provisioning.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use oagw_sdk::ServiceGatewayClientV1; use tracing::{info, warn}; -use crate::config::ProviderEntry; +use crate::config::{ExaSearchConfig, ProviderEntry}; /// Register OAGW upstreams and routes for each configured provider. /// @@ -73,6 +73,75 @@ pub async fn register_oagw_upstreams( Ok(()) } +/// Register the OAGW upstream + `POST /search` route for the exa web-search tool. +/// +/// Mirrors [`create_upstream`]/[`register_route`] but for a standalone service +/// (no `ProviderEntry`): host + apikey auth from [`ExaSearchConfig`], a single +/// route for `POST {alias}/search`. +#[allow(clippy::cognitive_complexity)] +pub async fn register_exa_upstream( + gateway: &Arc, + ctx: &toolkit_security::SecurityContext, + cfg: &ExaSearchConfig, +) -> anyhow::Result<()> { + use oagw_sdk::{ + AuthConfig, CreateRouteRequest, CreateUpstreamRequest, Endpoint, HttpMatch, HttpMethod, + MatchRules, PathSuffixMode, Scheme, Server, + }; + + let server = Server { + endpoints: vec![Endpoint { + scheme: Scheme::Https, + host: cfg.host.clone(), + port: 443, + }], + }; + + let mut builder = + CreateUpstreamRequest::builder(server, "gts.cf.core.oagw.protocol.v1~cf.core.oagw.http.v1") + .enabled(true); + if let Some(alias) = &cfg.upstream_alias { + builder = builder.alias(alias); + } + if let (Some(plugin_type), Some(config)) = (&cfg.auth_plugin_type, &cfg.auth_config) { + builder = builder.auth(AuthConfig { + plugin_type: plugin_type.clone(), + sharing: oagw_sdk::SharingMode::Inherit, + config: Some(config.clone()), + }); + } + + let upstream = gateway + .create_upstream(ctx.clone(), builder.build()) + .await + .map_err(|e| anyhow::anyhow!("OAGW exa upstream registration failed: {e}"))?; + info!(alias = %upstream.alias, upstream_id = %upstream.id, "OAGW exa upstream registered"); + + let match_rules = MatchRules { + http: Some(HttpMatch { + methods: vec![HttpMethod::Post], + path: "/search".to_owned(), + query_allowlist: vec![], + path_suffix_mode: PathSuffixMode::Disabled, + }), + grpc: None, + }; + match gateway + .create_route( + ctx.clone(), + CreateRouteRequest::builder(upstream.id, match_rules) + .enabled(true) + .build(), + ) + .await + { + Ok(route) => info!(route_id = %route.id, "OAGW exa /search route registered"), + Err(e) => warn!(error = %e, "OAGW exa route registration failed (may already exist)"), + } + + Ok(()) +} + /// Create an OAGW upstream for a single provider entry. /// /// Only passes `upstream_alias` to OAGW when explicitly configured diff --git a/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs b/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs index 6de969187..1cf4b34fb 100644 --- a/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs +++ b/gears/mini-chat/mini-chat/src/infra/plugins/static_model_policy/tests.rs @@ -74,6 +74,7 @@ fn make_entry(model_id: &str, tier: ModelTier) -> ModelCatalogEntry { }), system_prompt: String::new(), thread_summary_prompt: String::new(), + enabled_function_tools: Vec::new(), } }