diff --git a/README.md b/README.md index 5ec022c..8519b79 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,8 @@ alone; one that exists without it is reported rather than overwritten. If the daemon step fails, the provider and agent are already saved and setup says so instead of unwinding. -To do it by hand instead, add an OpenAI-compatible provider and then the global -chat agent: +To do it by hand instead, add a provider and then the global chat agent. The +default provider kind is an OpenAI-compatible Chat Completions endpoint: ```sh ./out/mininaru provider add \ @@ -184,6 +184,45 @@ chat agent: The first agent becomes the global agent used by the interactive client. +Native Anthropic Messages API providers use `--kind anthropic`. Its base URL is +the API origin, without the OpenAI-style `/v1` suffix: + +```sh +mininaru provider add \ + --name anthropic \ + --kind anthropic \ + --base-url https://api.anthropic.com \ + --api-key '' \ + --cache ephemeral + +mininaru agent add --name claude --provider anthropic --model claude-sonnet-4-6 +``` + +`--cache` accepts `auto`, `off`, `ephemeral`, or `ephemeral_1h`. `auto` leaves +OpenAI and other provider-native automatic caches alone, enables Anthropic's +automatic five-minute cache, and adds the same cache control for Claude models +routed through OpenRouter. The explicit ephemeral modes add top-level +`cache_control`; use `ephemeral_1h` only when the longer, more expensive cache +write is worthwhile. + +OpenRouter can also cache an entire identical API response. This is separate +from prompt caching and is deliberately opt-in because it can return a previous +answer without running the model: + +```sh +mininaru provider add \ + --name openrouter \ + --base-url https://openrouter.ai/api/v1 \ + --api-key '' \ + --cache auto \ + --response-cache \ + --response-cache-ttl 300 +``` + +Response caching stores request/response data temporarily and is unavailable +with OpenRouter account-level Zero Data Retention. Keep it off for sensitive or +time-dependent conversations. + ## Interactive prompts The commands that create or change something -- `provider add`, `provider @@ -322,11 +361,11 @@ Every model call made on a session's behalf is recorded against it, so total down by what spent it. `/usage` reports the running total inside the TUI. ``` -KIND PROMPT CACHED COMPLETION TOTAL -turn 120,411 94,208 8,204 128,615 -compaction 6,890 4,096 412 7,302 -subagent 31,204 24,576 2,110 33,314 -total 158,505 122,880 10,726 169,231 +KIND PROMPT CACHE READ CACHE WRITE COMPLETION TOTAL +turn 120,411 94,208 12,000 8,204 128,615 +compaction 6,890 4,096 1,024 412 7,302 +subagent 31,204 24,576 2,048 2,110 33,314 +total 158,505 122,880 15,072 10,726 169,231 ``` `turn` is the answers you asked for, `compaction` is the summarising above, and @@ -338,12 +377,12 @@ one, and each is billed, so its total is the sum of every round rather than the last. The same applies to what the HTTP API reports back: the `usage` in a response covers every round the server ran on the caller's behalf. -`CACHED` is the subset of prompt tokens the provider reports as cache hits. The -provider or inference server owns the actual prompt cache; mininaru keeps tool -definitions in a stable order so the shared prefix stays reusable, and records -`prompt_tokens_details.cached_tokens` when the OpenAI-compatible endpoint returns -it. A blank or zero value means the provider did not report a hit, not that -mininaru maintains a second local model cache. +`CACHE READ` is the subset of prompt tokens served from a provider cache, while +`CACHE WRITE` is the number written into a new cache entry. OpenAI-compatible +providers report these as `cached_tokens` and `cache_write_tokens`; Anthropic +reports `cache_read_input_tokens` and `cache_creation_input_tokens`. A zero +means the provider reported no cache activity (or the prompt was shorter than +that model's cache minimum), not that mininaru maintains a second local cache. **These are tokens, not money.** mininaru talks to whatever provider you point it at and has no idea what yours charges, so the conversion is yours to do. Usage diff --git a/cli/preference.go b/cli/preference.go index 9029203..04b065c 100644 --- a/cli/preference.go +++ b/cli/preference.go @@ -15,6 +15,10 @@ var ( providerNameRef string providerApiKeyRef string providerBaseURLRef string + providerKindRef string + providerCacheRef string + providerRespCache bool + providerRespTTL int agentNameRef string agentRoleRef string @@ -29,10 +33,10 @@ var ( var provider *cobra.Command = &cobra.Command{ Use: "provider", Short: "manage LLM providers", - Long: `Manage the OpenAI compatible endpoints agents talk to. + Long: `Manage the OpenAI-compatible or native Anthropic endpoints agents talk to. -A provider bundles a base URL and an API key. Agents pick a provider when they -are created, and new agents fall back to the default one.`, +A provider bundles an API kind, base URL, API key, and cache policy. Agents pick +a provider when they are created, and new agents fall back to the default one.`, Example: ` mininaru provider add --name openai --base-url https://api.openai.com/v1 mininaru provider list mininaru provider default openai`, @@ -245,9 +249,12 @@ func providerAddExecute(cmd *cobra.Command, args []string) error { } payload = core.Provider{ - Name: providerNameRef, - ApiKey: providerApiKeyRef, - BaseURL: providerBaseURLRef, + Name: providerNameRef, ApiKey: providerApiKeyRef, BaseURL: providerBaseURLRef, + Kind: providerKindRef, Cache: providerCacheRef, ResponseCache: providerRespCache, ResponseCacheTTL: providerRespTTL, + } + err = core.ProviderValidate(payload) + if err != nil { + return err } core.ProviderCreate(payload) @@ -278,7 +285,7 @@ func providerListExecute(cmd *cobra.Command, args []string) error { return nil } - rows = uiTable("ID", "NAME", "BASE URL", "API KEY", "") + rows = uiTable("ID", "NAME", "KIND", "CACHE", "RESPONSE CACHE", "BASE URL", "API KEY", "") for _, cur = range core.Providers { mark = "" @@ -286,7 +293,7 @@ func providerListExecute(cmd *cobra.Command, args []string) error { mark = "[default]" } - rows.row(cur.Id, cur.Name, cur.BaseURL, maskSecret(cur.ApiKey), mark) + rows.row(cur.Id, cur.Name, cur.ProviderKind(), cur.CachePolicy(), strconv.FormatBool(cur.ResponseCache), cur.BaseURL, maskSecret(cur.ApiKey), mark) } rows.flush() @@ -338,18 +345,21 @@ func providerUpdateAsk(current *core.Provider) error { func providerUpdateExecute(cmd *cobra.Command, args []string) error { var touched bool var current *core.Provider - var name, apiKey, baseURL *string + var name, apiKey, baseURL, kind, cache *string + var responseCache *bool + var responseTTL *int var err error - touched = cmd.Flags().Changed("name") || cmd.Flags().Changed("api-key") || cmd.Flags().Changed("base-url") + current, err = core.ProviderFind(args[0]) + if err != nil { + return err + } - if !touched && askInteractive() { - current, err = core.ProviderFind(args[0]) - if err != nil { - return err - } + touched = cmd.Flags().Changed("name") || cmd.Flags().Changed("api-key") || cmd.Flags().Changed("base-url") || + cmd.Flags().Changed("kind") || cmd.Flags().Changed("cache") || cmd.Flags().Changed("response-cache") || cmd.Flags().Changed("response-cache-ttl") + if !touched && askInteractive() { err = providerUpdateAsk(current) if err != nil { return err @@ -371,8 +381,20 @@ func providerUpdateExecute(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("base-url") { baseURL = &providerBaseURLRef } + if cmd.Flags().Changed("kind") { + kind = &providerKindRef + } + if cmd.Flags().Changed("cache") { + cache = &providerCacheRef + } + if cmd.Flags().Changed("response-cache") { + responseCache = &providerRespCache + } + if cmd.Flags().Changed("response-cache-ttl") { + responseTTL = &providerRespTTL + } - return core.ProviderUpdateFields(args[0], name, apiKey, baseURL) + return core.ProviderUpdateConfig(current.Id, name, apiKey, baseURL, kind, cache, responseCache, responseTTL) } func providerRemoveExecute(cmd *cobra.Command, args []string) error { @@ -815,14 +837,14 @@ func sessionUsageExecute(cmd *cobra.Command, args []string) error { return nil } - rows = uiTable("KIND", "PROMPT", "CACHED", "COMPLETION", "TOTAL") + rows = uiTable("KIND", "PROMPT", "CACHE READ", "CACHE WRITE", "COMPLETION", "TOTAL") for _, line = range totals.Lines { - rows.row(line.Kind, tokenCount(line.PromptTokens), tokenCount(line.CachedTokens), tokenCount(line.CompletionTokens), + rows.row(line.Kind, tokenCount(line.PromptTokens), tokenCount(line.CachedTokens), tokenCount(line.CacheWriteTokens), tokenCount(line.CompletionTokens), tokenCount(line.TotalTokens)) } - rows.row("total", tokenCount(totals.PromptTokens), tokenCount(totals.CachedTokens), tokenCount(totals.CompletionTokens), + rows.row("total", tokenCount(totals.PromptTokens), tokenCount(totals.CachedTokens), tokenCount(totals.CacheWriteTokens), tokenCount(totals.CompletionTokens), tokenCount(totals.TotalTokens)) rows.flush() @@ -873,10 +895,18 @@ func init() { providerAdd.Flags().StringVarP(&providerNameRef, "name", "n", "", "provider name") providerAdd.Flags().StringVarP(&providerApiKeyRef, "api-key", "k", "", "provider api key") providerAdd.Flags().StringVarP(&providerBaseURLRef, "base-url", "b", "", "provider base url") + providerAdd.Flags().StringVar(&providerKindRef, "kind", core.ProviderOpenAI, "provider API kind (openai or anthropic)") + providerAdd.Flags().StringVar(&providerCacheRef, "cache", core.CacheAuto, "prompt cache policy (auto, off, ephemeral, or ephemeral_1h)") + providerAdd.Flags().BoolVar(&providerRespCache, "response-cache", false, "enable OpenRouter whole-response caching") + providerAdd.Flags().IntVar(&providerRespTTL, "response-cache-ttl", 0, "OpenRouter response cache TTL in seconds") providerUpdate.Flags().StringVarP(&providerNameRef, "name", "n", "", "provider name") providerUpdate.Flags().StringVarP(&providerApiKeyRef, "api-key", "k", "", "provider api key") providerUpdate.Flags().StringVarP(&providerBaseURLRef, "base-url", "b", "", "provider base url") + providerUpdate.Flags().StringVar(&providerKindRef, "kind", "", "provider API kind (openai or anthropic)") + providerUpdate.Flags().StringVar(&providerCacheRef, "cache", "", "prompt cache policy (auto, off, ephemeral, or ephemeral_1h)") + providerUpdate.Flags().BoolVar(&providerRespCache, "response-cache", false, "enable OpenRouter whole-response caching") + providerUpdate.Flags().IntVar(&providerRespTTL, "response-cache-ttl", 0, "OpenRouter response cache TTL in seconds") provider.AddCommand(providerAdd, providerList, providerUpdate, providerRemove, providerDefault) diff --git a/cli/tui/client.go b/cli/tui/client.go index 5e435c0..2535b8f 100644 --- a/cli/tui/client.go +++ b/cli/tui/client.go @@ -625,8 +625,8 @@ func (c *client) usageCommand() tea.Cmd { return nil } - notice = fmt.Sprintf("%d tokens this session (%d prompt, %d completion, %d cached)", - totals.TotalTokens, totals.PromptTokens, totals.CompletionTokens, totals.CachedTokens) + notice = fmt.Sprintf("%d tokens this session (%d prompt, %d completion, %d cache read, %d cache write)", + totals.TotalTokens, totals.PromptTokens, totals.CompletionTokens, totals.CachedTokens, totals.CacheWriteTokens) if totals.TotalTokens == 0 { notice = "no token usage recorded for this session yet" diff --git a/core/agent.go b/core/agent.go index f0a37c7..351bb2f 100644 --- a/core/agent.go +++ b/core/agent.go @@ -15,10 +15,12 @@ import ( "sync" "time" + "github.com/anthropics/anthropic-sdk-go" + anthropicoption "github.com/anthropics/anthropic-sdk-go/option" "github.com/devproje/mininaru/util" "github.com/google/uuid" "github.com/openai/openai-go" - "github.com/openai/openai-go/option" + openaioption "github.com/openai/openai-go/option" ) type NaruAgent struct { @@ -29,7 +31,8 @@ type NaruAgent struct { Model string `json:"model"` ProviderId string `json:"provider_id"` - AI *openai.Client `json:"-"` + AI *openai.Client `json:"-"` + Anthropic *anthropic.Client `json:"-"` } var modelContextWindows sync.Map @@ -92,7 +95,7 @@ func (a *NaruAgent) ModelContextWindow(ctx context.Context) int64 { return 0 } provider, err = ProviderFind(a.ProviderId) - if err != nil || provider.BaseURL == "" { + if err != nil || provider.BaseURL == "" || provider.ProviderKind() == ProviderAnthropic { return 0 } cacheKey = a.modelContextCacheKey() @@ -152,7 +155,7 @@ var Agents []*NaruAgent var emptyAgentObj AgentConfig = AgentConfig{} func newClient(prov *Provider) *openai.Client { - var opts []option.RequestOption + var opts []openaioption.RequestOption var ai openai.Client if prov == nil { @@ -160,11 +163,18 @@ func newClient(prov *Provider) *openai.Client { } if prov.ApiKey != "" { - opts = append(opts, option.WithAPIKey(prov.ApiKey)) + opts = append(opts, openaioption.WithAPIKey(prov.ApiKey)) } if prov.BaseURL != "" { - opts = append(opts, option.WithBaseURL(prov.BaseURL)) + opts = append(opts, openaioption.WithBaseURL(prov.BaseURL)) + } + + if prov.ResponseCache && isOpenRouter(prov.BaseURL) { + opts = append(opts, openaioption.WithHeader("X-OpenRouter-Cache", "true")) + if prov.ResponseCacheTTL > 0 { + opts = append(opts, openaioption.WithHeader("X-OpenRouter-Cache-TTL", fmt.Sprintf("%d", prov.ResponseCacheTTL))) + } } ai = openai.NewClient(opts...) @@ -172,6 +182,39 @@ func newClient(prov *Provider) *openai.Client { return &ai } +func newAnthropicClient(prov *Provider) *anthropic.Client { + var opts []anthropicoption.RequestOption + var ai anthropic.Client + + if prov == nil || prov.ProviderKind() != ProviderAnthropic { + return nil + } + if prov.ApiKey != "" { + opts = append(opts, anthropicoption.WithAPIKey(prov.ApiKey)) + } + if prov.BaseURL != "" { + opts = append(opts, anthropicoption.WithBaseURL(prov.BaseURL)) + } + + ai = anthropic.NewClient(opts...) + return &ai +} + +func configureAgentClients(agent *NaruAgent, prov *Provider) { + if agent == nil { + return + } + + if prov != nil && prov.ProviderKind() == ProviderAnthropic { + agent.AI = nil + agent.Anthropic = newAnthropicClient(prov) + return + } + + agent.AI = newClient(prov) + agent.Anthropic = nil +} + func AgentNew(name, role, soul, model string, prov *Provider) *NaruAgent { var agent NaruAgent @@ -186,9 +229,8 @@ func AgentNew(name, role, soul, model string, prov *Provider) *NaruAgent { Soul: soul, Model: model, ProviderId: prov.Id, - - AI: newClient(prov), } + configureAgentClients(&agent, prov) return &agent } @@ -238,11 +280,11 @@ func AgentInit() error { Agents = cfg.Agents if Global != nil { - Global.AI = newClient(agentProvider(Global)) + configureAgentClients(Global, agentProvider(Global)) } for _, cur = range Agents { - cur.AI = newClient(agentProvider(cur)) + configureAgentClients(cur, agentProvider(cur)) } return nil @@ -412,7 +454,7 @@ func AgentRefreshClient(agent *NaruAgent) error { return err } - agent.AI = newClient(prov) + configureAgentClients(agent, prov) return nil } @@ -449,7 +491,7 @@ func AgentUpdateFields(id string, name, role, soul, model, providerId *string) e if providerId != nil { update.ProviderId = *providerId - update.AI = newClient(agentProvider(&update)) + configureAgentClients(&update, agentProvider(&update)) } Agents[index] = &update diff --git a/core/anthropic.go b/core/anthropic.go new file mode 100644 index 0000000..8ec6c40 --- /dev/null +++ b/core/anthropic.go @@ -0,0 +1,338 @@ +// SPDX-FileCopyrightText: 2026 Wonhyeok Kim (Project_IO) +// SPDX-License-Identifier: GPL-3.0-or-later + +package core + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/anthropics/anthropic-sdk-go/packages/param" + "github.com/anthropics/anthropic-sdk-go/packages/ssestream" + "github.com/devproje/mininaru/modules" + "github.com/openai/openai-go" +) + +type openAIWireMessage struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + ToolCallID string `json:"tool_call_id"` + ToolCalls []struct { + ID string `json:"id"` + Function struct { + Name string `json:"name"` + Arguments string `json:"arguments"` + } `json:"function"` + } `json:"tool_calls"` +} + +type openAIWireContent struct { + Type string `json:"type"` + Text string `json:"text"` + ImageURL struct { + URL string `json:"url"` + } `json:"image_url"` +} + +func splitDataURL(value string) (string, string, error) { + var header, data string + var ok bool + + header, data, ok = strings.Cut(value, ",") + if !ok || !strings.HasPrefix(header, "data:") || !strings.HasSuffix(header, ";base64") { + return "", "", fmt.Errorf("unsupported image data URL") + } + + return strings.TrimSuffix(strings.TrimPrefix(header, "data:"), ";base64"), data, nil +} + +func anthropicContent(raw json.RawMessage) ([]anthropic.ContentBlockParamUnion, error) { + var text string + var parts []openAIWireContent + var blocks []anthropic.ContentBlockParamUnion + var part openAIWireContent + var media, data string + + var err error + + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + if raw[0] == '"' { + err = json.Unmarshal(raw, &text) + if err != nil { + return nil, err + } + return []anthropic.ContentBlockParamUnion{anthropic.NewTextBlock(text)}, nil + } + + err = json.Unmarshal(raw, &parts) + if err != nil { + return nil, err + } + for _, part = range parts { + switch part.Type { + case "text": + blocks = append(blocks, anthropic.NewTextBlock(part.Text)) + case "image_url": + if strings.HasPrefix(part.ImageURL.URL, "data:") { + media, data, err = splitDataURL(part.ImageURL.URL) + if err != nil { + return nil, err + } + blocks = append(blocks, anthropic.NewImageBlockBase64(media, data)) + } else { + blocks = append(blocks, anthropic.NewImageBlock(anthropic.URLImageSourceParam{URL: part.ImageURL.URL})) + } + } + } + + return blocks, nil +} + +func anthropicMessages(messages []openai.ChatCompletionMessageParamUnion) ([]anthropic.TextBlockParam, []anthropic.MessageParam, error) { + var message openai.ChatCompletionMessageParamUnion + var raw []byte + var wire openAIWireMessage + var blocks []anthropic.ContentBlockParamUnion + var block anthropic.ContentBlockParamUnion + var system []anthropic.TextBlockParam + var converted []anthropic.MessageParam + var result string + var input any + var call struct { + ID string + Name string + Argument string + } + var index int + + var err error + + for _, message = range messages { + raw, err = json.Marshal(message) + if err != nil { + return nil, nil, err + } + wire = openAIWireMessage{} + err = json.Unmarshal(raw, &wire) + if err != nil { + return nil, nil, err + } + + blocks, err = anthropicContent(wire.Content) + if err != nil { + return nil, nil, err + } + if wire.Role == "system" { + for _, block = range blocks { + if block.OfText != nil { + system = append(system, *block.OfText) + } + } + continue + } + if wire.Role == "tool" { + json.Unmarshal(wire.Content, &result) + converted = append(converted, anthropic.NewUserMessage(anthropic.NewToolResultBlock(wire.ToolCallID, result, false))) + continue + } + + for index = range wire.ToolCalls { + call.ID = wire.ToolCalls[index].ID + call.Name = wire.ToolCalls[index].Function.Name + call.Argument = wire.ToolCalls[index].Function.Arguments + input = map[string]any{} + if call.Argument != "" { + err = json.Unmarshal([]byte(call.Argument), &input) + if err != nil { + return nil, nil, err + } + } + blocks = append(blocks, anthropic.NewToolUseBlock(call.ID, input, call.Name)) + } + if wire.Role == "assistant" { + converted = append(converted, anthropic.NewAssistantMessage(blocks...)) + } else { + converted = append(converted, anthropic.NewUserMessage(blocks...)) + } + } + + return system, converted, nil +} + +func anthropicTools(defs []modules.Def) []anthropic.ToolUnionParam { + var def modules.Def + var properties any + var required []string + var valuesString []string + var valuesAny []any + var value any + var name string + var ok bool + var schema anthropic.ToolInputSchemaParam + var tool anthropic.ToolParam + var tools []anthropic.ToolUnionParam + + for _, def = range defs { + properties = def.Parameters["properties"] + required = nil + valuesString, ok = def.Parameters["required"].([]string) + if ok { + required = valuesString + } else { + valuesAny, ok = def.Parameters["required"].([]any) + if ok { + for _, value = range valuesAny { + name, ok = value.(string) + if ok { + required = append(required, name) + } + } + } + } + schema = anthropic.ToolInputSchemaParam{Properties: properties, Required: required} + tool = anthropic.ToolParam{Name: def.Name, Description: param.NewOpt(def.Description), InputSchema: schema} + tools = append(tools, anthropic.ToolUnionParam{OfTool: &tool}) + } + + return tools +} + +func anthropicCacheControl(policy string) anthropic.CacheControlEphemeralParam { + var control anthropic.CacheControlEphemeralParam + + if policy == CacheOff { + return control + } + + control = anthropic.NewCacheControlEphemeralParam() + if policy == CacheEphemeral1h { + control.TTL = anthropic.CacheControlEphemeralTTLTTL1h + } + + return control +} + +func (r *completionRun) anthropicStream(ctx context.Context, params anthropic.MessageNewParams) (*anthropic.Message, error) { + var stream *ssestream.Stream[anthropic.MessageStreamEventUnion] + var message anthropic.Message + var event anthropic.MessageStreamEventUnion + + var err error + + stream = r.Anthropic.Messages.NewStreaming(ctx, params) + + for stream.Next() { + event = stream.Current() + err = message.Accumulate(event) + if err != nil { + stream.Close() + return nil, err + } + if event.Type == "content_block_delta" && event.Delta.Type == "text_delta" && r.OnContent != nil { + r.OnContent(event.Delta.Text) + } + if event.Type == "content_block_delta" && event.Delta.Type == "thinking_delta" && r.OnReasoning != nil { + r.OnReasoning(event.Delta.Thinking) + } + } + + err = stream.Err() + stream.Close() + if err != nil { + return nil, err + } + + return &message, nil +} + +func (r *completionRun) executeAnthropic(ctx context.Context) (*Completion, error) { + var params anthropic.MessageNewParams + var system []anthropic.TextBlockParam + var messages []anthropic.MessageParam + var message *anthropic.Message + var result Completion + var assistantBlocks []anthropic.ContentBlockParamUnion + var toolResults []anthropic.ContentBlockParamUnion + var record *ToolCall + var call openai.ChatCompletionMessageToolCall + var input any + var round int + var block anthropic.ContentBlockUnion + + var err error + + system, messages, err = anthropicMessages(r.Params.Messages) + if err != nil { + return nil, err + } + params = anthropic.MessageNewParams{Model: anthropic.Model(r.Params.Model), MaxTokens: 8192, System: system, + Messages: messages, Tools: anthropicTools(r.Defs), CacheControl: anthropicCacheControl(r.Provider.CachePolicy())} + if r.Params.ReasoningEffort != "" { + params.Thinking = anthropic.ThinkingConfigParamUnion{OfAdaptive: &anthropic.ThinkingConfigAdaptiveParam{}} + } + + for round = 0; round < maxToolRounds; round++ { + result.Content = "" + message, err = r.anthropicStream(ctx, params) + if err != nil { + return nil, err + } + + result.Usage.PromptTokens += message.Usage.InputTokens + message.Usage.CacheReadInputTokens + message.Usage.CacheCreationInputTokens + result.Usage.CompletionTokens += message.Usage.OutputTokens + result.Usage.TotalTokens += message.Usage.InputTokens + message.Usage.CacheReadInputTokens + message.Usage.CacheCreationInputTokens + message.Usage.OutputTokens + result.Usage.CachedTokens += message.Usage.CacheReadInputTokens + result.Usage.CacheWriteTokens += message.Usage.CacheCreationInputTokens + result.ContextTokens = message.Usage.InputTokens + message.Usage.CacheReadInputTokens + message.Usage.CacheCreationInputTokens + + assistantBlocks = nil + toolResults = nil + for _, block = range message.Content { + switch block.Type { + case "text": + result.Content += block.Text + assistantBlocks = append(assistantBlocks, anthropic.NewTextBlock(block.Text)) + case "thinking": + result.Reasoning += block.Thinking + assistantBlocks = append(assistantBlocks, anthropic.NewThinkingBlock(block.Signature, block.Thinking)) + case "tool_use": + input = map[string]any{} + if len(block.Input) > 0 { + json.Unmarshal(block.Input, &input) + } + assistantBlocks = append(assistantBlocks, anthropic.NewToolUseBlock(block.ID, input, block.Name)) + call = openai.ChatCompletionMessageToolCall{ID: block.ID} + call.Function.Name = block.Name + call.Function.Arguments = string(block.Input) + record, err = toolCallStart(r.MessageId, call) + if err != nil { + return nil, err + } + if r.OnTool != nil { + r.OnTool(ToolEvent{Phase: ToolEventStarted, CallId: record.CallId, Name: record.Name, Arguments: record.Arguments, Status: record.Status}) + } + record, err = executeTool(ctx, r.SessionId, record, r.Defs, r.AllowDangerous, r.AllowPrivileged, r.Approve) + if err != nil { + return nil, err + } + if r.OnTool != nil { + r.OnTool(ToolEvent{Phase: ToolEventFinished, CallId: record.CallId, Name: record.Name, Arguments: record.Arguments, + Result: record.Result, Status: record.Status, Error: record.Error}) + } + toolResults = append(toolResults, anthropic.NewToolResultBlock(block.ID, record.Result, record.Status != MessageCompleted)) + } + } + if len(toolResults) == 0 { + return &result, nil + } + params.Messages = append(params.Messages, anthropic.NewAssistantMessage(assistantBlocks...)) + params.Messages = append(params.Messages, anthropic.NewUserMessage(toolResults...)) + } + + return nil, fmt.Errorf("tool call limit exceeded after %d rounds", maxToolRounds) +} diff --git a/core/cache_test.go b/core/cache_test.go new file mode 100644 index 0000000..285467b --- /dev/null +++ b/core/cache_test.go @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: 2026 Wonhyeok Kim (Project_IO) +// SPDX-License-Identifier: GPL-3.0-or-later + +package core + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/openai/openai-go" +) + +func TestOpenAICachePolicyAddsTopLevelControl(t *testing.T) { + var params openai.ChatCompletionNewParams + var raw []byte + var body map[string]any + var control map[string]any + + var err error + + params.Model = "anthropic/claude-sonnet-4" + params.Messages = []openai.ChatCompletionMessageParamUnion{openai.UserMessage("hello")} + applyOpenAICache(¶ms, &Provider{Cache: CacheEphemeral1h}) + + raw, err = json.Marshal(params) + if err != nil { + t.Fatal(err) + } + err = json.Unmarshal(raw, &body) + if err != nil { + t.Fatal(err) + } + control, _ = body["cache_control"].(map[string]any) + if control["type"] != "ephemeral" || control["ttl"] != "1h" { + t.Fatalf("cache_control = %#v", control) + } +} + +func TestOpenRouterClaudeAutoEnablesPromptCache(t *testing.T) { + var params openai.ChatCompletionNewParams + var raw []byte + + var err error + + params.Model = "anthropic/claude-sonnet-4" + params.Messages = []openai.ChatCompletionMessageParamUnion{openai.UserMessage("hello")} + applyOpenAICache(¶ms, &Provider{BaseURL: "https://openrouter.ai/api/v1", Cache: CacheAuto}) + + raw, err = json.Marshal(params) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), `"cache_control":{"type":"ephemeral"}`) { + t.Fatalf("request did not enable automatic Claude caching: %s", raw) + } +} + +func TestAnthropicStreamingReportsCacheUsage(t *testing.T) { + var srv *httptest.Server + var provider *Provider + var agent *NaruAgent + var result *Completion + var streamed strings.Builder + var requestBody map[string]any + var control map[string]any + + var err error + + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/messages" { + t.Errorf("request path = %q", r.URL.Path) + } + err = json.NewDecoder(r.Body).Decode(&requestBody) + if err != nil { + t.Errorf("decode request: %v", err) + } + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintln(w, `event: message_start`) + fmt.Fprintln(w, `data: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-test","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":20,"cache_creation_input_tokens":60,"cache_read_input_tokens":40,"output_tokens":0}}}`) + fmt.Fprintln(w) + fmt.Fprintln(w, `event: content_block_start`) + fmt.Fprintln(w, `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`) + fmt.Fprintln(w) + fmt.Fprintln(w, `event: content_block_delta`) + fmt.Fprintln(w, `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}`) + fmt.Fprintln(w) + fmt.Fprintln(w, `event: content_block_stop`) + fmt.Fprintln(w, `data: {"type":"content_block_stop","index":0}`) + fmt.Fprintln(w) + fmt.Fprintln(w, `event: message_delta`) + fmt.Fprintln(w, `data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5}}`) + fmt.Fprintln(w) + fmt.Fprintln(w, `event: message_stop`) + fmt.Fprintln(w, `data: {"type":"message_stop"}`) + fmt.Fprintln(w) + })) + defer srv.Close() + + provider = &Provider{Id: "anthropic-test", Name: "anthropic", Kind: ProviderAnthropic, BaseURL: srv.URL, ApiKey: "test", Cache: CacheEphemeral1h} + Providers = []*Provider{provider} + DefaultProvider = provider + t.Cleanup(func() { + Providers = nil + DefaultProvider = nil + }) + agent = AgentNew("claude", "", "", "claude-test", provider) + + result, err = Complete(context.Background(), agent, + []openai.ChatCompletionMessageParamUnion{openai.UserMessage("hello")}, nil, "", func(value string) { streamed.WriteString(value) }, nil) + if err != nil { + t.Fatal(err) + } + if result.Content != "hello" || streamed.String() != "hello" { + t.Fatalf("content = %q, streamed = %q", result.Content, streamed.String()) + } + if result.Usage.PromptTokens != 120 || result.Usage.CachedTokens != 40 || result.Usage.CacheWriteTokens != 60 || result.Usage.CompletionTokens != 5 { + t.Fatalf("usage = %+v", result.Usage) + } + control, _ = requestBody["cache_control"].(map[string]any) + if control["ttl"] != "1h" { + t.Fatalf("request cache_control = %#v", control) + } +} + +func TestResponseCacheRequiresOpenRouter(t *testing.T) { + var err error + + err = ProviderValidate(Provider{BaseURL: "https://api.openai.com/v1", ResponseCache: true}) + if err == nil || !strings.Contains(err.Error(), "OpenRouter") { + t.Fatalf("validation error = %v", err) + } +} diff --git a/core/chat.go b/core/chat.go index f0089fe..845b492 100644 --- a/core/chat.go +++ b/core/chat.go @@ -204,7 +204,7 @@ func chatWithToolPolicy(ctx context.Context, session *Session, agent *NaruAgent, if session == nil || agent == nil { return nil, fmt.Errorf("session and agent are required to chat") } - if agent.AI == nil { + if agent.AI == nil && agent.Anthropic == nil { return nil, fmt.Errorf("agent %s has no available provider client", agent.Id) } @@ -243,6 +243,7 @@ func chatWithToolPolicy(ctx context.Context, session *Session, agent *NaruAgent, Messages: messages, } params.StreamOptions.IncludeUsage = param.NewOpt(true) + applyOpenAICache(¶ms, agentProvider(agent)) if len(defs) > 0 { params.Tools = toolParams(defs) } @@ -257,7 +258,7 @@ func chatWithToolPolicy(ctx context.Context, session *Session, agent *NaruAgent, } run = completionRun{ - AI: agent.AI, Params: params, Defs: defs, AllowDangerous: allowDangerous, AllowPrivileged: true, + AI: agent.AI, Anthropic: agent.Anthropic, Provider: agentProvider(agent), Params: params, Defs: defs, AllowDangerous: allowDangerous, AllowPrivileged: true, AgentId: agent.Id, SessionId: session.Id, MessageId: pending.Id, OnContent: onContent, OnReasoning: onReasoning, OnTool: onTool, Approve: approve, diff --git a/core/compact.go b/core/compact.go index 5c12420..e3c57e3 100644 --- a/core/compact.go +++ b/core/compact.go @@ -104,7 +104,7 @@ func summaryTranscript(previous string, dropped []*Message) string { return builder.String() } -func summarize(ctx context.Context, agent *NaruAgent, previous string, dropped []*Message) (string, openai.CompletionUsage, error) { +func summarize(ctx context.Context, agent *NaruAgent, previous string, dropped []*Message) (string, TokenUsage, error) { var messages []openai.ChatCompletionMessageParamUnion var result *Completion var text string @@ -116,7 +116,7 @@ func summarize(ctx context.Context, agent *NaruAgent, previous string, dropped [ result, err = Complete(ctx, agent, messages, nil, config.ThinkingOff, nil, nil) if err != nil { - return "", openai.CompletionUsage{}, err + return "", TokenUsage{}, err } text = strings.TrimSpace(result.Content) @@ -138,7 +138,7 @@ func CompactNow(ctx context.Context, agent *NaruAgent, session *Session) (bool, var text string var tail []*Message var updated string - var usage openai.CompletionUsage + var usage TokenUsage var err error @@ -189,7 +189,7 @@ func compactHistory(ctx context.Context, agent *NaruAgent, session *Session, his var text string var tail []*Message var updated string - var usage openai.CompletionUsage + var usage TokenUsage var tokens int64 var window int64 var known bool diff --git a/core/complete.go b/core/complete.go index f9493bd..87e2833 100644 --- a/core/complete.go +++ b/core/complete.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" + "github.com/anthropics/anthropic-sdk-go" "github.com/devproje/mininaru/config" "github.com/devproje/mininaru/modules" "github.com/openai/openai-go" @@ -17,6 +18,8 @@ import ( type completionRun struct { AI *openai.Client + Anthropic *anthropic.Client + Provider *Provider Params openai.ChatCompletionNewParams Defs []modules.Def AllowDangerous bool @@ -31,12 +34,13 @@ type completionRun struct { OnReasoning func(string) OnTool ToolEventFunc Approve ToolApprovalFunc + cacheWrites int64 } type Completion struct { Content string Reasoning string - Usage openai.CompletionUsage + Usage TokenUsage ContextTokens int64 } @@ -55,6 +59,7 @@ func (r *completionRun) stream(ctx context.Context, reply, reasoning *strings.Bu chunk = stream.Current() accumulator.AddChunk(chunk) accumulator.Usage.PromptTokensDetails.CachedTokens += chunk.Usage.PromptTokensDetails.CachedTokens + r.cacheWrites += cacheWriteTokens(chunk.Usage) if len(chunk.Choices) == 0 { continue } @@ -145,12 +150,16 @@ func (r *completionRun) execute(ctx context.Context) (*Completion, error) { var err error + if r.Anthropic != nil { + return r.executeAnthropic(ctx) + } if r.AI == nil { return nil, fmt.Errorf("no available provider client") } for round = 0; round < maxToolRounds; round++ { reply.Reset() + r.cacheWrites = 0 accumulator, err = r.stream(ctx, &reply, &reasoning) if err != nil { @@ -160,7 +169,8 @@ func (r *completionRun) execute(ctx context.Context) (*Completion, error) { result.Usage.PromptTokens += accumulator.Usage.PromptTokens result.Usage.CompletionTokens += accumulator.Usage.CompletionTokens result.Usage.TotalTokens += accumulator.Usage.TotalTokens - result.Usage.PromptTokensDetails.CachedTokens += accumulator.Usage.PromptTokensDetails.CachedTokens + result.Usage.CachedTokens += accumulator.Usage.PromptTokensDetails.CachedTokens + result.Usage.CacheWriteTokens += r.cacheWrites result.ContextTokens = accumulator.Usage.PromptTokens message = accumulator.Choices[0].Message @@ -188,7 +198,7 @@ func Complete(ctx context.Context, agent *NaruAgent, messages []openai.ChatCompl if agent == nil { return nil, fmt.Errorf("agent is required to complete") } - if agent.AI == nil { + if agent.AI == nil && agent.Anthropic == nil { return nil, fmt.Errorf("agent %s has no available provider client", agent.Id) } if len(messages) == 0 { @@ -202,6 +212,7 @@ func Complete(ctx context.Context, agent *NaruAgent, messages []openai.ChatCompl params.Model = agent.Model params.StreamOptions.IncludeUsage = param.NewOpt(true) + applyOpenAICache(¶ms, agentProvider(agent)) if len(defs) > 0 { params.Tools = toolParams(defs) @@ -211,7 +222,7 @@ func Complete(ctx context.Context, agent *NaruAgent, messages []openai.ChatCompl params.ReasoningEffort = openai.ReasoningEffort(thinking) } - run = completionRun{AI: agent.AI, Params: params, Defs: defs, AgentId: agent.Id, + run = completionRun{AI: agent.AI, Anthropic: agent.Anthropic, Provider: agentProvider(agent), Params: params, Defs: defs, AgentId: agent.Id, OnContent: onContent, OnReasoning: onReasoning} return run.execute(ctx) diff --git a/core/provider.go b/core/provider.go index c123475..fb915e8 100644 --- a/core/provider.go +++ b/core/provider.go @@ -6,17 +6,24 @@ package core import ( "encoding/json" "fmt" + "net/url" "os" + "strings" "github.com/devproje/mininaru/util" "github.com/google/uuid" + "github.com/openai/openai-go" ) type Provider struct { - Id string `json:"id"` - Name string `json:"name"` - ApiKey string `json:"api_key"` - BaseURL string `json:"base_url"` + Id string `json:"id"` + Name string `json:"name"` + ApiKey string `json:"api_key"` + BaseURL string `json:"base_url"` + Kind string `json:"kind,omitempty"` + Cache string `json:"cache,omitempty"` + ResponseCache bool `json:"response_cache,omitempty"` + ResponseCacheTTL int `json:"response_cache_ttl,omitempty"` } type ProviderConfig struct { @@ -24,13 +31,101 @@ type ProviderConfig struct { Providers []*Provider `json:"providers"` } -const PROVIDER_PATH = "provider.json" +const ( + PROVIDER_PATH = "provider.json" + + ProviderOpenAI = "openai" + ProviderAnthropic = "anthropic" + + CacheAuto = "auto" + CacheOff = "off" + CacheEphemeral = "ephemeral" + CacheEphemeral1h = "ephemeral_1h" +) var Providers []*Provider var DefaultProvider *Provider var emptyProviderObj ProviderConfig = ProviderConfig{Providers: []*Provider{}} +func (p *Provider) ProviderKind() string { + if p != nil && p.Kind == ProviderAnthropic { + return ProviderAnthropic + } + + return ProviderOpenAI +} + +func (p *Provider) CachePolicy() string { + if p == nil || p.Cache == "" { + return CacheAuto + } + + return p.Cache +} + +func isOpenRouter(baseURL string) bool { + var parsed *url.URL + var host string + + var err error + + parsed, err = url.Parse(baseURL) + if err != nil { + return false + } + host = strings.ToLower(parsed.Hostname()) + + return host == "openrouter.ai" || strings.HasSuffix(host, ".openrouter.ai") +} + +func applyOpenAICache(params *openai.ChatCompletionNewParams, provider *Provider) { + var control map[string]any + var policy string + + if params == nil || provider == nil { + return + } + policy = provider.CachePolicy() + if policy == CacheAuto && isOpenRouter(provider.BaseURL) && strings.HasPrefix(params.Model, "anthropic/") { + policy = CacheEphemeral + } + if policy != CacheEphemeral && policy != CacheEphemeral1h { + return + } + + control = map[string]any{"type": "ephemeral"} + if policy == CacheEphemeral1h { + control["ttl"] = "1h" + } + params.SetExtraFields(map[string]any{"cache_control": control}) +} + +func ProviderValidate(provider Provider) error { + var kind string + var cache string + + kind = provider.Kind + if kind == "" { + kind = ProviderOpenAI + } + cache = provider.CachePolicy() + if kind != ProviderOpenAI && kind != ProviderAnthropic { + return fmt.Errorf("unsupported provider kind %q", provider.Kind) + } + if cache != CacheAuto && cache != CacheOff && cache != CacheEphemeral && cache != CacheEphemeral1h { + return fmt.Errorf("unsupported cache policy %q", provider.Cache) + } + if provider.ResponseCache && !isOpenRouter(provider.BaseURL) { + return fmt.Errorf("response caching is only supported for OpenRouter providers") + } + if provider.ResponseCacheTTL < 0 || provider.ResponseCacheTTL > 86400 { + return fmt.Errorf("response cache TTL must be 0 or between 1 and 86400 seconds") + } + + return nil +} + func ProviderFind(ref string) (*Provider, error) { var cur *Provider @@ -61,6 +156,7 @@ func ProviderInit() error { var path string var buf []byte var cfg ProviderConfig + var provider *Provider var err error @@ -88,6 +184,12 @@ func ProviderInit() error { } Providers = cfg.Providers + for _, provider = range Providers { + err = ProviderValidate(*provider) + if err != nil { + return fmt.Errorf("provider %s: %w", provider.Name, err) + } + } DefaultProvider = nil if cfg.DefaultId != "" { @@ -151,48 +253,59 @@ func ProviderDefault(ref string) error { return ProviderSave() } -func ProviderUpdateFields(id string, name, apiKey, baseURL *string) error { +func ProviderUpdateConfig(id string, name, apiKey, baseURL, kind, cache *string, responseCache *bool, responseCacheTTL *int) error { var index int - var cur *Provider + var current *Provider var update Provider var err error - for index, cur = range Providers { - if cur.Id != id { + for index, current = range Providers { + if current.Id != id && current.Name != id { continue } - - update = *cur - + update = *current if name != nil { update.Name = *name } - if apiKey != nil { update.ApiKey = *apiKey } - if baseURL != nil { update.BaseURL = *baseURL } - - if DefaultProvider == cur { - DefaultProvider = &update + if kind != nil { + update.Kind = *kind } - - Providers[index] = &update - err = ProviderSave() + if cache != nil { + update.Cache = *cache + } + if responseCache != nil { + update.ResponseCache = *responseCache + } + if responseCacheTTL != nil { + update.ResponseCacheTTL = *responseCacheTTL + } + err = ProviderValidate(update) if err != nil { return err } - - return nil + if DefaultProvider == current { + DefaultProvider = &update + } + Providers[index] = &update + return ProviderSave() } - err = fmt.Errorf("cannot find provider id for %s", id) + return fmt.Errorf("cannot find provider id for %s", id) +} + +func ProviderUpdateOptions(id string, kind, cache *string, responseCache *bool, responseCacheTTL *int) error { + return ProviderUpdateConfig(id, nil, nil, nil, kind, cache, responseCache, responseCacheTTL) +} - return err +func ProviderUpdateFields(id string, name, apiKey, baseURL *string) error { + return ProviderUpdateConfig(id, name, apiKey, baseURL, nil, nil, nil, nil) } func ProviderUpdate(id string, payload Provider) error { diff --git a/core/subagent.go b/core/subagent.go index 7f1e20e..a978eab 100644 --- a/core/subagent.go +++ b/core/subagent.go @@ -75,13 +75,14 @@ func runSubagent(ctx context.Context, policy subagentPolicy, target *NaruAgent, params.StreamOptions.IncludeUsage = param.NewOpt(true) params.Messages = append(params.Messages, openai.SystemMessage(systemPrompt(target, defs))) params.Messages = append(params.Messages, openai.UserMessage(prompt)) + applyOpenAICache(¶ms, agentProvider(target)) if len(defs) > 0 { params.Tools = toolParams(defs) } run = completionRun{ - AI: target.AI, Params: params, Defs: defs, + AI: target.AI, Anthropic: target.Anthropic, Provider: agentProvider(target), Params: params, Defs: defs, AllowDangerous: policy.AllowDangerous, AllowPrivileged: policy.AllowPrivileged, AgentId: target.Id, SessionId: policy.SessionId, Depth: policy.Depth + 1, Approve: policy.Approve, diff --git a/core/usage.go b/core/usage.go index 662064b..d149f08 100644 --- a/core/usage.go +++ b/core/usage.go @@ -5,18 +5,29 @@ package core import ( "database/sql" + "strconv" "github.com/devproje/mininaru/util" "github.com/google/uuid" "github.com/openai/openai-go" + "github.com/openai/openai-go/packages/respjson" ) +type TokenUsage struct { + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + TotalTokens int64 `json:"total_tokens"` + CachedTokens int64 `json:"cached_tokens"` + CacheWriteTokens int64 `json:"cache_write_tokens"` +} + type UsageLine struct { Kind string `json:"kind"` PromptTokens int64 `json:"prompt_tokens"` CompletionTokens int64 `json:"completion_tokens"` TotalTokens int64 `json:"total_tokens"` CachedTokens int64 `json:"cached_tokens"` + CacheWriteTokens int64 `json:"cache_write_tokens"` } type UsageTotals struct { @@ -26,6 +37,7 @@ type UsageTotals struct { CompletionTokens int64 `json:"completion_tokens"` TotalTokens int64 `json:"total_tokens"` CachedTokens int64 `json:"cached_tokens"` + CacheWriteTokens int64 `json:"cache_write_tokens"` } const ( @@ -34,7 +46,34 @@ const ( UsageSubagent = "subagent" ) -func usageRecordWithContext(sessionId, messageId, kind string, usage openai.CompletionUsage, contextTokens, contextWindow int64) { +func cacheWriteTokens(usage openai.CompletionUsage) int64 { + var raw string + var value int64 + var field respjson.Field + var ok bool + + var err error + + if usage.PromptTokensDetails.JSON.ExtraFields == nil { + return 0 + } + field, ok = usage.PromptTokensDetails.JSON.ExtraFields["cache_write_tokens"] + if ok { + raw = field.Raw() + } + if raw == "" { + return 0 + } + + value, err = strconv.ParseInt(raw, 10, 64) + if err != nil { + return 0 + } + + return value +} + +func usageRecordWithContext(sessionId, messageId, kind string, usage TokenUsage, contextTokens, contextWindow int64) { var err error if sessionId == "" || usage.TotalTokens == 0 { @@ -42,18 +81,18 @@ func usageRecordWithContext(sessionId, messageId, kind string, usage openai.Comp } _, err = util.DB.Exec(`INSERT INTO token_usage - (id, session_id, message_id, kind, prompt_tokens, completion_tokens, total_tokens, context_tokens, context_window, cached_tokens) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`, + (id, session_id, message_id, kind, prompt_tokens, completion_tokens, total_tokens, context_tokens, context_window, cached_tokens, cache_write_tokens) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`, uuid.NewString(), sessionId, messageId, kind, usage.PromptTokens, usage.CompletionTokens, usage.TotalTokens, contextTokens, contextWindow, - usage.PromptTokensDetails.CachedTokens) + usage.CachedTokens, usage.CacheWriteTokens) if err != nil { util.Log.Warn("recording token usage failed", "session", sessionId, "kind", kind, "error", err) } } -func usageRecord(sessionId, messageId, kind string, usage openai.CompletionUsage) { +func usageRecord(sessionId, messageId, kind string, usage TokenUsage) { usageRecordWithContext(sessionId, messageId, kind, usage, 0, 0) } @@ -102,7 +141,7 @@ func SessionUsage(sessionId string) (*UsageTotals, error) { totals.SessionId = sessionId - rows, err = util.DB.Query(`SELECT kind, SUM(prompt_tokens), SUM(completion_tokens), SUM(total_tokens), SUM(cached_tokens) + rows, err = util.DB.Query(`SELECT kind, SUM(prompt_tokens), SUM(completion_tokens), SUM(total_tokens), SUM(cached_tokens), SUM(cache_write_tokens) FROM token_usage WHERE session_id = ? GROUP BY kind ORDER BY kind ASC;`, sessionId) if err != nil { return nil, err @@ -110,7 +149,7 @@ func SessionUsage(sessionId string) (*UsageTotals, error) { defer rows.Close() for rows.Next() { - err = rows.Scan(&line.Kind, &line.PromptTokens, &line.CompletionTokens, &line.TotalTokens, &line.CachedTokens) + err = rows.Scan(&line.Kind, &line.PromptTokens, &line.CompletionTokens, &line.TotalTokens, &line.CachedTokens, &line.CacheWriteTokens) if err != nil { return nil, err } @@ -120,6 +159,7 @@ func SessionUsage(sessionId string) (*UsageTotals, error) { totals.CompletionTokens += line.CompletionTokens totals.TotalTokens += line.TotalTokens totals.CachedTokens += line.CachedTokens + totals.CacheWriteTokens += line.CacheWriteTokens } err = rows.Err() diff --git a/core/usage_test.go b/core/usage_test.go index da9f90b..7cb169d 100644 --- a/core/usage_test.go +++ b/core/usage_test.go @@ -14,11 +14,10 @@ import ( "github.com/devproje/mininaru/modules" "github.com/devproje/mininaru/util" - "github.com/openai/openai-go" ) -func usageOf(prompt, completion int64) openai.CompletionUsage { - return openai.CompletionUsage{ +func usageOf(prompt, completion int64) TokenUsage { + return TokenUsage{ PromptTokens: prompt, CompletionTokens: completion, TotalTokens: prompt + completion, } } @@ -33,7 +32,7 @@ func cachedUsageChunk(id string, prompt, completion, cached int) string { return `data: {"id":"` + id + `","object":"chat.completion.chunk","created":1,"model":"m","choices":[],` + `"usage":{"prompt_tokens":` + strconv.Itoa(prompt) + `,"completion_tokens":` + strconv.Itoa(completion) + `,"total_tokens":` + strconv.Itoa(prompt+completion) + `,"prompt_tokens_details":{"cached_tokens":` + - strconv.Itoa(cached) + `}}}` + "\n\n" + strconv.Itoa(cached) + `,"cache_write_tokens":` + strconv.Itoa(prompt-cached) + `}}}` + "\n\n" } func usageRows(t *testing.T, sessionId, kind string) int { @@ -111,6 +110,9 @@ func TestTurnUsageIsRecordedAndSummedAcrossRounds(t *testing.T) { if totals.CachedTokens != 240 { t.Fatalf("cached tokens = %d, want both rounds summed", totals.CachedTokens) } + if totals.CacheWriteTokens != 60 { + t.Fatalf("cache write tokens = %d, want both rounds summed", totals.CacheWriteTokens) + } if usageRows(t, session.Id, UsageTurn) != 1 { t.Fatal("the turn should be one row carrying the sum, not one per round") } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b479f55..8cf0f62 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,7 @@ # Architecture -mininaru is a single Go binary that talks to OpenAI-compatible providers. It +mininaru is a single Go binary that talks to OpenAI-compatible providers and +the native Anthropic Messages API. It ships two front ends over one core: a terminal chat client and an HTTP API. ## Packages @@ -284,13 +285,17 @@ the third. Each round resends the whole conversation and each is billed, so the sum is the honest number. This also changes what the HTTP API returns: `usage` in a response now covers every round the server ran on the caller's behalf, which is not what OpenAI means by the field but is what the caller actually caused. -`prompt_tokens_details.cached_tokens` is summed alongside it and exposed as the -cached subset of prompt usage. +OpenAI-compatible `prompt_tokens_details.cached_tokens` and +`cache_write_tokens`, or Anthropic `cache_read_input_tokens` and +`cache_creation_input_tokens`, are summed alongside it and exposed separately. **Prompt caching stays provider-owned.** mininaru does not duplicate a model KV cache. It sorts permitted tool definitions by name before building both the -system prompt and request schema, keeping that large prefix deterministic, then -records whatever cache-hit count the OpenAI-compatible provider reports. +system prompt and request schema, keeping that large prefix deterministic. A +provider policy controls top-level `cache_control`; native Anthropic and Claude +through OpenRouter use automatic moving breakpoints. Cache reads and writes are +normalized into the same usage record. OpenRouter whole-response caching is a +separate provider opt-in implemented with `X-OpenRouter-Cache` headers. **Usage has to be asked for.** `stream_options.include_usage` was set only in `Complete`, so the session path and the delegation path never received usage at diff --git a/go.mod b/go.mod index fda9ad5..04750d1 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/devproje/mininaru go 1.26 require ( + github.com/anthropics/anthropic-sdk-go v1.61.0 github.com/bwmarrin/discordgo v0.29.0 github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 @@ -23,6 +24,8 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.2 // indirect github.com/charmbracelet/colorprofile v0.4.1 // indirect github.com/charmbracelet/x/ansi v0.11.6 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect @@ -38,6 +41,7 @@ require ( github.com/gorilla/css v1.0.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.14.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -48,11 +52,13 @@ require ( github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/spf13/pflag v1.0.10 // indirect + github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 // indirect github.com/tidwall/gjson v1.19.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -61,6 +67,7 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yuin/goldmark v1.7.8 // indirect github.com/yuin/goldmark-emoji v1.0.5 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/sync v0.22.0 // indirect diff --git a/go.sum b/go.sum index 2145786..2e2913f 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46 github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/anthropics/anthropic-sdk-go v1.61.0 h1:JRTnm1tPqn5xo1xd1zfrcFDlcoWXVMvV1K68YmhpZKw= +github.com/anthropics/anthropic-sdk-go v1.61.0/go.mod h1:3EfIfmFqxH6rbiLcIP4tPFyXL/IHakx2wDG4OU+TIEI= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -14,6 +16,10 @@ github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3v github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno= github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= @@ -43,8 +49,12 @@ github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEX github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= @@ -69,6 +79,8 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg= +github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= @@ -94,6 +106,10 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0= github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y= +github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP1nBjY= +github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -110,6 +126,10 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 h1:uOfcYT+3QungH6tIGSVCR/Y3KJmgJiHcojJbMTPDZAI= +github.com/standard-webhooks/standard-webhooks/libraries v0.0.1/go.mod h1:L1MQhA6x4dn9r007T033lsaZMv9EmBAdXyU/+EF40fo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= @@ -133,6 +153,8 @@ github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiav go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= +go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= @@ -163,6 +185,10 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= diff --git a/util/migration_test.go b/util/migration_test.go index 0dfb3c8..67e1bc1 100644 --- a/util/migration_test.go +++ b/util/migration_test.go @@ -309,7 +309,7 @@ func TestTokenUsageTableCreated(t *testing.T) { if count != 1 { t.Fatal("token_usage was not created on an existing database") } - rows, err = db.Query("SELECT context_tokens, context_window, cached_tokens FROM token_usage LIMIT 0;") + rows, err = db.Query("SELECT context_tokens, context_window, cached_tokens, cache_write_tokens FROM token_usage LIMIT 0;") if err != nil { t.Fatalf("token_usage context columns were not added: %v", err) } diff --git a/util/migrations/0013_cache_write_tokens.sql b/util/migrations/0013_cache_write_tokens.sql new file mode 100644 index 0000000..e6bc532 --- /dev/null +++ b/util/migrations/0013_cache_write_tokens.sql @@ -0,0 +1 @@ +ALTER TABLE token_usage ADD COLUMN cache_write_tokens INTEGER NOT NULL DEFAULT 0;