diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 1f802ef..bb8f804 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -1,7 +1,7 @@ import { dirname } from 'node:path' import { fileURLToPath } from 'node:url' import defineVersionedConfig from 'vitepress-versioning-plugin' -import { latestReleasedVersion, visibleVersions } from './versions' +import { developmentVersion, latestReleasedVersion, visibleVersions } from './versions' const base = process.env.VITEPRESS_BASE ?? '/docs/' const __dirname = dirname(fileURLToPath(import.meta.url)) @@ -11,6 +11,10 @@ type SidebarItem = { link?: string collapsed?: boolean items?: SidebarItem[] + // Restrict this item to specific versions, for pages that only exist in + // some version trees (e.g. new unreleased pages under `development`). + // Omit for items that exist in every visible version. + versions?: string[] } const baseSidebar: SidebarItem[] = [ @@ -35,7 +39,8 @@ const baseSidebar: SidebarItem[] = [ { text: 'Agent Sandboxes', link: '/agentops/sandboxes.html' }, { text: 'MCP Servers', link: '/agentops/mcp_servers.html' }, { text: 'Memory Stores', link: '/agentops/memory_stores.html' }, - { text: 'Agents', link: '/agentops/agents.html' } + { text: 'Agents', link: '/agentops/agents.html' }, + { text: 'LLM Serving', link: '/agentops/llm_serving.html', versions: [developmentVersion] } ] }, { @@ -63,6 +68,7 @@ const baseSidebar: SidebarItem[] = [ { text: 'Component Versions', link: '/platform/component_versions.html' }, { text: 'Observability', link: '/platform/observability.html' }, { text: 'System Status', link: '/platform/system_status.html' }, + { text: 'Agent Gateway', link: '/platform/agent_gateway.html', versions: [developmentVersion] }, { text: 'API Keys', link: '/platform/api_keys.html' } ] }, @@ -77,6 +83,7 @@ const baseSidebar: SidebarItem[] = [ { text: 'Network Policies', link: '/admin/network_policies.html' }, { text: 'Application Networking', link: '/admin/application_networking.html' }, { text: 'Application Authentication', link: '/admin/application_authentication.html' }, + { text: 'External Models', link: '/admin/external_models.html', versions: [developmentVersion] }, { text: 'Storage', link: '/admin/storage.html' }, { text: 'GPU Administration', link: '/admin/gpu.html' }, { text: 'Backup and Restore', link: '/admin/backup_restore.html' }, @@ -89,10 +96,12 @@ const baseSidebar: SidebarItem[] = [ function sidebarFor(version: string): SidebarItem[] { return baseSidebar.map((section) => ({ ...section, - items: section.items?.map((item) => ({ - ...item, - link: item.link ? `/${version}${item.link}` : item.link - })) + items: section.items + ?.filter((item) => !item.versions || item.versions.includes(version)) + .map((item) => ({ + ...item, + link: item.link ? `/${version}${item.link}` : item.link + })) })) } diff --git a/docs/_static/diagrams/agentops/agent-gateway-flow.svg b/docs/_static/diagrams/agentops/agent-gateway-flow.svg new file mode 100644 index 0000000..2f66d4b --- /dev/null +++ b/docs/_static/diagrams/agentops/agent-gateway-flow.svg @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + External apps & SDKs + + + CI/CD & automation + + + Upstream agents & apps + + + EXTERNAL, API-KEY-AUTHENTICATED CALLERS + + + + + + + + + Agent Gateway + Checks API key scope + workspace, then routes + built on agentgateway + + + + + + + + + + + kagent Agents + (A2A) + + + MCP Servers & + Memory Stores + + + Models + LLM Serving + External Models + + + Agent Sandboxes + + + + + + + + + + + + + + + + Internal (mesh) traffic — no Agent Gateway hop + An agent calling another agent, an MCP tool, or a model in the same workspace is authorized by + its Kubernetes/mesh identity automatically. No API key is issued, sent, or checked for this path. + diff --git a/docs/_static/screenshots/admin/external-models/ai-gateway-overview.png b/docs/_static/screenshots/admin/external-models/ai-gateway-overview.png new file mode 100644 index 0000000..390ad86 Binary files /dev/null and b/docs/_static/screenshots/admin/external-models/ai-gateway-overview.png differ diff --git a/docs/_static/screenshots/admin/external-models/grant-workspace-access-agents-tab.png b/docs/_static/screenshots/admin/external-models/grant-workspace-access-agents-tab.png new file mode 100644 index 0000000..70af129 Binary files /dev/null and b/docs/_static/screenshots/admin/external-models/grant-workspace-access-agents-tab.png differ diff --git a/docs/_static/screenshots/admin/external-models/grant-workspace-access.png b/docs/_static/screenshots/admin/external-models/grant-workspace-access.png new file mode 100644 index 0000000..95695f7 Binary files /dev/null and b/docs/_static/screenshots/admin/external-models/grant-workspace-access.png differ diff --git a/docs/_static/screenshots/agentops/agents/chat-tool-activity.png b/docs/_static/screenshots/agentops/agents/chat-tool-activity.png new file mode 100644 index 0000000..9e52949 Binary files /dev/null and b/docs/_static/screenshots/agentops/agents/chat-tool-activity.png differ diff --git a/docs/_static/screenshots/agentops/agents/model-configurations.png b/docs/_static/screenshots/agentops/agents/model-configurations.png new file mode 100644 index 0000000..920b996 Binary files /dev/null and b/docs/_static/screenshots/agentops/agents/model-configurations.png differ diff --git a/docs/_static/screenshots/agentops/llm-serving/automatic-tool-calling.png b/docs/_static/screenshots/agentops/llm-serving/automatic-tool-calling.png new file mode 100644 index 0000000..386d0e6 Binary files /dev/null and b/docs/_static/screenshots/agentops/llm-serving/automatic-tool-calling.png differ diff --git a/docs/_static/screenshots/agentops/mcp/landing-page-with-server-list-and-catalogue.png b/docs/_static/screenshots/agentops/mcp/landing-page-with-server-list-and-catalogue.png new file mode 100644 index 0000000..4c826f7 Binary files /dev/null and b/docs/_static/screenshots/agentops/mcp/landing-page-with-server-list-and-catalogue.png differ diff --git a/docs/_static/screenshots/agentops/mcp/playwright-details-logs.png b/docs/_static/screenshots/agentops/mcp/playwright-details-logs.png new file mode 100644 index 0000000..5e7124a Binary files /dev/null and b/docs/_static/screenshots/agentops/mcp/playwright-details-logs.png differ diff --git a/docs/_static/screenshots/agentops/mcp/playwright-details-metrics.png b/docs/_static/screenshots/agentops/mcp/playwright-details-metrics.png new file mode 100644 index 0000000..bd9292e Binary files /dev/null and b/docs/_static/screenshots/agentops/mcp/playwright-details-metrics.png differ diff --git a/docs/_static/screenshots/agentops/mcp/playwright-launch-dialog.png b/docs/_static/screenshots/agentops/mcp/playwright-launch-dialog.png new file mode 100644 index 0000000..65a015b Binary files /dev/null and b/docs/_static/screenshots/agentops/mcp/playwright-launch-dialog.png differ diff --git a/docs/_static/screenshots/agentops/mcp/playwright-live-browser-viewer.png b/docs/_static/screenshots/agentops/mcp/playwright-live-browser-viewer.png new file mode 100644 index 0000000..aeb2621 Binary files /dev/null and b/docs/_static/screenshots/agentops/mcp/playwright-live-browser-viewer.png differ diff --git a/docs/_static/screenshots/agentops/mcp/playwright-overview.png b/docs/_static/screenshots/agentops/mcp/playwright-overview.png new file mode 100644 index 0000000..3e952be Binary files /dev/null and b/docs/_static/screenshots/agentops/mcp/playwright-overview.png differ diff --git a/docs/_static/screenshots/platform/api-keys/api-key-usage-tab.png b/docs/_static/screenshots/platform/api-keys/api-key-usage-tab.png new file mode 100644 index 0000000..876a9ea Binary files /dev/null and b/docs/_static/screenshots/platform/api-keys/api-key-usage-tab.png differ diff --git a/docs/versions/development/admin/external_models.md b/docs/versions/development/admin/external_models.md new file mode 100644 index 0000000..13fc0af --- /dev/null +++ b/docs/versions/development/admin/external_models.md @@ -0,0 +1,83 @@ +# External Models + +The **AI Gateway** admin page connects centrally managed model providers to prokube. Administrators can add provider credentials once, register models from those providers, and grant specific workspaces access to specific models. + +Use this workflow when you need to: + +- offer Mistral AI, Azure OpenAI, GitHub Models, or a custom OpenAI-compatible endpoint; +- share a centrally managed provider credential without copying it into workspace Secrets; +- control which workspaces can use each external model. + +For OpenAI, Anthropic, or Gemini, workspace users can instead create their own Model Configuration backed by a workspace Kubernetes Secret. See [Agents](../agentops/agents.html#_2-choose-or-create-a-model-configuration). + +![AI Gateway page with provider catalog and platform-wide usage summary](../../../_static/screenshots/admin/external-models/ai-gateway-overview.png) + +## Provider Options + +The admin-managed provider catalog supports: + +- Anthropic; +- OpenAI; +- Mistral AI; +- Azure OpenAI; +- GitHub Models; +- custom OpenAI-compatible endpoints. + +Anthropic uses its native protocol. The other provider types use an OpenAI-compatible protocol. Custom and Azure OpenAI providers require an upstream host and path prefix. + +## Connect a Provider + +Open **AI Gateway** under **Admin**, then select **Add Provider**. Configure: + +- **Provider**: select an entry from the provider catalog. +- **Provider name**: unique name for the provider connection. +- **Provider API key**: credential stored centrally by prokube. +- **Upstream host** and **Path prefix**: required for custom and Azure OpenAI providers. +- **Models**: optional comma- or newline-separated model IDs to add immediately. + +Provider API keys are write-only. To rotate one later, edit the provider and enter a **Replacement API key**. + +## Add Models + +Open a provider and switch to its **Models** tab. Models can be added in two ways: + +- **Discover models** queries the provider and lets you select one or more returned model IDs. +- **Add manually** registers a model ID directly. + +The main **Models** table shows each model's provider, API protocol, workspace-access count, and base URL. A model cannot be deleted while a workspace grant still references it. Revoke those grants first. + +## Grant Workspace Access + +Select **Grant Workspace Access**, then choose: + +- **Workspace**: the workspace that should receive the model. +- **Route ID / ModelConfig name**: the name shown in that workspace's Model Configurations. +- **Resource**: the provider model to grant. + +The grant creates a Model Configuration in the target workspace. It appears automatically on the workspace's **Agents** page, tagged **AI Gateway** as its origin. Users can select it when creating an agent without creating their own provider Secret. + +![Workspace Access table with an external model grant](../../../_static/screenshots/admin/external-models/grant-workspace-access.png) + +![Granted external model shown as an AI Gateway Model Configuration](../../../_static/screenshots/admin/external-models/grant-workspace-access-agents-tab.png) + +The central provider credential is not exposed to the workspace. Revoke the grant from **Workspace Access** when the workspace should no longer use the model. + +An admin-granted model is consumed by kagent agents through its generated Model Configuration. It is not selectable as a workload when creating a user API key on the **API Keys** page. + +## Usage and Cost + +The **Usage and Cost** panel shows authenticated Agent Gateway traffic across all workspaces. Select Last hour, Last 24 hours, Last 7 days, or Last 30 days to view: + +- total, successful, and failed requests; +- request trends; +- LLM input and output tokens; +- estimated LLM cost. + +These figures are observability estimates, not billing records. Use the workspace [API Key Usage Dashboard](../platform/api_keys.html#usage-dashboard) to inspect usage attributed to visible keys in one workspace. + +## Related Pages + +- [Agents](../agentops/agents.html) +- [Agent Gateway](../agentops/agent_gateway.html) +- [API Keys](../platform/api_keys.html) +- [LLM Serving](../agentops/llm_serving.html) diff --git a/docs/versions/development/admin/index.md b/docs/versions/development/admin/index.md index 7c1dd11..dbf4557 100644 --- a/docs/versions/development/admin/index.md +++ b/docs/versions/development/admin/index.md @@ -22,6 +22,7 @@ Use these pages when you operate a prokube deployment, connect it to organizatio | [Network Policies](./network_policies.html) | Defining reusable egress profiles and assigning outbound network restrictions to workspaces. | | [Application Networking](./application_networking.html) | Exposing custom applications through prokube gateways and Istio routing. | | [Application Authentication](./application_authentication.html) | Choosing gateway authentication or direct OIDC for custom applications. | +| [External Models](./external_models.html) | Connecting centrally managed model providers and granting individual models to workspaces. | | [Storage](./storage.html) | Understanding StorageClasses, local storage, replicated storage, and PVC troubleshooting. | | [GPU Administration](./gpu.html) | Operating GPU nodes, NVIDIA GPU Operator, timeslicing, MIG, and GPU monitoring. | | [Backup and Restore](./backup_restore.html) | Disaster-recovery scope, backup storage, restore expectations, and validation. | diff --git a/docs/versions/development/agentops/agent_gateway.md b/docs/versions/development/agentops/agent_gateway.md index daab34c..da0ab24 100644 --- a/docs/versions/development/agentops/agent_gateway.md +++ b/docs/versions/development/agentops/agent_gateway.md @@ -1,29 +1,55 @@ # Agent Gateway -::: info Documentation in progress -This page is an early outline. Full Agent Gateway documentation is still being written and will be added here. -::: +Agent Gateway is prokube's shared routing and policy layer for external API traffic, the same layer that fronts classic model-serving endpoints and Knative services in MLOps. See [Agent Gateway](../platform/agent_gateway.html) in Foundation for the platform-wide routing model: path families, public vs. internal traffic, API keys, and the upstream [agentgateway](https://agentgateway.dev/) project it's built on. -Agent Gateway provides the external API access layer for AgentOps and selected MLOps workflows. +This page covers the AgentOps-specific angle: how Agent Gateway moves traffic between agents, tools, and models, and how agents reach external LLM providers. -It is not only an agent feature. The same gateway model can protect sandbox APIs, MCP servers, agent endpoints, and classic model-serving endpoints. +## How Agent Gateway Moves Agent Traffic -## What It Does +![Diagram: external callers reach Agent Gateway, which routes to kagent agents, MCP servers, models, and Agent Sandboxes. Agents, tools, and models inside the same workspace call each other directly over mesh identity instead.](../../../_static/diagrams/agentops/agent-gateway-flow.svg) -- Exposes public API path families such as `/sandbox`, `/mcp`, `/a2a`, `/ai`, and `/serving`. -- Enforces scoped API-key access. -- Routes requests to workspace-scoped backends. -- Keeps public API access separate from browser-based UI login. +An external caller (an SDK, a CI job, or another agent outside the workspace) authenticates with an API key scoped to one of the `/a2a`, `/mcp`, `/ai`, or `/sandbox` paths. Agent Gateway checks the key's scope and workspace, then forwards the request to: -## Common Use Cases +- a **kagent agent**, over agent-to-agent (A2A); +- an **MCP server or memory store**, for tool and retrieval access; +- a **model**, self-hosted through [LLM Serving](llm_serving.html) or granted through [External Models](../admin/external_models.html); +- an **Agent Sandbox**, for isolated code execution. -- Calling model-serving endpoints from external applications. -- Giving an agent access to a sandbox API without giving it browser credentials. -- Exposing MCP servers to external agent clients. -- Separating workspace and route scopes for automation clients. +Inside the same workspace, none of this needs an API key: an agent calling another agent, an MCP tool, or a model authenticates automatically over Kubernetes/mesh identity. You only reach for Agent Gateway, and an API key, when the caller is outside the workspace. -## API Keys +## When to Use Agent Gateway for Agents -API keys are managed through pkui. The current Python and TypeScript SDKs send API keys as `x-api-key`. Bearer-style keys are not the SDK default right now. +- Reach kagent agents through agent-to-agent (A2A) calls from outside the cluster. +- Expose MCP servers or memory stores to external agent clients. +- Give an agent access to a sandbox API without handing it browser credentials. +- Call a self-hosted or externally granted model from an external application, script, or CI job. -Do not store API keys in source code, notebooks, screenshots, tickets, or chat messages. +For interactive work in the prokube UI, use your normal user session instead. Agent Gateway is for programmatic clients. For the general routing/API-key mechanics behind all of this, see [Agent Gateway](../platform/agent_gateway.html) in Foundation. + +## External Models + +Agents can use external models through two different paths: + +| | User-created Model Configuration | Admin-managed external model | +|---|---|---| +| Providers | OpenAI, Anthropic, Gemini | Anthropic, OpenAI, Mistral AI, Azure OpenAI, GitHub Models, or a custom OpenAI-compatible endpoint | +| Credential | API key stored in a workspace Kubernetes Secret | Provider credential managed centrally by an administrator | +| Availability | Available only through that workspace's Model Configuration | Granted to selected workspaces and shown there as an **AI Gateway** Model Configuration | +| Routing | Agent connects to the provider through the Model Configuration | Model traffic is routed through Agent Gateway | + +Use a user-created Model Configuration for a workspace-specific provider credential. Use the admin-managed path when credentials should be shared centrally, when workspaces need explicit model grants, or when the provider is not available in the self-service list. + +Workspace users select either type from the same Model Configurations list when creating an agent. Administrators configure providers, models, and workspace grants under [External Models](../admin/external_models.html). + +## Related Pages + +- [Agent Gateway](../platform/agent_gateway.html) (Foundation: path families, API keys, public vs. internal traffic) +- [API Keys](../platform/api_keys.html) +- [Agents](agents.html) +- [LLM Serving](llm_serving.html) +- [Agent Sandboxes](sandboxes.html) +- [MCP Servers](mcp_servers.html) +- [Memory Stores](memory_stores.html) +- [External Models](../admin/external_models.html) +- [Model Serving](../mlops/model_serving.html) +- [Serverless](../mlops/knative.html) diff --git a/docs/versions/development/agentops/agents.md b/docs/versions/development/agentops/agents.md index 54e75a8..e52d3da 100644 --- a/docs/versions/development/agentops/agents.md +++ b/docs/versions/development/agentops/agents.md @@ -1,22 +1,110 @@ # Agents -::: info Documentation in progress -This page is an early outline. Full agent runtime documentation is still being written and will be added here. +prokube runs agents on [kagent](https://kagent.dev/), a Kubernetes-native agent runtime. Agents, their model configuration, and their tools are Kubernetes custom resources, managed through the pkui **Agents** page in your workspace. + +::: info kagent documentation +Upstream references: + +- [kagent documentation](https://kagent.dev/docs) +- [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) ::: -The Agents area covers managed agent runtimes, skills, tools, and agent-to-agent workflows. +This page covers the prokube-specific workflow for wiring a model, tools, and an agent together, and testing the result. For tool hosting itself, see [MCP Servers](mcp_servers.html). + +## When to Use Agents + +Use the Agents page when you need a conversational or task-driven agent that calls an LLM and tools exposed through MCP servers or other agents. Use it when you want the agent to run on the platform: its configuration versioned as Kubernetes resources, observable through pkui, and reachable by other workloads over A2A. + +## Typical Workflow + +Building a working agent is a short chain of dependent resources: + +1. If needed, store an external provider API key as a Kubernetes Secret. +2. Choose an available **Model Configuration**, or create one for an external provider or in-cluster model. Some models might already be available, if pre-configured by an administrator. +3. Create an **Agent**, attach the Model Configuration, and select MCP tools. +4. Test the agent in the built-in chat. + +### 1. Store the Provider API Key + +Skip this step if you will use an admin-provided Model Configuration or an **Internal** (in-cluster) model. Admin-provided configurations use a credential the administrator already manages centrally. Internal models don't need a stored credential at all: the platform authenticates them automatically as in-cluster workloads. + +Otherwise, for an **external** provider (OpenAI, Anthropic, Gemini), create a Kubernetes Secret first: open the user menu (top right) → **K8s Secrets** → **Add Secret**, and store the provider API key as a key/value pair in the workspace, for example `ANTHROPIC_API_KEY: sk-ant-...` or `OPENAI_API_KEY: sk-...`. See [Kubernetes Secrets](../platform/kubernetes.html#kubernetes-secrets) for the full flow. + +### 2. Choose or Create a Model Configuration + +Check **Model Configurations** on the **Agents** page first. An administrator may already have granted the workspace access to an external model through AI Gateway. These configurations appear automatically with **AI Gateway** as their origin and can be selected when creating an agent. + +![Model Configurations list with an AI Gateway model](../../../_static/screenshots/agentops/agents/model-configurations.png) + +If no suitable configuration exists, select **Create ModelConfig**. The **Provider** field selects how kagent reaches the backend: + +| Provider | Type | Needs a Secret? | +|---|---|---| +| OpenAI (external) | External | Yes | +| Anthropic (external) | External | Yes | +| Gemini (external) | External | Yes | +| Internal - OpenAI-compatible | In-cluster | No | +| Internal - Ollama | In-cluster | No | + +- For an **external** provider, select the Secret and the secret key created in step 1, and enter the model name. +- For **Internal - OpenAI-compatible**, either pick a **Ready** model deployed through [LLM Serving](llm_serving.html) in the workspace, or enter a custom in-cluster OpenAI-compatible base URL and model name manually. +- For **Internal - Ollama**, enter the in-cluster Ollama host, for example `http://ollama..svc.cluster.local:11434`. + +Azure OpenAI, Mistral AI, GitHub Models, and other OpenAI-compatible endpoints are not in this self-service list, even if you hold your own API key for them. Connecting one of these requires an administrator to configure the provider centrally. A workspace only gets a Model Configuration for it once an administrator grants access. See [Additional Providers (Administrators)](#additional-providers-administrators) below. + +::: info Tool calling with internal models +For a self-hosted text-generation model used with MCP tools, enable **automatic tool calling** in [LLM Serving: Enable Tool Calling for Agents](llm_serving.html#enable-tool-calling-for-agents) and select the parser that matches the model. This option is available for the vLLM and HuggingFace runtimes. +::: + +### 3. Create an Agent + +Still on the Agents page, open **Configured Agents** → **Create Agent**. Choose an **Agent type**: + +- **Declarative**: the common case. Requires a **Name**, the **Model Configuration** from step 2, and a required **System message** describing the agent's behavior. +- **BYO**: wraps your own container image, invoked over A2A, instead of a declarative kagent agent. + +For a declarative agent, attach tools: + +- **MCP tools**: a searchable, filterable table of every tool discovered from MCP servers (Tools) available to the workspace. Check individual tools, or use **"Use all current and future tools from ``"** per server to keep the agent in sync with a tool server automatically instead of a fixed snapshot. +- **Agent tools**: let this agent call other agents in the same workspace as tools. + +MCP tools come from **Tools** on the same Agents page. Connect an MCP endpoint there first if the tool you need isn't listed yet. See [MCP Servers](mcp_servers.html) for details. If the agent uses [Agent Sandboxes](sandboxes.html)-backed skills, attach the Tool that points at `sandbox-mcp`. + +### 4. Test the Agent + +Open the agent from the Agents list. The detail page's **Chat** tab is available once the agent is `Ready`, and shows: + +- A session sidebar (new/previous sessions). +- The chat itself. +- A **Connected Tools** panel listing attached tools and their readiness. + +Each response that used a tool shows a **Tool Activity** entry with the tool's arguments and result, so you can verify what the agent actually called and with what inputs. This is useful for debugging both prompt and tool wiring before relying on the agent elsewhere. + +![Agent chat showing connected tools and tool activity](../../../_static/screenshots/agentops/agents/chat-tool-activity.png) + +For programmatic access instead of the chat UI, the agent's Overview tab lists its A2A endpoint. External callers need an API key scoped to the agent (`a2a`). See [API Keys](../platform/api_keys.html). Callers inside the same workspace can reach the agent over the internal A2A path without a key. + +## Additional Providers (Administrators) + +Users can create Model Configurations for: + +- OpenAI, Anthropic, and Gemini using a workspace Secret; +- internal OpenAI-compatible and Ollama models without a provider Secret. + +Administrators can: -This part of AgentOps is still evolving. The documentation should explain how agents use shared platform capabilities instead of duplicating them: +- connect Mistral AI, Azure OpenAI, GitHub Models, or a custom OpenAI-compatible endpoint; +- grant selected workspaces access to individual models from those providers. -- Agent Gateway for model, tool, and agent traffic -- MCP Servers for governed tool access -- Sandboxes for isolated code execution -- Memory Stores for persistent context -- Observability for logs, metrics, traces, and audit trails +An admin-granted model appears automatically as a selectable Model Configuration, tagged **AI Gateway** as its origin. Users do not need their own provider Secret. See [External Models](../admin/external_models.html) for the administrator workflow. -## Documentation TODOs +## Related Pages -- Define the supported agent runtime model. -- Document the relationship between kagent, tools, skills, and MCP servers. -- Document agent-to-agent access patterns. -- Document default observability and audit expectations. +- [Agent Gateway](agent_gateway.html) +- [LLM Serving](llm_serving.html) +- [MCP Servers](mcp_servers.html) +- [Agent Sandboxes](sandboxes.html) +- [Memory Stores](memory_stores.html) +- [External Models](../admin/external_models.html) +- [API Keys](../platform/api_keys.html) +- [Kubernetes Resources](../platform/kubernetes.html) diff --git a/docs/versions/development/agentops/index.md b/docs/versions/development/agentops/index.md index b0c82a1..740cc82 100644 --- a/docs/versions/development/agentops/index.md +++ b/docs/versions/development/agentops/index.md @@ -1,32 +1,49 @@ # AgentOps -::: info Documentation in progress -This page is an early outline. Full AgentOps documentation is still being written and will be added here. -::: +AgentOps in prokube covers the operational path for building, connecting, exposing, and observing AI agents on controlled workspace infrastructure. -AgentOps is the prokube product track for building and operating AI agents on controlled infrastructure. +Use [Labs](../labs/index.md) for interactive development and debugging. Use AgentOps when agent workflows need managed models, governed tool access, sandboxed execution, memory, external API access, or agent-to-agent integration. -It combines governed model access, tool access, memory, isolated code execution, and observability into one platform track. The goal is to let teams run agents without giving them uncontrolled credentials, unmanaged SaaS dependencies, or direct access to sensitive infrastructure. +## Main Workflows -## Core Capabilities +| Workflow | Use | +|---|---| +| [Agents](agents.md) | Create kagent agents, attach model configurations and tools, test conversations, and expose agents over A2A. | +| [LLM Serving](llm_serving.md) | Deploy self-hosted OpenAI-compatible models for agents and other LLM clients, including tool-calling support where available. | +| [MCP Servers](mcp_servers.md) | Run governed MCP tool servers in a workspace and expose their tools to agents, OpenCode, and external MCP clients. | +| [Agent Sandboxes](sandboxes.md) | Give agents isolated execution environments for code, shell commands, files, package installation, and stateful task execution. | +| [Memory Stores](memory_stores.md) | Provide workspace-scoped state and retrieval capabilities for agent workflows. | +| [Agent Gateway](agent_gateway.md) | Expose models, MCP servers, agents, sandboxes, and serving endpoints to external clients through scoped API keys. | -- **Agent Gateway** for public API routes, scoped API keys, routing, and policy enforcement. -- **Agent Sandboxes** for isolated code execution in Kubernetes-native environments. -- **MCP Servers** for exposing tools, skills, and internal APIs as governed capabilities. -- **Memory Stores** for agent state and retrieval-backed workflows. -- **Agents** for managed agent runtimes, skills, tools, and agent-to-agent patterns. +## Typical Flow -## Shared Platform Services +1. Choose a model path: use an administrator-granted external model, create a workspace Model Configuration, or deploy an in-cluster model with [LLM Serving](llm_serving.md). +2. Add tools when the agent needs capabilities beyond model inference: deploy [MCP Servers](mcp_servers.md), attach sandbox-backed tools, or connect another agent. +3. Create an [Agent](agents.md), attach the Model Configuration, select the MCP or agent tools it may call, and test it in the built-in chat. +4. Use [Agent Sandboxes](sandboxes.md) when the workflow needs isolated code execution, file operations, package installation, or long-running task state. +5. Expose programmatic access through [Agent Gateway](agent_gateway.md) only when external clients, SDKs, CI jobs, or other integrations need to call the service without a browser session. +6. Use logs, events, metrics, traces, and API-key usage data to debug behavior and review operational impact. -AgentOps uses the same foundation as the MLOps track: +## Access Model -- Workspaces and identity -- Role-based access control -- Observability and audit trails -- Object storage and persistent volumes -- GitOps-managed operations -- Kubernetes-native resource isolation +AgentOps separates browser-based UI work, internal workspace traffic, and external API traffic. -## Current Status +| Access path | Typical caller | Authentication | +|---|---|---| +| UI | A user working in pkui | Browser session and workspace permissions | +| Internal workspace traffic | Agents, MCP servers, models, sandboxes, and other workloads in the same workspace | Kubernetes and mesh identity | +| External API traffic | SDKs, automation, CI jobs, external MCP clients, and A2A callers | Agent Gateway API key scoped to the target service | -The AgentOps feature set is under active integration. Some components are already used in pilot deployments, while the final installer profile and public documentation structure are still being shaped. +You usually need API keys only for external clients. In-workspace agents and tools normally use the platform's internal identity and routing model. + +## Foundation + +AgentOps workloads run on the shared prokube foundation: workspaces, Kubernetes namespaces, RBAC, secrets, storage, observability, and API-key management. + +Start with these cross-cutting pages when you need platform behavior rather than feature-specific usage: + +- [Workspaces](../platform/workspaces.md) +- [Kubernetes Resources](../platform/kubernetes.md) +- [Kubernetes Secrets](../platform/kubernetes.html#kubernetes-secrets) +- [Observability](../platform/observability.md) +- [API Keys](../platform/api_keys.md) diff --git a/docs/versions/development/agentops/llm_serving.md b/docs/versions/development/agentops/llm_serving.md new file mode 100644 index 0000000..e2911c6 --- /dev/null +++ b/docs/versions/development/agentops/llm_serving.md @@ -0,0 +1,95 @@ +# LLM Serving + +LLM Serving deploys self-hosted models as OpenAI-compatible inference endpoints on KServe. It supports text generation, embedding, reranking, text-to-speech, and speech-to-text models. Use it when you want to host a model in the cluster instead of calling an external provider such as OpenAI, Anthropic, or Gemini. + +::: info Upstream references +LLM Serving builds on: + +- [KServe](https://kserve.github.io/website/) for the serving control plane and OpenAI-compatible protocol +- [vLLM](https://docs.vllm.ai/) and [KServe's HuggingFace runtime](https://kserve.github.io/website/latest/modelserving/v1beta1/llm/huggingface/) for text generation and embeddings +- [Hugging Face Text Embeddings Inference (TEI)](https://huggingface.co/docs/text-embeddings-inference) for CPU-optimized embeddings +- [faster-whisper](https://github.com/SYSTRAN/faster-whisper) for CPU speech-to-text +::: + +LLM Serving is an AgentOps capability. For classic model serving with scikit-learn, PyTorch, and similar predictors, see [Model Serving](../mlops/model_serving.html). Both features use KServe, but they provide different deployment forms and endpoints. + +## When to Use LLM Serving + +- Host an open-weight model (Llama, Mistral, Qwen, Gemma, and similar) in-cluster instead of calling an external API. +- Serve embedding or reranking models for retrieval workloads. +- Serve text-to-speech or speech-to-text models. +- Provide an in-cluster model for a kagent [Agent](agents.html) via an **Internal** Model Configuration, with or without tool calling. +- Keep model traffic and data inside the cluster instead of sending it to an external provider. + +If you only need OpenAI, Anthropic, or Gemini, create a Model Configuration directly instead. See [Agents: Choose or Create a Model Configuration](agents.html#_2-choose-or-create-a-model-configuration). + +## Deploy a Model + +Open **LLM Serving** in the sidebar. The page lists deployed models for the selected workspace, filterable by type (**All / Text Generation / Embedding / Reranking**), with columns for name, status, model ID, type, runtime, and age. + +Click **Deploy Model** and choose a preset, or select **Deploy Custom Model** to configure the deployment from scratch. Presets provide curated defaults and can be searched or filtered by GPU count, task type, and verification status. Both paths open the same form: + +- **Deployment Name**: unique name in the workspace. +- **Model Type**: Text Generation, Embedding, Reranking, Text to Speech, or Speech to Text. Typing a HuggingFace-style model ID (`org/model`) auto-detects the type. +- **Model ID**: a HuggingFace model ID, for example `meta-llama/Llama-2-7b-chat-hf`. HuggingFace is the only model source available in the form; use the YAML editor for other storage URIs. +- **Runtime**: filtered to runtimes that support the selected type. Options include HuggingFace (recommended for text generation and embeddings), TEI (CPU-optimized embeddings), vLLM, vLLM Omni (audio models), and faster-whisper (CPU speech-to-text). vLLM requires a custom ClusterServingRuntime. + +If the model is gated on HuggingFace, the form warns you before deploying: accept the license on huggingface.co, create an access token, and store it as a Kubernetes Secret named `storage-config` with key `HF_TOKEN` in the workspace (see [Kubernetes Secrets](../platform/kubernetes.html#kubernetes-secrets)). Deployment fails without it. + +Select **Or edit YAML manifest directly** if the form does not cover a setting you need, such as a custom storage URI or extra container arguments. + +## Advanced Configuration + +The **Advanced Configuration** section is collapsed by default. It covers: + +- [**Deployment Mode**](https://kserve.github.io/website/docs/concepts/architecture/control-plane): **Serverless (Knative)** is the default and supports scale-to-zero. **Raw (also called Standard in newer KServe versions) Deployment** creates a plain Kubernetes Deployment for clusters without Knative or for [KEDA](https://keda.sh/docs/latest/concepts/scaling-deployments/) autoscaling. Raw deployments require at least one replica. +- **Runtime settings**: vLLM-based runtimes support quantization options such as AWQ, GPTQ, and FP8, and data types such as Float16, BFloat16, and Float32. The HuggingFace runtime uses vLLM internally, so these settings also apply to it. +- **Resource Requests**: configure CPU, memory, and GPU count and type. GPU options come from the cluster inventory. If limits are left blank, the CPU limit defaults to twice the request and the memory limit matches the request. +- **Auto-Scaling**: configure minimum and maximum replicas, with a maximum of 10. Serverless mode allows a minimum of 0 for scale-to-zero; Raw Deployment requires at least one replica. +- **Automatic tool calling**: configure support for agents that use MCP tools. See [Enable Tool Calling for Agents](#enable-tool-calling-for-agents). + +Raw deployments can use [HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) for CPU or memory-based scaling, or [KEDA](https://keda.sh/docs/latest/concepts/scaling-deployments/) for custom Prometheus metrics such as vLLM token throughput. KEDA requires a PromQL query and a scale threshold calibrated against observed traffic. See [Serving Autoscaling](../mlops/model_serving_autoscaling.html) for the general KEDA pattern in prokube. + +## Enable Tool Calling for Agents + +Not every runtime supports OpenAI-style tool calling. The **Enable automatic tool calling** option appears only for **vLLM** and **HuggingFace** text-generation models. It is not available for TEI, vLLM Omni, faster-whisper, or other model types. + +If you plan to attach MCP tools to a kagent agent that uses this model (an **Internal** Model Configuration), turn this on and select a **Tool-call parser** matching the model's tool-call format: + +| Parser | Matches | +|---|---| +| Hermes | Qwen 2.5, QwQ, and Nous Hermes formats | +| Qwen3 XML | Qwen3-Coder tool-call format | +| Llama 3 JSON | Llama models using JSON tool calls | +| Pythonic | Llama models using Python-style tool calls | +| Mistral | Mistral tool-call format | +| DeepSeek V3 | DeepSeek V3 format | +| DeepSeek V3.1 | DeepSeek V3.1 format | + +![LLM deployment form with automatic tool calling and parser selection](../../../_static/screenshots/agentops/llm-serving/automatic-tool-calling.png) + +This list matches the parsers built into the platform's currently deployed vLLM runtime and may change as that runtime is upgraded; use YAML editing for a parser or chat template not listed here. Without a matching parser, an agent's tool calls against this model will not work reliably even though the Model Configuration and deployment are otherwise valid. + +## Test a Deployed Model + +Open a **Ready** model from the list. Text-generation models have a **Chat** tab with a built-in streaming chat tester. Embedding and reranking models have an **API** tab. Audio models have an **API** or **Test** tab, depending on the task. These tabs show the endpoint path and a ready-to-run `curl` example for the supported operation. + +The Configuration tab also shows Basic Information, Resources, Scaling, and API Endpoints for the model, alongside Metrics, Logs, and Conditions tabs for troubleshooting. While model weights are downloading, a storage panel shows progress (queued, preparing, downloading, or failed) instead of the usual tabs. + +## Edit a Model + +From the model list or detail page, select **Edit** to change Model Type, Quantization, Data Type, tool-calling settings, resource requests and limits, or auto-scaling. Model ID, runtime, and deployment mode are fixed after creation. To change them, delete and redeploy the model or use the **YAML** tab. + +## External Access + +Endpoints shown on the model's detail/API tabs are the workspace-internal serving URL, useful for testing from inside the platform. For external clients (SDKs, CI jobs, applications outside the cluster), call the model through [Agent Gateway](../platform/agent_gateway.html) instead, using a URL of the form `/ai//models//v1/...` and an API key scoped to the model. See [API Keys](../platform/api_keys.html). Models granted through [External Models](../admin/external_models.html) are reachable through this same URL pattern. + +To use a deployed model from a kagent agent instead of an external client, create an **Internal - OpenAI-compatible** Model Configuration and pick this model from the **LLM Serving model** dropdown. See [Agents](agents.html#_2-choose-or-create-a-model-configuration). + +## Related Pages + +- [Agents](agents.html) +- [Agent Gateway](agent_gateway.html) +- [API Keys](../platform/api_keys.html) +- [Model Serving](../mlops/model_serving.html) +- [Serving Autoscaling](../mlops/model_serving_autoscaling.html) diff --git a/docs/versions/development/agentops/mcp_servers.md b/docs/versions/development/agentops/mcp_servers.md index a963010..d832aa8 100644 --- a/docs/versions/development/agentops/mcp_servers.md +++ b/docs/versions/development/agentops/mcp_servers.md @@ -1,23 +1,167 @@ # MCP Servers -::: info Documentation in progress -This page is an early outline. Full MCP server documentation is still being written and will be added here. +::: info Upstream documentation +For MCP and ToolHive concepts that are not specific to prokube, use the upstream documentation: + +- [Model Context Protocol documentation](https://modelcontextprotocol.io/) +- [ToolHive documentation](https://docs.stacklok.com/toolhive/) +- [ToolHive Kubernetes CRD reference](https://docs.stacklok.com/toolhive/reference/crds/) ::: -MCP Servers expose tools, skills, and internal APIs to AI agents through the Model Context Protocol. +MCP servers expose tools, data sources, and internal APIs to AI assistants through the Model Context Protocol. In prokube, MCP servers run as Kubernetes workloads managed by ToolHive instead of local processes on a developer machine. + +Use MCP servers when an agent or MCP-capable client needs governed access to a tool, for example sandbox execution, browser automation, databases, internal APIs, or other services that should not be called with broad user credentials. + +## How prokube Runs MCP Servers + +Open **MCP** from the prokube UI sidebar under **AgentOps**. Select the workspace before deploying or inspecting servers. + +An MCP server is deployed into the selected workspace namespace as a [ToolHive `MCPServer`](https://docs.stacklok.com/toolhive/reference/crds/) resource. prokube provides the UI, workspace authorization, registry integration, logs, events, metrics, and optional Agent Gateway access. ToolHive handles the server runtime and local MCP proxy for that resource. + +The MCP page contains two main sections: + +- **Deployed Servers**: MCP servers currently running in the selected workspace. +- **Server Catalog**: registry entries that can be deployed with preconfigured images, tools, metadata, and required configuration fields. + +![MCP Servers page with deployed servers and server catalog](../../../_static/screenshots/agentops/mcp/landing-page-with-server-list-and-catalogue.png) + +## Deploy from the Catalog + +Use the catalog for known server images and common integrations. Catalog cards show the server name, description, provided tools, source, tier, repository link, and compatibility badges such as **Requires Root**. + +The catalog can be filtered by: + +- search text; +- tier: **Official** or **Community**; +- source: **prokube.ai Only** or **Third Party Only**; +- sort order: stars or name. + +Click a catalog card to deploy it. The deploy dialog shows: + +- **Namespace**: target workspace namespace. +- **Server Name**: Kubernetes resource name for the MCP server. +- **Configuration**: required and optional environment variables from the registry entry. +- **Registry Credentials**: image pull secrets available in the workspace. +- **Resource Limits**: optional CPU and memory requests and limits. +- **Technical Details**: image, transport, and provided tools. + +![Deploy dialog for a catalog MCP server](../../../_static/screenshots/agentops/mcp/playwright-launch-dialog.png) + +Environment variables can be entered directly or read from a Kubernetes Secret in the workspace. Use Secrets for tokens, passwords, API keys, and other sensitive values. See [Kubernetes Secrets](../platform/kubernetes.html#kubernetes-secrets). + +### Catalog Source and Trust + +prokube uses the upstream [ToolHive Catalog](https://github.com/stacklok/toolhive-catalog), a community-curated registry of MCP servers and skills. The catalog gives you a better starting point than searching for arbitrary container images: entries are described in a common format, reviewed through the ToolHive project, and include metadata such as the publisher, repository, required configuration, tools, and maintenance tier. + +Treat the catalog as a curated trust signal, not as a blanket approval for every environment: + +- **Official** entries are maintained by the MCP team, the upstream project, or platform owners. +- **Community** entries are contributed and maintained by the community. +- **prokube.ai Only** narrows the list to entries published or curated by prokube.ai where available. + +Before giving an MCP server credentials or access to internal systems, still review what it connects to, who maintains it, which tools it exposes, and whether it needs broad permissions. Prefer Official or internally maintained servers for production workflows. + +## Deploy a Custom Server + +Use **Deploy Custom Server** when the server is not in the catalog or when you maintain your own image. + +Required fields: + +- **Server Name**: Kubernetes-compatible resource name. +- **Container Image**: image that runs the MCP server. +- **Transport Protocol**: `stdio`, `sse`, or `streamable-http`. +- **Proxy Port**: port exposed by the ToolHive proxy. + +Optional fields: + +- environment variables, either direct values or Secret references; +- image pull credentials for private registries; +- CPU and memory requests and limits; +- container arguments; +- root or writable-filesystem options for images that require them; +- persistent storage for servers that need data to survive pod restarts; +- live view for browser-based servers such as noVNC-backed Playwright images. + +Prefer custom images that run as non-root and work with a read-only root filesystem. Images that require root or write access are harder to run in restricted workspaces and have a larger security footprint. + +## Use YAML for Advanced Configuration + +As with other Kubernetes-backed resources in prokube, you can use YAML when the form does not expose a setting you need. + +The deploy dialog can generate a ToolHive `MCPServer` manifest from the form fields. Use it to inspect the resource before deployment, adjust advanced fields, or submit a reviewed manifest directly through the UI. + +The namespace is set by prokube to the selected workspace namespace. Custom YAML must still be a ToolHive `MCPServer` resource using a supported `toolhive.stacklok.dev` API version. + +## Connect Clients + +The **Deployed Servers** table shows each server's status, image, transport, proxy port, and URL when available. Deploying an MCP server also registers it with the workspace's federated MCP endpoint in Agent Gateway. + +![MCP server overview with external and internal connection details](../../../_static/screenshots/agentops/mcp/playwright-overview.png) + +For external clients, create an [API key](../platform/api_keys.html) scoped to the MCP server. Use the authentication format expected by the client. + +For kagent agents, open **Agents** and select the discovered MCP tools when creating or editing the agent. Tools from workspace MCP servers appear through the managed `gateway-mcp` endpoint; create a separate Tool only when you need to connect an additional external MCP endpoint. + +OpenCode and other MCP-capable clients can use the endpoint URL shown in prokube. In OpenCode Labs, add it through the OpenCode MCP manager and configure the required headers or OAuth settings there. See [OpenCode: Add MCP Servers](../labs/opencode.html#add-mcp-servers). + +## Sandbox MCP + +The catalog includes `sandbox-mcp`, a prokube-provided MCP server for Agent Sandbox operations. + +It exposes tools for common sandbox tasks, including creating sandboxes, claiming existing sandboxes, running commands, executing code, reading and writing files, and managing sandbox pools. + +When deploying `sandbox-mcp`, prokube pre-fills deployment context for the selected workspace: + +- `PROKUBE_API_URL`: backend API URL reachable from the MCP server pod; +- `PROKUBE_WORKSPACE`: selected workspace namespace; +- `PROKUBE_USER_ID`: current user identity used for backend authorization. + +You can also set `SANDBOX_NAME` to auto-connect to a specific sandbox on the first tool call. + +## Browser Automation Servers + +Some catalog entries, such as the prokube Playwright noVNC image, include live browser viewing and trace support. + +For these servers, the details page can show: + +- **Live View** for an interactive browser session; +- **Traces** for recorded Playwright sessions; +- **Logs** and **Events** for debugging startup and runtime issues; +- **Metrics** for runtime monitoring. + +![Live browser view for a browser automation MCP server](../../../_static/screenshots/agentops/mcp/playwright-live-browser-viewer.png) + +![MCP server metrics for a browser automation server](../../../_static/screenshots/agentops/mcp/playwright-details-metrics.png) + +Live view is only available for servers that declare live-view support in the catalog or custom configuration. + +## Security and Operations + +- Deploy servers only in workspaces where the intended users should have access to the exposed tools. +- Store sensitive configuration in [Kubernetes Secrets](../platform/kubernetes.html#kubernetes-secrets) instead of direct environment variable values. +- Prefer **Official** or internally maintained catalog entries for production use. +- Review third-party images before granting access to internal data or network destinations. +- Avoid root and writable-root-filesystem options unless the image requires them. +- Restricted workspace security policies can reject servers that request root privileges. +- Set resource requests and limits for long-running or shared servers. +- Delete MCP servers that are no longer used. -In prokube, MCP servers are part of the AgentOps track and are managed as platform resources rather than ad-hoc local processes. +## Troubleshooting -## Goals +| Symptom | Check | +|---|---| +| Server stays `Pending` | Open the details page and check **Events** and **Logs**. Also verify image pull credentials and workspace quota. | +| Image cannot be pulled | Confirm the image name and select the required registry credential for private registries. See [Registry Credentials](../platform/kubernetes.html#registry-credentials). | +| Deployment is rejected by security policy | The image may require root or a writable filesystem in a restricted workspace. Use a compliant image or ask an administrator to review the workspace policy. | +| Required configuration is missing | Check the catalog entry's required environment variables and provide direct values or Secret references. | +| Client cannot connect | Confirm the server is `Running`, copy the current URL, and verify the API key is scoped to the MCP server. | +| Tool calls fail after connecting | Check server logs, required upstream credentials, workspace network policy, and whether the tool depends on an external service. | -- Make tools discoverable and manageable. -- Keep tool access scoped by workspace and identity. -- Support MCP-capable agent clients. -- Connect agents to internal APIs without distributing broad credentials. +![MCP server logs for a browser automation server](../../../_static/screenshots/agentops/mcp/playwright-details-logs.png) -## Related Components +## Related Pages -- ToolHive operator -- pkui MCP Servers module -- Agent Gateway public routes -- Workspace identity and policy controls +- [API Keys](../platform/api_keys.html) +- [Kubernetes Resources](../platform/kubernetes.html) +- [Agent Sandboxes](./sandboxes.html) +- [Agents](./agents.html) diff --git a/docs/versions/development/mlops/knative.md b/docs/versions/development/mlops/knative.md index 06d38a6..4e38a19 100644 --- a/docs/versions/development/mlops/knative.md +++ b/docs/versions/development/mlops/knative.md @@ -125,7 +125,7 @@ Replace `` and `` with your values. After applying, t ## Access Notes -Inside the cluster, call the Knative service directly through its internal URL (`..svc.cluster.local`). From outside, requests to `/serving/*` require an API key. See [API Keys](../platform/api_keys.md) for details. +Inside the cluster, call the Knative service directly through its internal URL (`..svc.cluster.local`). From outside, requests to `/serving/*` go through [Agent Gateway](../platform/agent_gateway.html) and require an API key. See [API Keys](../platform/api_keys.md) for details. From a Lab terminal, test the `hello` service with: diff --git a/docs/versions/development/mlops/model_serving.md b/docs/versions/development/mlops/model_serving.md index 9ab7b4c..0b51b29 100644 --- a/docs/versions/development/mlops/model_serving.md +++ b/docs/versions/development/mlops/model_serving.md @@ -19,11 +19,11 @@ Use KServe InferenceServices when a trained model should be available as an API - scale inference replicas automatically based on request concurrency, QPS, or custom metrics; - test model behaviour interactively before integrating into production; - use the v2 inference protocol for framework-agnostic model access; -- expose endpoints for external applications through the Agent Gateway with API key authentication. +- expose endpoints for external applications through [Agent Gateway](../platform/agent_gateway.html) with API key authentication. Use [Labs](../labs/index.md) or [Pipelines](pipelines.md) for training and exporting models. Move to Model Serving when the model should become a reachable endpoint. -This page covers classic KServe model serving: deploying sklearn, PyTorch, MLflow, and similar models as inference endpoints. For LLM-focused serving (vLLM, TGI, OpenAI-compatible APIs), see the [AgentOps documentation](../agentops/index.md) – large language models follow a different operational pattern and are documented separately there. +This page covers classic KServe model serving: deploying sklearn, PyTorch, MLflow, and similar models as inference endpoints. For LLM-focused serving (vLLM, HuggingFace, TGI runtimes with OpenAI-compatible APIs), see [LLM Serving](../agentops/llm_serving.html) – large language models follow a different operational pattern and are documented separately there. ## Get Started @@ -226,7 +226,7 @@ A custom [`mlflow-storage-initializer`](https://github.com/prokube/prokube-image ## External Access -To call a model endpoint from outside the cluster, you need a workspace-scoped API key. Create one on the **API Keys** page under AI Gateway — keys can be scoped to a specific workspace or to individual services. See [API Keys](../platform/api_keys.md) for details. +To call a model endpoint from outside the cluster, you need a workspace-scoped API key. Create one on the **API Keys** page under AI Gateway — keys can be scoped to a specific workspace or to individual services. See [API Keys](../platform/api_keys.md) for details. This external path is served by [Agent Gateway](../platform/agent_gateway.html), the same routing layer used across MLOps and AgentOps. Include the key in requests: diff --git a/docs/versions/development/platform/agent_gateway.md b/docs/versions/development/platform/agent_gateway.md new file mode 100644 index 0000000..4747f4f --- /dev/null +++ b/docs/versions/development/platform/agent_gateway.md @@ -0,0 +1,65 @@ +# Agent Gateway + +Agent Gateway is the shared routing and policy layer for external API traffic in prokube, built on [agentgateway](https://agentgateway.dev/), an AI-native Gateway API implementation. It gives SDKs, automation, CI jobs, and agent clients a single, API-key-authenticated way to reach model, tool, agent, and sandbox endpoints running in a workspace, without a browser session and without exposing each service through its own ad hoc ingress. + +::: info Upstream references +- [agentgateway documentation](https://agentgateway.dev/docs/) +- [agentgateway on Kubernetes](https://agentgateway.dev/docs/kubernetes/latest) +::: + +Agent Gateway is not an agent-only feature. The same routing and policy model fronts classic model-serving endpoints, Knative services, MCP servers, memory stores, kagent A2A agents, and Agent Sandboxes. It's a Foundation-level concept used by [MLOps](../mlops/model_serving.html), [AgentOps](../agentops/agent_gateway.html), and [Labs](../labs/opencode.html) alike. This page covers the shared routing model. For the AgentOps-specific view (how it moves traffic between agents, tools, and models), see [Agent Gateway for AgentOps](../agentops/agent_gateway.html). + +## When to Use Agent Gateway + +- Call model-serving endpoints from an external application, script, or CI job. +- Give an agent access to a sandbox API without handing it browser credentials. +- Expose MCP servers or memory stores to external agent clients. +- Reach kagent agents through agent-to-agent (A2A) calls from outside the cluster. +- Call a Knative-served endpoint from outside the platform. + +This list is not exhaustive: any external, API-key-authenticated call to a workspace service goes through Agent Gateway. + +For interactive work in the prokube UI, use your normal user session instead. Agent Gateway is for programmatic clients. + +## Path Families + +Agent Gateway exposes one public path family per service type. Each path is workspace-scoped and requires an API key with a matching scope. + +| Family | Path pattern | Backed by | +|---|---|---| +| `/ai` | `/ai//models//v1/...` | OpenAI-compatible LLM traffic (chat completions, embeddings, rerank): self-hosted models deployed through [LLM Serving](../agentops/llm_serving.html), and [external models](../admin/external_models.html) granted by an administrator. Classic (non-LLM) KServe models never use this family. | +| `/serving` | `/serving//` | Classic KServe InferenceServices (V1/V2 predict protocol) and Knative Services. Both share this one path family: `` resolves against whichever resource matches. | +| `/mcp` | `/mcp//` | MCP servers and MCP-compatible memory stores | +| `/a2a` | `/a2a//` | kagent agent-to-agent access | +| `/sandbox` | `/sandbox//...` | Agent Sandbox API | + +A key only authorizes the exact routes it was scoped to at creation. A request to a path outside the key's scope is rejected even if the key is otherwise valid. See [API Keys](api_keys.html#create-a-key) for how scopes are selected. + +## Public vs. Internal Traffic + +Agent Gateway separates two kinds of callers: + +- **Public**: external clients calling a `/ai`, `/serving`, `/mcp`, `/a2a`, or `/sandbox` path with an API key. This is what SDKs, CI jobs, and external integrations use. +- **Internal**: workloads running inside the same workspace (for example, an agent calling another service in-cluster) are authorized by their Kubernetes/mesh identity instead of an API key. This path is automatic for in-cluster workloads and does not require you to create or manage a key. + +You only need to think about API keys for the public path. + +## Managing Access + +API keys are the unit of access control for Agent Gateway. Create, scope, rotate, and disable them from the **API Keys** sidebar page. Each key belongs to one workspace and, optionally, to a specific set of services in that workspace. + +See [API Keys](api_keys.html) for the full create/edit/rotate/disable workflow, client authentication formats, and troubleshooting. + +## Usage Dashboard + +The API Keys page also has a **Usage** tab showing request volume, LLM token/cost estimates, and per-key activity for the selected workspace. Any workspace member can see this. Administrators additionally see aggregate-only traffic that isn't tied to a specific key. See [Usage Dashboard](api_keys.html#usage-dashboard) for details. + +## Related Pages + +- [Agent Gateway for AgentOps](../agentops/agent_gateway.html) +- [API Keys](api_keys.html) +- [Model Serving](../mlops/model_serving.html) +- [Serverless](../mlops/knative.html) +- [MCP Servers](../agentops/mcp_servers.html) +- [Agent Sandboxes](../agentops/sandboxes.html) +- [External Models](../admin/external_models.html) diff --git a/docs/versions/development/platform/api_keys.md b/docs/versions/development/platform/api_keys.md index a39f5d1..d821a8c 100644 --- a/docs/versions/development/platform/api_keys.md +++ b/docs/versions/development/platform/api_keys.md @@ -2,7 +2,9 @@ API keys provide scoped programmatic access to selected prokube services without a browser session. Use them for SDKs, automation, CI jobs, serving clients, sandbox clients, MCP clients, and external integrations. -A key belongs to the workspace that was selected when it was created. prokube routes public API traffic through [Agent Gateway](../agentops/agent_gateway.html), the shared routing and policy layer for API clients, but you do not need to configure Agent Gateway to create a key. +Each key can be scoped to specific services/workloads instead of the whole workspace, sent with either of two client authentication headers, and given an optional expiration date. + +A key belongs to the workspace that was selected when it was created. prokube routes public API traffic through [Agent Gateway](agent_gateway.html), the shared routing and policy layer for API clients, but you do not need to configure Agent Gateway to create a key. Use API keys when a workload or external client needs repeatable access without an interactive login. For browser-based work in the prokube UI, use your normal user session instead. @@ -103,6 +105,54 @@ Rotating a key invalidates the old value immediately. Update every client or sec API key action menu with edit, disable, rotate, and delete actions +## Usage Dashboard + +Switch to the **Usage** tab on the API Keys page to see how keys in the selected workspace are being used. The dashboard covers requests that Agent Gateway authenticated with a key. It does not cover internal in-mesh traffic (see [Agent Gateway: Public vs. Internal Traffic](agent_gateway.html#public-vs-internal-traffic)). + +![API key usage dashboard with request trend, estimated LLM usage, and per-key activity](../../../_static/screenshots/platform/api-keys/api-key-usage-tab.png) + +### Time Window and Cohorts + +Choose a **time window** to control both the summary numbers and the request trend chart below them. Options are Last hour, Last 24 hours (default), Last 7 days, and Last 30 days. + +The dashboard shows up to three cohort cards, split by how a request was attributed: + +- **External attributed**: requests attributed to an authenticated key. This is the only cohort regular users see. +- **Internal aggregate** and **Unattributed public**: aggregate-only counts, visible to administrators. These are never distributed across individual keys, so they cannot be used to identify which key or workload generated a given request. + +The **request trend** chart plots request volume per time bucket for the selected window. If Prometheus retained only part of the window, the dashboard shows a note that the trend is truncated rather than silently showing partial data as complete. + +### Token and Cost Estimate + +The **Estimated LLM usage** panel shows input tokens, output tokens, and an estimated cost for LLM traffic in the window. Estimated cost is derived from a model pricing catalog: + +- If pricing is missing for some models in the window, the panel notes that the estimate excludes those models. +- If no pricing data is available at all, the panel says catalog pricing is unavailable rather than showing a misleading total. + +Token and cost figures apply only to LLM (`/ai`) traffic. Other path families report request counts but not tokens or cost. + +### Usage by Key + +The **Usage by key** table lists per-key activity for the window: + +| Column | Meaning | +|---|---| +| Key | Key name. A deleted key that still has recorded usage shows as **Historical** rather than disappearing from the table. | +| Owner | The user who created the key. | +| Requests | Total requests attributed to the key in the window. | +| Failed | Requests that failed (non-2xx/3xx) in the window. | +| LLM tokens | Input + output tokens, for `/ai` traffic only. | +| Estimated cost | Estimated cost for the key's LLM traffic in the window. | +| Last Active | Approximate time since the key's last recorded request, bounded by the selected window and its sampling granularity, not an exact last-used timestamp. | + +Regular users see **Keys owned by you**. Administrators see **All workspace keys**. Keys with no recorded activity in the window are omitted, except historical (deleted) keys that administrators can still see for auditing. + +### Reading the Numbers Correctly + +- Request counts, tokens, and cost are estimates for observability, not billing records. +- Prometheus retention limits how far back the dashboard can report. Older activity outside the retention window will not appear even if the key was used. +- If usage telemetry is not configured for the cluster, the dashboard shows a notice instead of numbers. Contact your administrator if you expect usage data and see this notice. + ## Security Guidance - Prefer service-specific keys over broad workspace access. @@ -124,6 +174,7 @@ Rotating a key invalidates the old value immediately. Update every client or sec ## Related Pages +- [Agent Gateway](agent_gateway.html) - [Sandboxes](../agentops/sandboxes.html) - [MCP Servers](../agentops/mcp_servers.html) - [Model Serving](../mlops/model_serving.html)