diff --git a/.github/workflows/bazel-rbe.yml b/.github/workflows/bazel-rbe.yml index 810b49e..cce6c43 100644 --- a/.github/workflows/bazel-rbe.yml +++ b/.github/workflows/bazel-rbe.yml @@ -45,13 +45,16 @@ jobs: timeout-minutes: 20 steps: - name: Checkout - uses: actions/checkout@de0facc42343c4c3d0f2fb3f1f251f9f7e3ce75b # v6.0.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Go for Bazel tools - uses: actions/setup-go@4a3601d38874680202d6939c4edd8cf679bd637a # v6.4.0 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: "1.26.3" + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Install Bazel tools env: GOBIN: ${{ runner.temp }}/go-bin diff --git a/README.md b/README.md index b9037f6..a0fdced 100644 --- a/README.md +++ b/README.md @@ -1,474 +1,188 @@ -# Agent Harness - Unified OpenAI & Anthropic SDK Interface +# Agent Harness -[![CI](https://github.com/evalops/agent-harness/workflows/CI/badge.svg)](https://github.com/evalops/agent-harness/actions) -[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - -A production-ready harness for **hot-swapping** between OpenAI Agents SDK and Anthropic Claude Agent SDK with a unified tool registry and common API. +Agent Harness is a small Python adapter for running the same tool registry through +OpenAI Agents SDK or Anthropic Claude Agent SDK provider implementations. The core package +has no runtime SDK dependency; provider SDKs are imported lazily only when that provider is +used. -## Key Features +[![CI](https://github.com/evalops/agent-harness/actions/workflows/ci.yml/badge.svg)](https://github.com/evalops/agent-harness/actions/workflows/ci.yml) +[![Bazel RBE](https://github.com/evalops/agent-harness/actions/workflows/bazel-rbe.yml/badge.svg)](https://github.com/evalops/agent-harness/actions/workflows/bazel-rbe.yml) +[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) -✅ **Unified Tool Registry** - Register tools once, use with any provider -✅ **Hot-Swapping** - Switch providers at runtime without code changes -✅ **Lazy Loading** - SDKs imported only when needed -✅ **Streaming Support** - Real-time response streaming with consistent deltas -✅ **Provider Comparison** - Run same prompt on multiple providers in parallel -✅ **Type-Safe** - Full type hints and automatic JSON Schema generation -✅ **Production-Ready** - Error handling, retries, timeouts, structured logging -✅ **Resource Management** - Async context managers and proper cleanup -✅ **Thread-Safe** - Concurrent tool registration and execution -✅ **Extensible** - Easy to add new providers +## What It Provides -## Architecture +- A thread-safe global tool registry via `@register_tool`. +- JSON Schema generation from Python type hints, including `Optional`, `T | None`, + `list[T]`, `dict`, `Literal`, and multi-type unions. +- A common `HarnessConfig` and `AgentResponse` shape across providers. +- Lazy OpenAI and Claude provider adapters. +- Provider comparison helpers for running the same prompt against multiple adapters. +- A built-in optional You.com search tool adapter. -``` -┌─────────────────────────────────────────┐ -│ AgentHarness │ -│ • switch_provider() │ -│ • run() / stream() │ -│ • compare_providers() │ -└─────────────┬───────────────────────────┘ - │ - ┌──────┴──────┐ - │ │ -┌──────▼──────┐ ┌───▼──────────┐ -│BaseHarness │ │ToolRegistry │ -│(Abstract) │ │@register_tool│ -└──────┬──────┘ └──────────────┘ - │ - ┌────┴────┐ - │ │ -┌─▼─────┐ ┌─▼────────┐ -│OpenAI │ │Claude │ -│Harness│ │Harness │ -└───┬───┘ └───┬──────┘ - │ │ -┌───▼───┐ ┌───▼──────┐ -│OpenAI │ │Claude │ -│Agents │ │Agent SDK │ -│SDK │ │+ MCP │ -└───────┘ └──────────┘ -``` - -## Quick Start +The registry and schema generation work without OpenAI or Anthropic packages installed. +Actual model runs require the provider extras and API credentials. -### 1. Installation +## Install ```bash -# Install with optional dependencies -pip install -e ".[all]" # Both providers -pip install -e ".[openai]" # OpenAI only -pip install -e ".[anthropic]" # Claude only -pip install -e ".[dev]" # Development tools - -# Or install SDKs separately -pip install openai-agents claude-agent-sdk +pip install -e . +pip install -e ".[openai]" +pip install -e ".[anthropic]" +pip install -e ".[all]" +pip install -e ".[dev]" ``` -### 2. Set API Keys +Provider credentials: ```bash export OPENAI_API_KEY="sk-..." export ANTHROPIC_API_KEY="sk-ant-..." ``` -### 3. Basic Usage +## Quick Start ```python import asyncio + from agent_harness import AgentHarness, HarnessConfig, register_tool -# Register a tool once - works with both providers @register_tool(description="Get weather for a city") def get_weather(city: str) -> str: - return f"Weather in {city}: sunny, 72°F" + return f"Weather in {city}: sunny, 72F" -async def main(): +async def main() -> None: config = HarnessConfig( - system_prompt="You are a helpful assistant", + system_prompt="You are a helpful assistant.", tool_names=["get_weather"], max_turns=5, - timeout_sec=30.0 + timeout_sec=30.0, ) - - # Use async context manager for proper cleanup + async with AgentHarness(provider="openai", config=config) as harness: - result = await harness.run("What's the weather in Tokyo?") - print(f"OpenAI: {result.final_output}") - print(f"Latency: {result.latency_ms}ms") - - # Hot-swap to Claude + response = await harness.run("What is the weather in Tokyo?") + print(response.final_output) + await harness.switch_provider("claude") - result = await harness.run("What's the weather in Tokyo?") - print(f"Claude: {result.final_output}") - print(f"Latency: {result.latency_ms}ms") + response = await harness.run("What is the weather in Tokyo?") + print(response.final_output) asyncio.run(main()) ``` -## What's New in v0.2.0 - -🎉 **Major improvements for production use:** - -- **Automatic JSON Schema generation** from Python type hints -- **Custom error taxonomy** with retryable errors -- **Structured logging** with request IDs and timing -- **Request timeouts and retry logic** with exponential backoff -- **Async context managers** for proper resource cleanup -- **Thread-safe registry** for concurrent operations -- **Enhanced configuration** with validation -- **Comprehensive test suite** with pytest -- **CI/CD pipeline** with GitHub Actions +## Tool Registry -See [CHANGELOG.md](CHANGELOG.md) for full details. - -## Core Concepts - -### Tool Registry - -The `@register_tool` decorator adds tools to a global registry accessible by all providers: +Register a tool once and let each provider adapter wrap it for its SDK: ```python -from agent_harness import register_tool - -@register_tool(description="Add two numbers") -def add(a: float, b: float) -> float: - """Adds two numbers together""" - return a + b - -@register_tool(description="Calculate factorial") -def factorial(n: int) -> int: - """Calculate n!""" - if n <= 1: - return 1 - return n * factorial(n - 1) -``` - -Tools are automatically: -- Extracted with parameter types via introspection -- Wrapped for OpenAI using `function_tool` -- Wrapped for Claude using `@tool` + in-process MCP server -- Available to both providers without duplication - -#### Optional provider: You.com web search +from typing import Literal -If your agents need live web results, register the built-in You.com adapter as a tool: +from agent_harness import get_registry, register_tool -```python -import os -from agent_harness import AgentHarness, HarnessConfig, register_you_com_search_tool -os.environ["YDC_API_KEY"] = "your_ydc_api_key" -register_you_com_search_tool(name="you_search") +@register_tool(description="Score a deployment risk") +def score_risk(service: str, mode: Literal["fast", "safe"], retries: int | None = None) -> str: + return f"{service}: {mode}, retries={retries}" -config = HarnessConfig( - system_prompt="Use web search when the user asks for fresh information.", - tool_names=["you_search"], -) -# The tool call shape is: -# you_search(query: str, num_results: int = 5) +tool = get_registry().get("score_risk") +print(tool.json_schema) ``` -Environment variables: -- `YDC_API_KEY` (required, recommended) -- `YOUCOM_API_KEY` (legacy alias supported for compatibility) - -Fallback/error behavior: -- Missing API key returns a non-fatal setup message (no crash) -- Network/API failures return a concise error string to the agent -- Empty result sets return `No web results found.` - -### Configuration +Default values are not required in the generated schema. Parameters without annotations are +treated as strings. -`HarnessConfig` provides unified configuration: +## Configuration ```python from agent_harness import HarnessConfig config = HarnessConfig( - system_prompt="You are an expert data analyst", - model="gpt-4o", # Provider-specific model - max_turns=10, # Max conversation turns - temperature=0.7, # LLM temperature (0.0-2.0) - timeout_sec=30.0, # Request timeout - max_output_tokens=1000, # Max output tokens - top_p=0.9, # Top-p sampling - stop_sequences=["STOP"], # Stop sequences - tool_names=["add", "multiply"], # Specific tools to use - retry_attempts=3, # Number of retries - retry_backoff=1.0, # Backoff multiplier - request_id="custom-id", # Custom request ID (auto-generated if None) - provider_options={ # Provider-specific options - "permission_mode": "acceptEdits" # Claude-specific - } + system_prompt="You are an expert operator.", + model="gpt-4o", + max_turns=10, + temperature=0.7, + timeout_sec=30.0, + max_output_tokens=1000, + top_p=0.9, + stop_sequences=["STOP"], + tool_names=["score_risk"], + retry_attempts=3, + retry_backoff=1.0, + request_id="optional-stable-id", + provider_options={"permission_mode": "acceptEdits"}, ) ``` -### Base Harness Interface +Validation rejects non-positive turns, timeout, retry backoff, and output token limits; +temperatures outside `0.0..2.0`; `top_p` outside `0.0..1.0`; and negative retry counts. -All providers implement `BaseHarness`: +## Optional You.com Search Tool ```python -class BaseHarness(ABC): - @abstractmethod - async def run(self, prompt: str) -> AgentResponse: - """Run agent and return final response""" - pass - - @abstractmethod - async def stream(self, prompt: str) -> AsyncIterator[str]: - """Stream responses in real-time""" - pass -``` - -## Advanced Usage - -### Streaming Responses - -```python -harness = AgentHarness(provider="openai", config=config) - -async for chunk in harness.stream("Write a story about AI"): - print(chunk, end="", flush=True) -``` - -### Provider Comparison - -Compare both providers side-by-side: - -```python -results = await harness.compare_providers( - "Explain quantum computing in simple terms" -) - -for provider, response in results.items(): - print(f"\n{provider.upper()}:") - print(response.final_output) -``` - -### Dynamic Tool Registration - -Register tools at runtime: - -```python -from agent_harness import register_tool, get_registry - -@register_tool(description="Convert miles to kilometers") -def miles_to_km(miles: float) -> float: - return miles * 1.60934 - -print(f"Registered: {list(get_registry().get_all().keys())}") -``` - -### Provider-Specific Features - -#### OpenAI: Guardrails & Handoffs +import os -```python -config = HarnessConfig( - system_prompt="You are an assistant", - provider_options={ - "guardrails": [my_guardrail], - "handoffs": [other_agent] - } -) -``` +from agent_harness import HarnessConfig, register_you_com_search_tool -#### Claude: MCP Servers & Hooks +os.environ["YDC_API_KEY"] = "your_ydc_api_key" +register_you_com_search_tool(name="you_search") -```python config = HarnessConfig( - system_prompt="You are a coding assistant", - provider_options={ - "allowed_tools": ["Read", "Write", "Bash"], - "permission_mode": "acceptEdits", - "cwd": "/path/to/project", - "hooks": { - "PreToolUse": [check_bash_hook] - } - } + system_prompt="Use web search when fresh context matters.", + tool_names=["you_search"], ) ``` -## API Reference - -### AgentHarness - -Main class for hot-swapping providers. +The adapter is intentionally non-fatal: -```python -class AgentHarness: - def __init__( - self, - provider: str = "openai", - config: Optional[HarnessConfig] = None, - api_key: Optional[str] = None - ) - - def switch_provider(self, provider: str) -> None - async def run(self, prompt: str) -> AgentResponse - async def stream(self, prompt: str) -> AsyncIterator[str] - async def compare_providers( - self, - prompt: str, - providers: Optional[list[str]] = None - ) -> dict[str, AgentResponse] -``` - -**Providers:** `"openai"`, `"claude"`, `"anthropic"` - -### HarnessConfig - -```python -@dataclass -class HarnessConfig: - system_prompt: str = "You are a helpful assistant." - model: Optional[str] = None - max_turns: int = 10 - temperature: float = 1.0 - tool_names: Optional[list[str]] = None - provider_options: dict[str, Any] = field(default_factory=dict) -``` - -### AgentResponse - -```python -@dataclass -class AgentResponse: - final_output: str - messages: list[Any] = field(default_factory=list) - metadata: dict[str, Any] = field(default_factory=dict) -``` - -### Tool Registry - -```python -def register_tool( - name: Optional[str] = None, - description: Optional[str] = None -) -> Callable - -def get_registry() -> ToolRegistry - -def register_you_com_search_tool( - name: str = "you_search", - description: str = "Search the web and return summarized results.", - api_key_env: str = "YDC_API_KEY", - endpoint: str = "https://ydc-index.io/v1/search", - timeout_sec: float = 20.0, -) -> str -``` - -## Implementation Details - -### OpenAI Harness - -- Lazily imports `agents` module -- Wraps tools using `function_tool` decorator -- Creates `Agent` with `model_config` -- Uses `Runner.run()` for execution -- Streams via `Runner.run_streamed()` +- Missing API key returns a setup message to the agent. +- HTTP or JSON failures return a concise error string. +- Empty result sets return `No web results found.` -### Claude Harness +## Provider Notes -- Lazily imports `claude_agent_sdk` -- Wraps tools with `@tool` decorator -- Creates in-process MCP server via `create_sdk_mcp_server()` -- Uses `ClaudeAgentOptions` for configuration -- Executes via `query()` or `ClaudeSDKClient` -- Extracts `TextBlock` content from responses +OpenAI provider: -## Examples +- Installs through `.[openai]`. +- Lazily imports `agents`. +- Wraps registered tools with `function_tool`. +- Runs through `Runner.run()` and streams through `Runner.run_streamed()`. -See [example_usage.py](example_usage.py) for comprehensive examples: +Claude provider: -- ✅ Basic hot-swapping -- ✅ Tool usage with both providers -- ✅ Streaming responses -- ✅ Provider comparison -- ✅ Dynamic tool registration -- ✅ Provider-specific options +- Installs through `.[anthropic]`. +- Lazily imports `claude_agent_sdk`. +- Exposes registered tools through an in-process MCP server. +- Accepts Claude-specific options through `provider_options`. -Run examples: +## Development ```bash -python example_usage.py -``` - -## Design Principles - -1. **Single Responsibility**: Each provider adapter handles only its SDK -2. **Lazy Loading**: SDKs imported only when provider is selected -3. **Don't Repeat Yourself**: Tools registered once, used everywhere -4. **Open/Closed**: Easy to add new providers by extending `BaseHarness` -5. **Dependency Inversion**: Code depends on `BaseHarness` abstraction - -## Comparison: OpenAI vs Claude - -| Feature | OpenAI Agents SDK | Claude Agent SDK | -|---------|------------------|------------------| -| **Release** | March 2025 | 2025 | -| **Core API** | Responses API | Model Context Protocol (MCP) | -| **Tool Format** | `function_tool` decorator | `@tool` + MCP server | -| **Streaming** | `run_streamed()` | `query()` iterator | -| **Built-in Tools** | Web search, file search, code interpreter | Read, Write, Bash, etc. | -| **Handoffs** | Native support | Programmatic subagents | -| **Guardrails** | Built-in | Hooks (PreToolUse, PostToolUse) | -| **Sessions** | SQLite, Redis | Session forking | - -## Extending the Harness - -To add a new provider: - -```python -class MyProviderHarness(BaseHarness): - async def run(self, prompt: str) -> AgentResponse: - # Import SDK lazily - from my_sdk import Agent - - # Get registered tools - tools = self._get_registered_tools() - - # Wrap and execute - # ... - - return AgentResponse(...) - - async def stream(self, prompt: str) -> AsyncIterator[str]: - # Implement streaming - # ... - -# Register in AgentHarness.PROVIDERS -AgentHarness.PROVIDERS["myprovider"] = MyProviderHarness +python3 -m venv .venv +source .venv/bin/activate +pip install -e ".[dev]" +python3 -m pytest -q +make bazel-check ``` -## Testing +Remote execution smoke: ```bash -# Run examples -python example_usage.py - -# Test with both providers -export OPENAI_API_KEY="sk-..." -export ANTHROPIC_API_KEY="sk-ant-..." -python example_usage.py +make bazel-rbe-smoke ``` -## Repository Structure - -``` -agent_sdk/ -├── agent_harness.py # Core harness implementation -├── example_usage.py # Comprehensive examples -├── README.md # This file -├── openai-agents-python/ # OpenAI SDK (cloned) -└── claude-agent-sdk-python/ # Claude SDK (cloned) -``` +The `Bazel RBE` GitHub Actions workflow runs on the EvalOps `bazel-rbe-dev` farm when +`BAZEL_RBE_ENABLED=true` is set for the repository. It uses the +`evalops-agent-harness-rbe` and `bazel-rbe` self-hosted labels. -## License +## Repository Layout -MIT +```text +agent_harness.py core registry, config, provider adapters +tests/ pytest coverage +example_usage.py provider examples +BUILD.bazel Bazel pytest target +MODULE.bazel Bazel module dependencies +``` diff --git a/agent_harness.py b/agent_harness.py index d0f3e69..cdfe8c0 100644 --- a/agent_harness.py +++ b/agent_harness.py @@ -11,6 +11,7 @@ import logging import os import threading +import types import urllib.error import urllib.parse import urllib.request @@ -19,9 +20,12 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum -from typing import Any, AsyncIterator, Callable, Optional, Union, get_args, get_origin +from typing import Any, AsyncIterator, Callable, Literal, Optional, Union, get_args, get_origin logger = logging.getLogger(__name__) +_UNION_ORIGINS = (Union,) +if hasattr(types, "UnionType"): + _UNION_ORIGINS = (Union, types.UnionType) class ProviderType(Enum): @@ -82,14 +86,26 @@ def python_type_to_json_schema(param_type: type) -> dict: """ origin = get_origin(param_type) + if param_type is None or param_type is type(None): + return {"type": "null"} + + if origin is Literal: + values = list(get_args(param_type)) + schema: dict[str, Any] = {"enum": values} + value_types = {type(value) for value in values if value is not None} + if len(value_types) == 1: + schema.update(python_type_to_json_schema(next(iter(value_types)))) + return schema + # Handle Optional[T] -> Union[T, None] - if origin is Union: + if origin in _UNION_ORIGINS: args = get_args(param_type) non_none_args = [arg for arg in args if arg is not type(None)] + if not non_none_args: + return {"type": "null"} if len(non_none_args) == 1: return python_type_to_json_schema(non_none_args[0]) - # Multiple types (not Optional) - use first for now - return python_type_to_json_schema(non_none_args[0]) + return {"anyOf": [python_type_to_json_schema(arg) for arg in non_none_args]} # Handle List[T] if origin is list: @@ -388,6 +404,15 @@ def __post_init__(self): if self.retry_attempts < 0: raise ValueError("retry_attempts must be non-negative") + if self.retry_backoff <= 0: + raise ValueError("retry_backoff must be positive") + + if self.max_output_tokens is not None and self.max_output_tokens <= 0: + raise ValueError("max_output_tokens must be positive") + + if self.top_p is not None and not (0.0 <= self.top_p <= 1.0): + raise ValueError("top_p must be between 0.0 and 1.0") + if not self.request_id: self.request_id = str(uuid.uuid4()) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..67bd142 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +import pytest + +from agent_harness import get_registry + + +@pytest.fixture(autouse=True) +def clear_global_registry(): + get_registry().clear() + yield + get_registry().clear() diff --git a/tests/test_config.py b/tests/test_config.py index 7cb4aa7..191303c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -65,6 +65,42 @@ def test_config_validation_retry(): assert config.retry_attempts == 0 +def test_config_validation_retry_backoff(): + """Test retry_backoff validation""" + with pytest.raises(ValueError, match="retry_backoff must be positive"): + HarnessConfig(retry_backoff=0) + + with pytest.raises(ValueError, match="retry_backoff must be positive"): + HarnessConfig(retry_backoff=-1) + + +def test_config_validation_max_output_tokens(): + """Test max_output_tokens validation""" + with pytest.raises(ValueError, match="max_output_tokens must be positive"): + HarnessConfig(max_output_tokens=0) + + with pytest.raises(ValueError, match="max_output_tokens must be positive"): + HarnessConfig(max_output_tokens=-1) + + config = HarnessConfig(max_output_tokens=1) + assert config.max_output_tokens == 1 + + +def test_config_validation_top_p(): + """Test top_p validation""" + with pytest.raises(ValueError, match="top_p must be between"): + HarnessConfig(top_p=-0.1) + + with pytest.raises(ValueError, match="top_p must be between"): + HarnessConfig(top_p=1.1) + + config = HarnessConfig(top_p=0.0) + assert config.top_p == 0.0 + + config = HarnessConfig(top_p=1.0) + assert config.top_p == 1.0 + + def test_config_custom_request_id(): """Test custom request_id""" config = HarnessConfig(request_id="custom-id-123") diff --git a/tests/test_registry.py b/tests/test_registry.py index 81d6be1..7d46420 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,7 +1,8 @@ """Tests for ToolRegistry""" import asyncio -from typing import Optional +import sys +from typing import Literal, Optional import pytest @@ -24,6 +25,28 @@ def test_python_type_to_json_schema_optional(): assert schema == {"type": "string"} +def test_python_type_to_json_schema_pep604_optional(): + """Test Python 3.10 union syntax handling""" + if sys.version_info < (3, 10): + pytest.skip("PEP 604 union syntax requires Python 3.10+") + schema = python_type_to_json_schema(str | None) + assert schema == {"type": "string"} + + +def test_python_type_to_json_schema_union(): + """Test multiple non-null union types""" + if sys.version_info < (3, 10): + pytest.skip("PEP 604 union syntax requires Python 3.10+") + schema = python_type_to_json_schema(str | int) + assert schema == {"anyOf": [{"type": "string"}, {"type": "integer"}]} + + +def test_python_type_to_json_schema_literal(): + """Test Literal enum generation""" + schema = python_type_to_json_schema(Literal["fast", "safe"]) + assert schema == {"type": "string", "enum": ["fast", "safe"]} + + def test_python_type_to_json_schema_list(): """Test List[T] handling""" from typing import List @@ -133,6 +156,17 @@ def test_registry_clear(): assert len(registry.get_all()) == 0 +def test_registry_get_all_returns_copy(): + """Test get_all returns a copy, not the backing registry""" + registry = ToolRegistry() + + registry.register("sample", "Sample", lambda x: x) + tools = registry.get_all() + tools.clear() + + assert registry.get("sample") is not None + + def test_registry_thread_safety(): """Test concurrent access to registry""" import threading