diff --git a/.claude/.codebase-info/.map-state.json b/.claude/.codebase-info/.map-state.json index 4257d781..3f9fee66 100644 --- a/.claude/.codebase-info/.map-state.json +++ b/.claude/.codebase-info/.map-state.json @@ -1,8 +1,8 @@ { "tool": "codebase-mapper", - "version": "2.8.0", - "mappedAt": "2026-07-18", - "gitCommit": "73dbc8be54f4ca32e71d28f5d9a7648dd9628c9b", + "version": "2.11.2", + "mappedAt": "2026-07-23", + "gitCommit": "8a90c1a87de30788e59fd28c606a4e2a2c9c89ed", "documents": [ "architecture.md", "tech-landscape.md", @@ -13,19 +13,21 @@ "dependencies.md", "patterns.md", "coding-style.md", - "onboarding.md" + "onboarding.md", + "structure.md" ], "hashes": { - "INDEX.md": "17eaa0ba82f159465e126597aac349c6e4429330887b58742636734e15854313", + "INDEX.md": "6ca6422261a5a3fd84c714c2aeba11d54a7b9c62d1e0ccf4fb8d22dc42d2bddc", "architecture.md": "7c908f047e6b0f08824ccd7c3f674ad47bd547dde234d0bd424a6d2784c0cbb6", "tech-landscape.md": "cad1a11e5e63013a10cf07657d248196d0e3e4e14203eba7b777c1c123628198", - "directory-structure.md": "7e3fc4c3c8b89b6627beabcb6ddebd74b0ac7094e52718926fc87a94c932df84", - "entry-points.md": "6bd4ccae3d975196e03fc2c4e86007eba89f1f6427232fd8adce78ec97d6b432", - "modules.md": "1238c76664edb1c1667fda23f39507bcab5b760b57a3a3728d7104b2cf8032e7", + "directory-structure.md": "8ae4de1f8554a2cd074a9195b86107e75cdaa78c3b206e2a8d76ad3b78769b9c", + "entry-points.md": "a6dc81ecc5b3bfc67fa8f7c1dcc9f36d80b9be13bab4eb6a6eac238000f5e074", + "modules.md": "1bd4b75fb7af2d455aa2cc12ac9604c7766802158bcaf804e1c9819328f925bf", "communication.md": "49741a7455b0f162881e1cfb2d797dc8e3832cc8782796bbcb8a10529f2c5a0b", "dependencies.md": "b445a920c121e27006fec824add6865d01cf71212a517e5ff8b2058d3999037a", - "patterns.md": "12b96c2d4f8bc7507bcae39b2b385b9db4d0dec95ef6a2049958663aff021705", + "patterns.md": "9c6e46d129b45b3de7fedf9b581af3ecce074c95f1970b56ada1958ee6523f36", "coding-style.md": "2f939544333b927cf8731f5bb906ce03cb3a46afa50c9735011ff8f7e4caca9e", - "onboarding.md": "3d28c671dc095a2aadb297bd7ab200c677779a99c7534e03e55a3d64c935f07e" + "onboarding.md": "ccc30bd73115d603afa7dc1b39344174542f3142a7dde25182fb94fad3e63ded", + "structure.md": "a6d1f09a2b0f5aa3d68ec8f008244acf6cf032050670e9dcf76c8812ab8467b1" } } diff --git a/.claude/.codebase-info/INDEX.md b/.claude/.codebase-info/INDEX.md index b99e8ac1..ffe3388a 100644 --- a/.claude/.codebase-info/INDEX.md +++ b/.claude/.codebase-info/INDEX.md @@ -1,10 +1,11 @@ # Codebase Map — Atomic Agents -*Last Updated: 2026-07-18* +*Last Updated: 2026-07-23* Atomic Agents is a lightweight, modular Python framework for building agentic AI applications as composable, schema-driven building blocks (built on Instructor + Pydantic). This repository is a -`uv`-workspace **monorepo**: the core framework, a TUI tool installer, a tool library, and examples. +`uv`-workspace **monorepo**: the core framework, a Forge CLI client, a vendored-code tool registry, and +examples. **Stack:** Python ≥3.12 · Instructor · Pydantic v2 · LiteLLM · MCP · Textual · uv + Hatchling **Shape:** Monorepo — `atomic-agents/` (core lib) · `atomic-assembler/` (CLI) · `atomic-forge/` (tools) · `atomic-examples/` (examples) @@ -24,6 +25,7 @@ composable, schema-driven building blocks (built on Instructor + Pydantic). This | [patterns.md](./patterns.md) | Atomicity, schema-driven I/O, context providers, testing | | [coding-style.md](./coding-style.md) | Formatting, linting, naming conventions | | [onboarding.md](./onboarding.md) | Setup with uv, tests, docs, common tasks | +| [structure.md](./structure.md) | Layout intent: where new code goes, non-obvious conventions | ## How to use this map diff --git a/.claude/.codebase-info/directory-structure.md b/.claude/.codebase-info/directory-structure.md index 219439e4..227d83af 100644 --- a/.claude/.codebase-info/directory-structure.md +++ b/.claude/.codebase-info/directory-structure.md @@ -1,6 +1,6 @@ # Directory Structure -*Last Updated: 2026-07-18* +*Last Updated: 2026-07-23* ## Root Layout @@ -14,11 +14,14 @@ atomic-agents/ # repo root (uv workspace) │ ├── connectors/mcp/ # Model Context Protocol integration │ └── utils/ # token counter, tool-message formatting │ └── tests/ # pytest suite (agents/, base/, context/, connectors/, utils/) -├── atomic-assembler/ # Textual TUI (`atomic` command) to install forge tools -│ └── atomic_assembler/ # main.py, app.py, screens/, widgets/, utils.py, constants.py -├── atomic-forge/ # library of standalone tools (NOT a package) -│ ├── tools// # one folder per tool: tool/.py, tests/, pyproject.toml -│ └── guides/ # tool authoring guides (e.g. tool_structure.md) +├── atomic-assembler/ # Textual TUI + noninteractive `atomic` Forge client +│ └── atomic_assembler/ # main.py, source/index/download utilities, TUI screens/widgets +├── atomic-forge/ # vendored-code tool registry (NOT a package) +│ ├── tools// # standalone package: tool/, tests/, pyproject.toml, requirements.txt +│ ├── conformance/ # registry package-contract test suite +│ ├── scripts/ # deterministic index generator +│ ├── index.json # generated tool catalog +│ └── guides/ # tool authoring guides ├── atomic-examples/ # 16 runnable example apps (each its own project) ├── claude-plugin/atomic-agents/ # AI-assistant plugin: 7 skills + 2 subagents (Claude Code plugin, │ # also installable cross-tool via `npx skills add eigenwise/atomic-agents`) @@ -42,15 +45,16 @@ prompts and stores conversation history. `connectors/mcp/` bridges to MCP server token accounting via LiteLLM. ### `atomic-assembler/atomic_assembler/` -A Textual terminal UI launched by the `atomic` command (`main.py:main`). `app.py` routes between -`screens/` (main menu, tool explorer, file picker, README viewer); `utils.py` clones the GitHub repo -and copies a selected tool into the user's project. - -### `atomic-forge/tools/` -13 self-contained tools (`arxiv_search`, `calculator`, `tavily_search`, `weather`, -`webpage_scraper`, `wikipedia_search`, …). Each tool folder contains `tool/.py` (Input/Output -`BaseIOSchema` + a `BaseToolConfig` + a `BaseTool` subclass), `tests/`, and its own -`pyproject.toml`/`requirements.txt`. Tools are copied into user projects, not pip-installed. +The `atomic` entry point runs the Textual UI with no subcommand and supports scripting with `list`, +`download`, and `sources` subcommands. Source utilities clone configured Git repositories, resolve an +indexed package only inside the configured tools directory, and copy the full standalone package into the +user’s project. Source/index metadata is treated as untrusted before it reaches the terminal or filesystem. + +### `atomic-forge/` +A shadcn-style collection of 13 self-contained tool packages. Each package contains `tool/.py` +(Input/Output `BaseIOSchema`, `BaseToolConfig`, `BaseTool`), tests, `pyproject.toml`, and +`requirements.txt`; those files stay with the package when it is downloaded. `index.json` is generated +from each package’s metadata, and `conformance/` plus CI enforce the registry contract. ### `atomic-examples/` 16 standalone example apps (`quickstart`, `rag-chatbot`, `deep-research`, `web-search-agent`, diff --git a/.claude/.codebase-info/entry-points.md b/.claude/.codebase-info/entry-points.md index e1129128..155cce34 100644 --- a/.claude/.codebase-info/entry-points.md +++ b/.claude/.codebase-info/entry-points.md @@ -1,6 +1,6 @@ # Entry Points -*Last Updated: 2026-07-05* +*Last Updated: 2026-07-23* ## 1. Library API (the primary entry point) @@ -23,7 +23,12 @@ result = agent.run(BasicChatInputSchema(chat_message="Hello")) | Entry | Type | Purpose | File | |-------|------|---------|------| -| `atomic` | Textual TUI | Browse & install forge tools into a project | `atomic-assembler/atomic_assembler/main.py:main` | +| `atomic` | Textual TUI + command-line interface | Browse and install vendored Forge tools | `atomic-assembler/atomic_assembler/main.py:main` | + +The noninteractive surface is `atomic list`, `atomic download [--dest DIR]`, and +`atomic sources list|add|remove`. Sources are Git repositories, including private ones accessed through +normal Git SSH or credential-helper setup. A qualified name such as `company/weather` resolves a collision +between source catalogs. The TUI remains available when no subcommand is supplied. Declared in `pyproject.toml` (`[project.scripts] atomic = "atomic_assembler.main:main"`). Flags: `--enable-logging`, `--version`. diff --git a/.claude/.codebase-info/modules.md b/.claude/.codebase-info/modules.md index 05be8b89..03af00e5 100644 --- a/.claude/.codebase-info/modules.md +++ b/.claude/.codebase-info/modules.md @@ -1,6 +1,6 @@ # Key Modules -*Last Updated: 2026-07-05* +*Last Updated: 2026-07-23* ## Core framework — `atomic-agents/atomic_agents/` @@ -46,18 +46,20 @@ ### atomic-assembler (CLI) - **Location:** `atomic-assembler/atomic_assembler/` -- **Purpose:** Textual TUI to browse and install forge tools into a user project. -- **Key files:** `main.py` (`main()`; argparse `--enable-logging`, `--version`), `app.py` - (`AtomicAssembler(App)`), `screens/` (`main_menu`, `atomic_tool_explorer`, `file_explorer`, - `tool_info_screen`), `widgets/`, `utils.py` (`GithubRepoCloner`, `AtomicToolManager`), - `constants.py` (GitHub URL, `TOOLS_SUBFOLDER`). +- **Purpose:** Textual TUI plus the `atomic` command-line client for vendoring Forge tool packages. +- **Key files:** `main.py` (argparse commands and TUI entry), `app.py` (`AtomicAssembler(App)`), + `screens/`, `widgets/`, `utils.py` (source clone/index resolution/download), `constants.py` + (`ForgeSource`, source validation and display safety). +- **Source model:** configured sources live in `~/.atomic-assembler/sources.json`; each source names a + Git URL, branch, and tools directory. Git authentication stays with SSH or the user’s credential helper. ### atomic-forge (tools) -- **Location:** `atomic-forge/tools/` -- **Purpose:** 13 standalone tools, each an independent mini-project following the `BaseTool` pattern, - copied into user projects by the assembler (build files such as `pyproject.toml` / `requirements.txt` - / `uv.lock` are skipped on copy). -- **Authoring guide:** `atomic-forge/guides/tool_structure.md`. +- **Location:** `atomic-forge/` +- **Purpose:** a vendored-code registry of 13 standalone tools. `index.json` is generated from package + metadata by `scripts/generate_index.py`; `conformance/` verifies package structure and metadata. + CI requires a fresh index and runs every tool’s test suite. +- **Authoring guide:** `atomic-forge/guides/tool_structure.md`; user-facing workflow: + `docs/guides/atomic_forge.md`. ### atomic-examples - **Location:** `atomic-examples/` diff --git a/.claude/.codebase-info/onboarding.md b/.claude/.codebase-info/onboarding.md index 46c12ea4..ed42b219 100644 --- a/.claude/.codebase-info/onboarding.md +++ b/.claude/.codebase-info/onboarding.md @@ -1,6 +1,6 @@ # Onboarding -*Last Updated: 2026-06-13* +*Last Updated: 2026-07-23* ## Prerequisites - Python **≥3.12** @@ -22,6 +22,8 @@ it. To launch the tool-installer TUI: `uv run atomic` (or `atomic` once it's on |---------|---------| | `uv sync` | Install/update all workspace dependencies | | `uv run pytest --cov=atomic_agents atomic-agents` | Run the core test suite with coverage | +| `uv run pytest atomic-assembler` | Run Atomic Assembler CLI and source-handling tests | +| `uv run pytest atomic-forge/conformance` | Verify Forge package and registry contract | | `uv run black --check atomic-agents atomic-assembler atomic-examples atomic-forge` | Format check | | `uv run flake8 --extend-exclude=.venv atomic-agents atomic-assembler atomic-examples atomic-forge` | Lint | | `cd docs && uv run make html` | Build the Sphinx docs | @@ -30,9 +32,12 @@ it. To launch the tool-installer TUI: `uv run atomic` (or `atomic` once it's on ## Common tasks - **Build a new agent:** define input/output `BaseIOSchema` subclasses, wrap an LLM client with Instructor, pass it to `AtomicAgent[In, Out](AgentConfig(...))`. See `patterns.md` + `entry-points.md`. +- **Use a Forge tool:** run `atomic list`, then `atomic download ` (or + `atomic download /` for a collision). Add private Git registries with + `atomic sources add --tools-path tools`. - **Add a forge tool:** create `atomic-forge/tools//` following - `atomic-forge/guides/tool_structure.md` (input/output schemas, `BaseToolConfig`, a `BaseTool` - subclass, `tests/`). + `atomic-forge/guides/tool_structure.md`, give it tests and dependency metadata, regenerate + `atomic-forge/index.json`, then run the conformance suite. - **Add an example:** create `atomic-examples//` with its own `pyproject.toml`. - **Release:** `build_and_deploy.ps1 ` (needs `PYPI_TOKEN`). diff --git a/.claude/.codebase-info/patterns.md b/.claude/.codebase-info/patterns.md index d1797297..24464fe6 100644 --- a/.claude/.codebase-info/patterns.md +++ b/.claude/.codebase-info/patterns.md @@ -1,6 +1,6 @@ # Patterns & Conventions -*Last Updated: 2026-07-05* +*Last Updated: 2026-07-23* ## Atomicity Build with small, single-purpose, composable parts ("LEGO blocks"): each agent, tool, and context @@ -21,8 +21,11 @@ the next's input. See `atomic-examples/deep-research` and `orchestration-agent`. ## Tools - A tool is a `BaseTool[InputSchema, OutputSchema]` with a `run(params) -> OutputSchema` method and an optional `BaseToolConfig` (override `title`/`description` to disambiguate similar tools). -- Forge tool layout (`atomic-forge/guides/tool_structure.md`): imports → input schema → output - schema(s) → config → tool class + logic → example usage. +- A Forge tool is a complete, vendorable package: source, tests, README, `pyproject.toml`, and + `requirements.txt`. Downloaded tools are owned by the receiving project, never installed as hidden + framework runtime behavior. +- Forge package layout and conformance: `atomic-forge/guides/tool_structure.md` and + `atomic-forge/conformance/`. Check the Forge catalog before generating a tool from scratch. ## Memory - `BaseChatHistory` (`context/base_chat_history.py`) is an interface-only ABC declaring the memory diff --git a/.claude/.codebase-info/structure.md b/.claude/.codebase-info/structure.md new file mode 100644 index 00000000..1dbd7e4b --- /dev/null +++ b/.claude/.codebase-info/structure.md @@ -0,0 +1,31 @@ +# Project structure + +*Last Updated: 2026-07-23* + +Light notes on intent only — the actual tree is in [directory-structure.md](./directory-structure.md). + +## What this project is +The Atomic Agents monorepo: a Python framework for building agentic AI apps as schema-driven, +composable components, published on PyPI as `atomic-agents`, plus its CLI, tool library, examples, +and the Claude Code plugin distributed from this repo. + +## Organizing principle +uv-workspace monorepo whose **root is the published package**. Everything else (assembler, forge +tools, examples) is a workspace member that depends on the core but never the reverse. + +## Where things go +| Kind of thing | Lives in | Notes | +|---------------|----------|-------| +| Framework code | `atomic-agents/atomic_agents/` | the only code that ships to PyPI | +| New forge tool | `atomic-forge/tools//` | follow `atomic-forge/guides/tool_structure.md`; NOT bundled with the package | +| New example | `atomic-examples//` | self-contained project with its own `pyproject.toml` | +| Claude Code plugin | `claude-plugin/` (+ `.claude-plugin/` manifest) | versioned separately from the PyPI package | +| Docs | `docs/` | Sphinx + MyST | + +## Conventions that aren't obvious from the tree +- Tools are deliberately downloadable, not importable: never add a forge tool as a core dependency. +- The package version lives in the root `pyproject.toml`; `__version__` reads installed metadata, + don't hardcode it anywhere. +- Every `BaseIOSchema` subclass needs a non-empty docstring — enforced at class-definition time, + and it flows into the LLM prompt. +- Releases go through `build_and_deploy.ps1` (see the `release` skill), not manual builds. diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 3145f17d..82d9355a 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -40,6 +40,11 @@ jobs: uv sync uv run pip list # Verify installation + - name: Verify Forge Index + run: | + uv run python atomic-forge/scripts/generate_index.py + git diff --exit-code -- atomic-forge/index.json + - name: Verify Black Installation run: | uv run which black || echo "Black not found" @@ -57,3 +62,34 @@ jobs: - name: Run Tests run: uv run pytest --cov=atomic_agents atomic-agents if: success() + + forge-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install Forge test dependencies + run: uv sync --all-packages + + - name: Run Forge conformance suite + run: uv run pytest atomic-forge/conformance + + - name: Run every Forge test suite + run: | + for tool in atomic-forge/tools/*; do + if [ ! -d "$tool/tests" ]; then + echo "Missing test directory: $tool/tests" + exit 1 + fi + uv run --project "$tool" pytest "$tool/tests" + done diff --git a/AGENTS.md b/AGENTS.md index d9e4b7ca..02f32db8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -393,6 +393,8 @@ This project captures repeatable agent workflows as small, single-purpose, verified skills under `.claude/skills/`. Build and maintain them with the **skill-forge** skill — never do a repeatable workflow ad hoc. +Before writing a tool from scratch, check the Forge for an existing one. + **Proactively (notice → propose → ask):** while working, if you notice a multi-step workflow being repeated with no skill, a SKILL.md growing past ~500 lines or sprouting a second capability, a skill whose instructions have drifted diff --git a/atomic-assembler/atomic_assembler/constants.py b/atomic-assembler/atomic_assembler/constants.py index d6b20723..97b18013 100644 --- a/atomic-assembler/atomic_assembler/constants.py +++ b/atomic-assembler/atomic_assembler/constants.py @@ -1,21 +1,98 @@ -from typing import List, Dict, Any, Optional +import re +import unicodedata from dataclasses import dataclass from enum import Enum +from typing import Any, Dict, List, Optional +from urllib.parse import urlsplit, urlunsplit -# Color configuration PRIMARY_COLOR: str = "#AAAA00" SECONDARY_COLOR: str = "#AA00AA" BORDER_STYLE: str = f"bold {SECONDARY_COLOR}" - -# Title configuration TITLE_FONT: str = "big" - -# Repository configuration TOOLS_SUBFOLDER: str = "atomic-forge/tools" - -# Base URL for GitHub repository GITHUB_BASE_URL: str = "https://github.com/eigenwise/atomic-agents.git" GITHUB_BRANCH: str = "main" +_TERMINAL_ESCAPE_SEQUENCE = re.compile(r"\x1b\](?:[^\x07\x1b]|\x1b(?!\\))*(?:\x07|\x1b\\|$)|\x1b\[[0-?]*[ -/]*[@-~]") + + +def _display_safe_source_url(url: str) -> str: + without_escape_sequences = _TERMINAL_ESCAPE_SEQUENCE.sub("", url) + return "".join( + ( + " " + if unicodedata.category(character).startswith("C") or unicodedata.category(character) in {"Zl", "Zp"} + else character + ) + for character in without_escape_sequences + ).strip() + + +def source_url_has_userinfo(url: str) -> bool: + try: + parsed_url = urlsplit(url) + except ValueError: + return False + return "@" in parsed_url.netloc and not _has_ssh_host_user(parsed_url) + + +def _has_ssh_host_user(parsed_url) -> bool: + return parsed_url.scheme.lower() == "ssh" and bool(parsed_url.username) and parsed_url.password is None + + +def redact_source_url(url: str) -> str: + try: + parsed_url = urlsplit(url) + except ValueError: + return "" + _userinfo, separator, host = parsed_url.netloc.rpartition("@") + return _display_safe_source_url( + urlunsplit( + parsed_url._replace( + netloc=( + parsed_url.netloc if _has_ssh_host_user(parsed_url) else f"***@{host}" if separator else parsed_url.netloc + ), + query="***" if parsed_url.query else "", + fragment="***" if parsed_url.fragment else "", + ) + ) + ) + + +def validate_source_url(url: str) -> None: + try: + parsed_url = urlsplit(url) + except ValueError as error: + raise ValueError("Forge source URL is invalid: ") from error + if source_url_has_userinfo(url) or parsed_url.query or parsed_url.fragment: + raise ValueError( + f"Forge source URLs cannot include userinfo, query parameters, or fragments: {redact_source_url(url)}. " + "Use Git credential helpers for private source access." + ) + + +@dataclass(frozen=True) +class ForgeSource: + """A git repository that exposes an Atomic Forge tool index.""" + + name: str + url: str + branch: str + tools_path: str + + def __post_init__(self) -> None: + validate_source_url(self.url) + + def to_dict(self) -> dict[str, str]: + validate_source_url(self.url) + return { + "name": self.name, + "url": self.url, + "branch": self.branch, + "tools_path": self.tools_path, + } + + +DEFAULT_SOURCES: tuple[ForgeSource, ...] = (ForgeSource("official", GITHUB_BASE_URL, GITHUB_BRANCH, TOOLS_SUBFOLDER),) @dataclass @@ -27,7 +104,6 @@ class MenuOption: params: Optional[Dict[str, Any]] = None -# Define menu options as a list of MenuOption instances MENU_OPTIONS: List[MenuOption] = [ MenuOption("Download Tools", "download_tools"), MenuOption("Open Atomic Agents on GitHub", "open_github"), diff --git a/atomic-assembler/atomic_assembler/main.py b/atomic-assembler/atomic_assembler/main.py index 49f4345c..80f1b674 100644 --- a/atomic-assembler/atomic_assembler/main.py +++ b/atomic-assembler/atomic_assembler/main.py @@ -1,7 +1,12 @@ -import logging import argparse -from importlib.metadata import version, PackageNotFoundError +import logging +import sys +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + from atomic_assembler.app import AtomicAssembler +from atomic_assembler.constants import ForgeSource, redact_source_url +from atomic_assembler.utils import AtomicToolManager, GithubRepoCloner, display_safe_text, load_sources, save_sources def setup_logging(enable_logging: bool): @@ -9,35 +14,227 @@ def setup_logging(enable_logging: bool): logging.basicConfig( level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s", - handlers=[ - logging.FileHandler("atomic_assembler.log"), - ], + handlers=[logging.FileHandler("atomic_assembler.log")], ) else: logging.basicConfig(level=logging.CRITICAL) -logger = logging.getLogger(__name__) +def build_parser(pkg_version: str) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Atomic Assembler", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=f"Version: {pkg_version}" + ) + parser.add_argument("--enable-logging", action="store_true", help="Enable logging") + parser.add_argument("--version", action="version", version=f"%(prog)s {pkg_version}") + subparsers = parser.add_subparsers(dest="command") + subparsers.add_parser("list", help="List available Atomic Forge tools") + download_parser = subparsers.add_parser("download", help="Download an Atomic Forge tool") + download_parser.add_argument("name", help="Name of the tool, or source/name when names overlap") + download_parser.add_argument("--dest", help="Directory where the tool will be copied") + + sources_parser = subparsers.add_parser("sources", help="Manage Atomic Forge sources") + source_commands = sources_parser.add_subparsers(dest="sources_command", required=True) + source_commands.add_parser("list", help="List configured Forge sources") + add_parser = source_commands.add_parser("add", help="Add a Forge source") + add_parser.add_argument("name") + add_parser.add_argument("url") + add_parser.add_argument("--branch", default="main") + add_parser.add_argument("--tools-path", default="atomic-forge/tools") + remove_parser = source_commands.add_parser("remove", help="Remove a Forge source") + remove_parser.add_argument("name") + return parser + + +def source_tools(source: ForgeSource) -> list[dict]: + cloner = GithubRepoCloner(source.url, branch=source.branch, tools_path=source.tools_path) + try: + cloner.clone() + tools = AtomicToolManager.get_indexed_forge_tools(cloner.repo_path, cloner.tools_path) + return [{**tool, "source": source.name} for tool in tools] + finally: + cloner.cleanup() + + +def get_forge_tools(sources: list[ForgeSource] | None = None) -> tuple[list[dict], list[str], int]: + tools = [] + failures = [] + successful_sources = 0 + for source in sources or load_sources(): + try: + tools.extend(source_tools(source)) + successful_sources += 1 + except Exception as error: + failures.append(display_safe_text(f"Could not read source '{source.name}': {error}")) + return tools, failures, successful_sources -def main(): +def list_tools(sources: list[ForgeSource] | None = None) -> int: + try: + tools, failures, successful_sources = get_forge_tools(sources) + except Exception as error: + print(display_safe_text(f"Could not load Atomic Forge sources: {error}"), file=sys.stderr) + return 1 + + for failure in failures: + print(failure, file=sys.stderr) + for tool in tools: + print( + f"{display_safe_text(tool['source'])}/{display_safe_text(tool['name'])} - {display_safe_text(tool['description'])}" + ) + return 0 if successful_sources else 1 + + +def matching_tools(name: str, tools: list[dict]) -> list[dict]: + normalized_name = name.lower() + return [tool for tool in tools if normalized_name in {tool["name"].lower(), Path(tool["path"]).name.lower()}] + + +def download_tool(name: str, destination: str | None, sources: list[ForgeSource] | None = None) -> int: + try: + configured_sources = sources or load_sources() + except Exception as error: + print(display_safe_text(f"Could not load Atomic Forge sources: {error}"), file=sys.stderr) + return 1 + + source_name, separator, tool_name = name.partition("/") + if separator: + selected_sources = [source for source in configured_sources if source.name == source_name] + if not selected_sources: + print(f"Unknown source '{display_safe_text(source_name)}'.", file=sys.stderr) + return 1 + else: + selected_sources = configured_sources + tool_name = name + + cloners = [] + tools = [] + try: + for source in selected_sources: + cloner = GithubRepoCloner(source.url, branch=source.branch, tools_path=source.tools_path) + cloners.append(cloner) + try: + cloner.clone() + tools.extend( + {**tool, "source": source.name} + for tool in AtomicToolManager.get_indexed_forge_tools(cloner.repo_path, cloner.tools_path) + ) + except Exception as error: + print(display_safe_text(f"Could not read source '{source.name}': {error}"), file=sys.stderr) + + matches = matching_tools(tool_name, tools) + if not matches: + available_tools = ", ".join( + f"{display_safe_text(tool['source'])}/{display_safe_text(tool['name'])}" for tool in tools + ) + print(f"Unknown tool '{display_safe_text(name)}'. Available tools: {available_tools}", file=sys.stderr) + return 1 + if len(matches) > 1: + choices = ", ".join(f"{display_safe_text(tool['source'])}/{display_safe_text(tool['name'])}" for tool in matches) + print( + f"Tool '{display_safe_text(tool_name)}' exists in multiple sources. Use /: {choices}", + file=sys.stderr, + ) + return 1 + + tool = matches[0] + target = Path(destination) if destination else Path.cwd() / Path(tool["path"]).name + if target.exists(): + print(f"Destination already exists: {display_safe_text(str(target))}", file=sys.stderr) + return 1 + + copied_path = AtomicToolManager.copy_atomic_tool_to_destination(tool["path"], target) + except Exception as error: + print(display_safe_text(f"Could not download Atomic Forge tool: {error}"), file=sys.stderr) + return 1 + finally: + for cloner in cloners: + cloner.cleanup() + + print( + f"Downloaded {display_safe_text(tool['source'])}/{display_safe_text(tool['name'])} to {display_safe_text(copied_path)}" + ) + print(f"Install dependencies with: pip install -r {display_safe_text(str(Path(copied_path) / 'requirements.txt'))}") + return 0 + + +def list_sources() -> int: + try: + sources = load_sources() + except Exception as error: + print(display_safe_text(f"Could not load Atomic Forge sources: {error}"), file=sys.stderr) + return 1 + for source in sources: + print( + " ".join( + display_safe_text(value) + for value in (source.name, redact_source_url(source.url), source.branch, source.tools_path) + ) + ) + return 0 + + +def add_source(name: str, url: str, branch: str, tools_path: str) -> int: + if "/" in name: + print("Source names cannot contain '/'.", file=sys.stderr) + return 1 + try: + sources = load_sources() + except Exception as error: + print(display_safe_text(f"Could not load Atomic Forge sources: {error}"), file=sys.stderr) + return 1 + if any(source.name == name for source in sources): + print(f"Source already exists: {display_safe_text(name)}", file=sys.stderr) + return 1 + try: + source = ForgeSource(name, url, branch, tools_path) + except ValueError as error: + print(display_safe_text(str(error)), file=sys.stderr) + return 1 + save_sources([*sources, source]) + print(f"Added source '{display_safe_text(name)}'.") + return 0 + + +def remove_source(name: str) -> int: + try: + sources = load_sources() + except Exception as error: + print(display_safe_text(f"Could not load Atomic Forge sources: {error}"), file=sys.stderr) + return 1 + remaining_sources = [source for source in sources if source.name != name] + if len(remaining_sources) == len(sources): + print(f"Unknown source '{display_safe_text(name)}'.", file=sys.stderr) + return 1 + save_sources(remaining_sources) + print(f"Removed source '{display_safe_text(name)}'.") + return 0 + + +def main() -> int: try: pkg_version = version("atomic-agents") except PackageNotFoundError: pkg_version = "unknown" - parser = argparse.ArgumentParser( - description="Atomic Assembler", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=f"Version: {pkg_version}" - ) - parser.add_argument("--enable-logging", action="store_true", help="Enable logging") - parser.add_argument("--version", action="version", version=f"%(prog)s {pkg_version}") + parser = build_parser(pkg_version) args = parser.parse_args() - setup_logging(args.enable_logging) + if args.command == "list": + return list_tools() + if args.command == "download": + return download_tool(args.name, args.dest) + if args.command == "sources": + if args.sources_command == "list": + return list_sources() + if args.sources_command == "add": + return add_source(args.name, args.url, args.branch, args.tools_path) + return remove_source(args.name) + app = AtomicAssembler() app.run() + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/atomic-assembler/atomic_assembler/utils.py b/atomic-assembler/atomic_assembler/utils.py index 6515a3da..ddac6a1f 100644 --- a/atomic-assembler/atomic_assembler/utils.py +++ b/atomic-assembler/atomic_assembler/utils.py @@ -1,27 +1,120 @@ +import json import logging import os +import re import shutil import tempfile +import unicodedata +from pathlib import Path import git import yaml -from pathlib import Path -from atomic_assembler.constants import GITHUB_BRANCH, TOOLS_SUBFOLDER + +from atomic_assembler.constants import DEFAULT_SOURCES, GITHUB_BRANCH, TOOLS_SUBFOLDER, ForgeSource, validate_source_url + +SOURCES_CONFIG_PATH = Path.home() / ".atomic-assembler" / "sources.json" +_TERMINAL_ESCAPE_SEQUENCE = re.compile(r"\x1b\](?:[^\x07\x1b]|\x1b(?!\\))*(?:\x07|\x1b\\|$)|\x1b\[[0-?]*[ -/]*[@-~]") + + +def display_safe_text(value: str) -> str: + without_escape_sequences = _TERMINAL_ESCAPE_SEQUENCE.sub("", value) + return "".join( + ( + " " + if unicodedata.category(character).startswith("C") or unicodedata.category(character) in {"Zl", "Zp"} + else character + ) + for character in without_escape_sequences + ).strip() + + +def _validated_index_text(value: object, field_name: str) -> str: + if not isinstance(value, str): + raise ValueError(f"Forge index tool {field_name} must be a string") + safe_value = display_safe_text(value) + if not safe_value: + raise ValueError(f"Forge index tool {field_name} must contain displayable text") + return safe_value + + +def _validate_tool_symlink(tool_directory: Path, symlink: Path) -> None: + link_target = Path(os.readlink(symlink)) + try: + resolved_target = (symlink.parent / link_target).resolve() + resolved_target.relative_to(tool_directory) + except (OSError, RuntimeError, ValueError): + relative_link = display_safe_text(str(symlink.relative_to(tool_directory))) + raise ValueError(f"Tool contains an unsafe symlink: {relative_link}") from None + if link_target.is_absolute(): + relative_link = display_safe_text(str(symlink.relative_to(tool_directory))) + raise ValueError(f"Tool contains an unsafe symlink: {relative_link}") + + +def _validate_tool_symlinks(tool_path: str | Path) -> None: + tool_directory = Path(tool_path).resolve() + for directory, directory_names, file_names in os.walk(tool_directory, followlinks=False): + for name in [*directory_names, *file_names]: + candidate = Path(directory, name) + if candidate.is_symlink(): + _validate_tool_symlink(tool_directory, candidate) + + +def source_config_path() -> Path: + return SOURCES_CONFIG_PATH + + +def load_sources(config_path: Path | None = None) -> list[ForgeSource]: + config_path = config_path or source_config_path() + if not config_path.is_file(): + return list(DEFAULT_SOURCES) + + with config_path.open(encoding="utf-8") as file: + configured_sources = json.load(file) + if isinstance(configured_sources, dict): + configured_sources = configured_sources.get("sources") + if not isinstance(configured_sources, list): + raise ValueError("sources.json must contain a list of sources") + + sources = [] + for source in configured_sources: + if not isinstance(source, dict): + raise ValueError("Each configured source must be an object") + try: + name = source["name"] + url = source["url"] + except KeyError as error: + raise ValueError(f"Configured source is missing {error.args[0]}") from error + branch = source.get("branch", GITHUB_BRANCH) + tools_path = source.get("tools_path", TOOLS_SUBFOLDER) + if not all(isinstance(value, str) and value for value in (name, url, branch, tools_path)): + raise ValueError("Configured source fields must be non-empty strings") + sources.append(ForgeSource(name, url, branch, tools_path)) + return sources + + +def save_sources(sources: list[ForgeSource], config_path: Path | None = None) -> None: + config_path = config_path or source_config_path() + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text(json.dumps([source.to_dict() for source in sources], indent=2) + "\n", encoding="utf-8") class GithubRepoCloner: - def __init__(self, base_url: str, branch: str = "main"): + def __init__(self, base_url: str, branch: str = GITHUB_BRANCH, tools_path: str = TOOLS_SUBFOLDER): + validate_source_url(base_url) self.repo_url = base_url + self.branch = branch self.temp_dir = tempfile.mkdtemp() - self.repo_path = os.path.join(self.temp_dir, os.path.basename(base_url).replace(".git", "")) - self.tools_path = os.path.join(self.repo_path, TOOLS_SUBFOLDER) + repo_name = Path(base_url.rstrip("/")).name.removesuffix(".git") + self.repo_path = os.path.join(self.temp_dir, repo_name) + self.tools_path = os.path.join(self.repo_path, tools_path) def clone(self): + validate_source_url(self.repo_url) try: - _ = git.Repo.clone_from(self.repo_url, self.repo_path, branch=GITHUB_BRANCH) - logging.info(f"Repository cloned to {self.repo_path} on branch {GITHUB_BRANCH}") - except git.GitCommandError as e: - logging.error(f"Failed to clone repository: {e}") + _ = git.Repo.clone_from(self.repo_url, self.repo_path, branch=self.branch) + logging.info(f"Repository cloned to {self.repo_path} on branch {self.branch}") + except git.GitCommandError as error: + logging.error(f"Failed to clone repository: {error}") raise def cleanup(self): @@ -33,68 +126,119 @@ class AtomicToolManager: def read_tool_config(tool_path): config_path = os.path.join(tool_path, "config.yaml") try: - with open(config_path, "r") as f: - return yaml.safe_load(f) + with open(config_path, "r") as file: + return yaml.safe_load(file) except FileNotFoundError: return None - except Exception as e: - return f"Error reading config file: {e}" + except Exception as error: + return f"Error reading config file: {error}" @staticmethod def get_atomic_tools(tools_path: str) -> list[dict]: - """ - Get a list of atomic tools from the given tools path. - - Args: - tools_path (str): The path to the directory containing atomic tools. - - Returns: - list[dict]: A list of dictionaries containing tool information. - """ tools = [] - for item in os.listdir(tools_path): + for item in sorted(os.listdir(tools_path)): item_path = os.path.join(tools_path, item) if os.path.isdir(item_path): - # Convert snake_case to Title Case - tool_name = " ".join(word.capitalize() for word in item.split("_")) - tools.append( - { - "name": tool_name, - "path": item_path, - } - ) + tools.append({"name": " ".join(word.capitalize() for word in item.split("_")), "path": item_path}) return tools + @staticmethod + def get_forge_tools(repo_path: str, tools_path: str) -> list[dict]: + index_path = Path(tools_path).parent / "index.json" + if index_path.is_file(): + return AtomicToolManager.get_indexed_forge_tools(repo_path, tools_path) + + return [ + { + "name": Path(tool["path"]).name.replace("_", "-"), + "path": tool["path"], + "description": AtomicToolManager.get_tool_description(tool["path"]), + } + for tool in AtomicToolManager.get_atomic_tools(tools_path) + ] + + @staticmethod + def get_indexed_forge_tools(repo_path: str, tools_path: str) -> list[dict]: + index_path = Path(tools_path).parent / "index.json" + if not index_path.is_file(): + raise FileNotFoundError(f"Missing forge index: {index_path}") + with index_path.open(encoding="utf-8") as file: + index = json.load(file) + if not isinstance(index.get("tools"), list): + raise ValueError(f"Forge index has no tools list: {index_path}") + + tools_directory = Path(tools_path).resolve() + if not tools_directory.is_dir(): + raise NotADirectoryError(f"Configured tools directory is not a directory: {tools_path}") + + indexed_tools = [] + for tool in index["tools"]: + tool_path = tool.get("path") if isinstance(tool, dict) else None + if not isinstance(tool_path, str) or not tool_path: + raise ValueError(f"Forge index tool path must be a non-empty string: {index_path}") + + safe_tool_path = display_safe_text(tool_path) + relative_tool_path = Path(tool_path) + if relative_tool_path.is_absolute() or ".." in relative_tool_path.parts: + raise ValueError(f"Forge index tool path must stay within configured tools directory: {safe_tool_path}") + + resolved_tool_path = (index_path.parent / relative_tool_path).resolve() + try: + resolved_tool_path.relative_to(tools_directory) + except ValueError as error: + raise ValueError( + f"Forge index tool path must stay within configured tools directory: {safe_tool_path}" + ) from error + if not resolved_tool_path.is_dir(): + raise NotADirectoryError(f"Forge index tool path is not a directory: {safe_tool_path}") + + indexed_tools.append( + { + "name": _validated_index_text(tool.get("name"), "name"), + "path": str(resolved_tool_path), + "description": _validated_index_text(tool.get("description", "No description available."), "description"), + } + ) + return indexed_tools + + @staticmethod + def get_tool_description(tool_path: str) -> str: + try: + readme = Path(tool_path, "README.md").read_text(encoding="utf-8") + except FileNotFoundError: + return "No description available." + + for line in readme.splitlines(): + description = line.strip() + if description and not description.startswith("#"): + return description + return "No description available." + @staticmethod def copy_atomic_tool(tool_path, destination): logging.info(f"copy_atomic_tool called with tool_path: {tool_path}, destination: {destination}") try: - tool_name = os.path.basename(tool_path) - local_tool_path = os.path.join(destination, tool_name) - logging.info(f"Copying tool from {tool_path} to {local_tool_path}") - - if not os.path.exists(tool_path): - logging.error(f"Source path does not exist: {tool_path}") - raise FileNotFoundError(f"Source path does not exist: {tool_path}") - - if not os.path.exists(destination): - logging.error(f"Destination path does not exist: {destination}") - raise FileNotFoundError(f"Destination path does not exist: {destination}") - - shutil.copytree( - tool_path, - local_tool_path, - ignore=shutil.ignore_patterns("requirements.txt", "pyproject.toml", ".coveragerc", "uv.lock"), - ) - logging.info(f"Tool successfully copied to {local_tool_path}") - return local_tool_path - except Exception as e: - logging.error(f"Error copying tool: {str(e)}", exc_info=True) - raise Exception(f"Error copying tool: {e}") + local_tool_path = os.path.join(destination, os.path.basename(tool_path)) + return AtomicToolManager.copy_atomic_tool_to_destination(tool_path, local_tool_path) + except Exception as error: + logging.error(f"Error copying tool: {error}", exc_info=True) + raise Exception(f"Error copying tool: {error}") from error + + @staticmethod + def copy_atomic_tool_to_destination(tool_path, destination): + logging.info(f"Copying tool from {tool_path} to {destination}") + if not os.path.exists(tool_path): + raise FileNotFoundError(f"Source path does not exist: {tool_path}") + if os.path.exists(destination): + raise FileExistsError(f"Destination already exists: {destination}") + + _validate_tool_symlinks(tool_path) + shutil.copytree(tool_path, destination, symlinks=True, ignore=shutil.ignore_patterns(".coveragerc", "uv.lock")) + logging.info(f"Tool successfully copied to {destination}") + return str(destination) @staticmethod def load_env_file(file_path: Path) -> dict: - """Load environment variables from a .env file.""" env_vars = {} if file_path.exists(): with open(file_path, "r") as file: @@ -107,20 +251,11 @@ def load_env_file(file_path: Path) -> dict: @staticmethod def read_readme(tool_path: str) -> str: - """ - Read the README.md file from the tool directory. - - Args: - tool_path (str): The path to the tool directory. - - Returns: - str: The contents of the README.md file, or an error message if not found. - """ readme_path = os.path.join(tool_path, "README.md") try: - with open(readme_path, "r", encoding="utf-8") as f: - return f.read() + with open(readme_path, "r", encoding="utf-8") as file: + return file.read() except FileNotFoundError: return "README.md not found for this tool." - except Exception as e: - return f"Error reading README.md: {str(e)}" + except Exception as error: + return f"Error reading README.md: {error}" diff --git a/atomic-assembler/tests/test_utils.py b/atomic-assembler/tests/test_utils.py new file mode 100644 index 00000000..7cf899f7 --- /dev/null +++ b/atomic-assembler/tests/test_utils.py @@ -0,0 +1,577 @@ +import json +import os +import tempfile +from pathlib import Path + +import git +import pytest + +from atomic_assembler import main as assembler_main +from atomic_assembler import utils as assembler_utils +from atomic_assembler.constants import DEFAULT_SOURCES, ForgeSource, redact_source_url +from atomic_assembler.utils import AtomicToolManager, GithubRepoCloner, load_sources, save_sources + + +def _symlinks_supported() -> bool: + with tempfile.TemporaryDirectory() as probe_dir: + try: + (Path(probe_dir) / "link").symlink_to("target") + except (OSError, NotImplementedError): + return False + return True + + +requires_symlinks = pytest.mark.skipif( + not _symlinks_supported(), + reason="creating symlinks requires a privilege this platform does not grant", +) + + +def test_github_repo_cloner_uses_configured_branch(monkeypatch): + cloned = {} + + def clone_from(repo_url, repo_path, branch): + cloned.update(repo_url=repo_url, repo_path=repo_path, branch=branch) + + monkeypatch.setattr("atomic_assembler.utils.git.Repo.clone_from", clone_from) + cloner = GithubRepoCloner("https://github.com/example/tools.git", branch="feature/test") + + try: + cloner.clone() + finally: + cloner.cleanup() + + assert cloned["branch"] == "feature/test" + + +def test_github_repo_cloner_rejects_credentials_before_clone(monkeypatch): + clone_called = False + + def clone_from(repo_url, repo_path, branch): + nonlocal clone_called + clone_called = True + + monkeypatch.setattr("atomic_assembler.utils.git.Repo.clone_from", clone_from) + + with pytest.raises(ValueError) as error: + GithubRepoCloner("https://example.com/forge.git?access_token=diagnostic-secret") + + assert "diagnostic-secret" not in str(error.value) + assert not clone_called + + +def test_copy_atomic_tool_keeps_dependency_metadata(tmp_path): + tool_path = tmp_path / "example_tool" + tool_path.mkdir() + for filename in ("pyproject.toml", "requirements.txt", "uv.lock", ".coveragerc"): + (tool_path / filename).write_text(filename) + + destination = tmp_path / "destination" + destination.mkdir() + + copied_path = Path(AtomicToolManager.copy_atomic_tool(tool_path, destination)) + + assert (copied_path / "pyproject.toml").is_file() + assert (copied_path / "requirements.txt").is_file() + assert not (copied_path / "uv.lock").exists() + assert not (copied_path / ".coveragerc").exists() + + +@requires_symlinks +@pytest.mark.parametrize( + ("link_path", "link_target"), + [ + (Path("escape.txt"), Path("..", "outside.txt")), + (Path("nested", "escape.txt"), Path("..", "..", "outside.txt")), + ], +) +def test_copy_atomic_tool_rejects_symlinks_that_escape_tool_directory(tmp_path, link_path, link_target): + outside_file = tmp_path / "outside.txt" + outside_file.write_text("sensitive") + tool_path = tmp_path / "example_tool" + tool_path.mkdir() + symlink = tool_path / link_path + symlink.parent.mkdir(parents=True, exist_ok=True) + symlink.symlink_to(link_target) + destination = tmp_path / "downloaded" + + with pytest.raises(ValueError, match="unsafe symlink"): + AtomicToolManager.copy_atomic_tool_to_destination(tool_path, destination) + + assert not destination.exists() + + +@requires_symlinks +def test_copy_atomic_tool_preserves_safe_relative_symlink(tmp_path): + tool_path = tmp_path / "example_tool" + tool_path.mkdir() + (tool_path / "target.txt").write_text("tool data") + (tool_path / "link.txt").symlink_to("target.txt") + destination = tmp_path / "downloaded" + + AtomicToolManager.copy_atomic_tool_to_destination(tool_path, destination) + + copied_link = destination / "link.txt" + assert copied_link.is_symlink() + assert copied_link.readlink() == Path("target.txt") + + +def test_build_parser_parses_download_destination_and_source_commands(): + parser = assembler_main.build_parser("1.2.3") + download_args = parser.parse_args(["download", "calculator", "--dest", "tools/calculator"]) + add_args = parser.parse_args(["sources", "add", "team", "file:///tools.git", "--branch", "release"]) + + assert download_args.command == "download" + assert download_args.name == "calculator" + assert download_args.dest == "tools/calculator" + assert add_args.sources_command == "add" + assert add_args.name == "team" + assert add_args.branch == "release" + + +def test_source_config_uses_defaults_until_first_write(tmp_path): + config_path = tmp_path / "sources.json" + source = ForgeSource("team", "file:///team-forge.git", "main", "tools") + + assert load_sources(config_path) == list(DEFAULT_SOURCES) + + save_sources([source], config_path) + + assert load_sources(config_path) == [source] + + +def test_source_commands_create_and_update_config(tmp_path, monkeypatch, capsys): + config_path = tmp_path / ".atomic-assembler" / "sources.json" + monkeypatch.setattr(assembler_utils, "SOURCES_CONFIG_PATH", config_path) + + assert assembler_main.add_source("team", "file:///team-forge.git", "release", "tools") == 0 + assert load_sources(config_path) == [ + *DEFAULT_SOURCES, + ForgeSource("team", "file:///team-forge.git", "release", "tools"), + ] + assert assembler_main.remove_source("team") == 0 + assert load_sources(config_path) == list(DEFAULT_SOURCES) + assert "Added source 'team'." in capsys.readouterr().out + + +def test_add_source_rejects_url_with_userinfo(tmp_path, monkeypatch, capsys): + config_path = tmp_path / ".atomic-assembler" / "sources.json" + monkeypatch.setattr(assembler_utils, "SOURCES_CONFIG_PATH", config_path) + + assert assembler_main.add_source("team", "https://user:super-secret@example.com/forge.git", "main", "tools") == 1 + + assert not config_path.exists() + error = capsys.readouterr().err + assert "super-secret" not in error + assert "https://***@example.com/forge.git" in error + + +def test_add_source_accepts_ssh_uri(tmp_path, monkeypatch): + config_path = tmp_path / ".atomic-assembler" / "sources.json" + monkeypatch.setattr(assembler_utils, "SOURCES_CONFIG_PATH", config_path) + source_url = "ssh://git@github.com/eigenwise/atomic-agents.git" + + assert assembler_main.add_source("team", source_url, "main", "tools") == 0 + assert load_sources(config_path) == [*DEFAULT_SOURCES, ForgeSource("team", source_url, "main", "tools")] + + +def test_add_source_accepts_scp_style_ssh_url(tmp_path, monkeypatch): + config_path = tmp_path / ".atomic-assembler" / "sources.json" + monkeypatch.setattr(assembler_utils, "SOURCES_CONFIG_PATH", config_path) + source_url = "git@github.com:eigenwise/atomic-agents.git" + + assert assembler_main.add_source("team", source_url, "main", "tools") == 0 + assert load_sources(config_path) == [*DEFAULT_SOURCES, ForgeSource("team", source_url, "main", "tools")] + + +def test_add_source_rejects_ssh_password_without_exposing_it(tmp_path, monkeypatch, capsys): + config_path = tmp_path / ".atomic-assembler" / "sources.json" + monkeypatch.setattr(assembler_utils, "SOURCES_CONFIG_PATH", config_path) + + assert ( + assembler_main.add_source("team", "ssh://git:super-secret@github.com/eigenwise/atomic-agents.git", "main", "tools") + == 1 + ) + + assert not config_path.exists() + error = capsys.readouterr().err + assert "super-secret" not in error + assert "ssh://***@github.com/eigenwise/atomic-agents.git" in error + + +@pytest.mark.parametrize( + ("url", "secret", "redacted_url"), + [ + ("https://example.com/forge.git?access_token=query-secret", "query-secret", "https://example.com/forge.git?***"), + ("https://example.com/forge.git#token=fragment-secret", "fragment-secret", "https://example.com/forge.git#***"), + ], +) +def test_add_source_rejects_url_credentials_outside_userinfo(tmp_path, monkeypatch, capsys, url, secret, redacted_url): + config_path = tmp_path / ".atomic-assembler" / "sources.json" + monkeypatch.setattr(assembler_utils, "SOURCES_CONFIG_PATH", config_path) + + assert assembler_main.add_source("team", url, "main", "tools") == 1 + + assert not config_path.exists() + error = capsys.readouterr().err + assert secret not in error + assert redacted_url in error + + +def test_save_sources_revalidates_url_before_persistence(tmp_path): + config_path = tmp_path / "sources.json" + source = ForgeSource("team", "https://example.com/forge.git", "main", "tools") + object.__setattr__(source, "url", "https://example.com/forge.git?access_token=stored-secret") + + with pytest.raises(ValueError) as error: + save_sources([source], config_path) + + assert "stored-secret" not in str(error.value) + assert not config_path.exists() + + +def test_load_sources_rejects_legacy_userinfo_without_exposing_credentials(tmp_path): + config_path = tmp_path / "sources.json" + config_path.write_text( + json.dumps( + [ + { + "name": "team", + "url": "https://user:super-secret@example.com/forge.git", + "branch": "main", + "tools_path": "tools", + } + ] + ) + ) + + with pytest.raises(ValueError) as error: + load_sources(config_path) + + assert "super-secret" not in str(error.value) + assert "https://***@example.com/forge.git" in str(error.value) + + +@pytest.mark.parametrize( + ("url", "secret"), + [ + ("https://example.com/forge.git?access_token=query-secret", "query-secret"), + ("https://example.com/forge.git#token=fragment-secret", "fragment-secret"), + ], +) +def test_load_sources_rejects_legacy_url_credentials_without_exposing_them(tmp_path, url, secret): + config_path = tmp_path / "sources.json" + config_path.write_text(json.dumps([{"name": "team", "url": url, "branch": "main", "tools_path": "tools"}])) + + with pytest.raises(ValueError) as error: + load_sources(config_path) + + assert secret not in str(error.value) + + +def test_list_sources_redacts_userinfo(monkeypatch, capsys): + source = ForgeSource("team", "https://example.com/forge.git", "main", "tools") + object.__setattr__(source, "url", "https://user:super-secret@example.com/forge.git") + monkeypatch.setattr(assembler_main, "load_sources", lambda: [source]) + + assert assembler_main.list_sources() == 0 + + output = capsys.readouterr().out + assert "super-secret" not in output + assert "https://***@example.com/forge.git" in output + assert redact_source_url(source.url) in output + + +@pytest.mark.parametrize( + ("url", "secret", "redacted_url"), + [ + ("https://example.com/forge.git?access_token=query-secret", "query-secret", "https://example.com/forge.git?***"), + ("https://example.com/forge.git#token=fragment-secret", "fragment-secret", "https://example.com/forge.git#***"), + ], +) +def test_list_sources_redacts_url_credentials_outside_userinfo(monkeypatch, capsys, url, secret, redacted_url): + source = ForgeSource("team", "https://example.com/forge.git", "main", "tools") + object.__setattr__(source, "url", url) + monkeypatch.setattr(assembler_main, "load_sources", lambda: [source]) + + assert assembler_main.list_sources() == 0 + + output = capsys.readouterr().out + assert secret not in output + assert redacted_url in output + + +@pytest.mark.parametrize("control", ["\x1b]52;c;clipboard\x07", "\x1b[31m", "\N{RIGHT-TO-LEFT OVERRIDE}"]) +def test_source_management_output_cannot_emit_terminal_controls(monkeypatch, capsys, control): + source = ForgeSource( + f"team{control}", + f"https://example.com/forge{control}.git", + f"main{control}", + f"tools{control}", + ) + monkeypatch.setattr(assembler_main, "load_sources", lambda: [source]) + + assert assembler_main.list_sources() == 0 + + output = capsys.readouterr().out + assert control not in output + assert "\x1b" not in output + assert "\x07" not in output + assert "\N{RIGHT-TO-LEFT OVERRIDE}" not in output + + +@pytest.mark.parametrize("control", ["\x1b]52;c;clipboard\x07", "\x1b[31m", "\N{RIGHT-TO-LEFT OVERRIDE}"]) +def test_rejected_source_url_cannot_emit_terminal_controls(tmp_path, monkeypatch, capsys, control): + config_path = tmp_path / ".atomic-assembler" / "sources.json" + monkeypatch.setattr(assembler_utils, "SOURCES_CONFIG_PATH", config_path) + + assert ( + assembler_main.add_source("team", f"https://example.com/forge{control}.git?access_token=secret", "main", "tools") == 1 + ) + + output = capsys.readouterr().err + assert "secret" not in output + assert control not in output + assert "\x1b" not in output + assert "\x07" not in output + assert "\N{RIGHT-TO-LEFT OVERRIDE}" not in output + + +@pytest.mark.parametrize("control", ["\x1b]52;c;clipboard\x07", "\x1b[31m", "\N{RIGHT-TO-LEFT OVERRIDE}"]) +def test_source_errors_cannot_emit_terminal_controls(monkeypatch, capsys, control): + source = ForgeSource(f"team{control}", "https://example.com/forge.git", "main", "tools") + + def raise_source_error(_source): + raise RuntimeError(f"unavailable{control}") + + monkeypatch.setattr(assembler_main, "source_tools", raise_source_error) + + assert assembler_main.list_tools([source]) == 1 + + output = capsys.readouterr().err + assert control not in output + assert "\x1b" not in output + assert "\x07" not in output + assert "\N{RIGHT-TO-LEFT OVERRIDE}" not in output + + +def test_get_forge_tools_falls_back_to_tool_directories(tmp_path): + tools_path = tmp_path / "atomic-forge" / "tools" + tool_path = tools_path / "example_tool" + tool_path.mkdir(parents=True) + (tool_path / "README.md").write_text("# Example Tool\n\nOne-line description.\n") + + tools = AtomicToolManager.get_forge_tools(tmp_path, tools_path) + + assert tools == [ + { + "name": "example-tool", + "path": str(tool_path), + "description": "One-line description.", + } + ] + + +@pytest.mark.parametrize( + ("indexed_path", "error_type", "error_message"), + [ + ("/outside", ValueError, "stay within configured tools directory"), + ("../outside", ValueError, "stay within configured tools directory"), + ("tools/../tools/calculator", ValueError, "stay within configured tools directory"), + ("tools/missing", NotADirectoryError, "not a directory"), + ("tools/not-a-tool.txt", NotADirectoryError, "not a directory"), + ], +) +def test_get_indexed_forge_tools_rejects_unsafe_tool_paths(tmp_path, indexed_path, error_type, error_message): + tools_path = tmp_path / "atomic-forge" / "tools" + (tools_path / "calculator").mkdir(parents=True) + (tools_path / "not-a-tool.txt").write_text("not a tool") + index_path = tools_path.parent / "index.json" + index_path.write_text( + json.dumps( + { + "schema": 1, + "name": "test", + "tools": [{"name": "calculator", "path": indexed_path}], + } + ) + ) + + with pytest.raises(error_type, match=error_message): + AtomicToolManager.get_indexed_forge_tools(tmp_path, tools_path) + + +def test_list_tools_reads_each_source_and_labels_tools(tmp_path, capsys): + official = create_git_forge(tmp_path, "official", "calculator") + team = create_git_forge(tmp_path, "team", "internal-search") + + assert assembler_main.list_tools([official, team]) == 0 + + assert capsys.readouterr().out.splitlines() == [ + "official/calculator - calculator for tests", + "team/internal-search - internal-search for tests", + ] + + +def test_list_index_metadata_cannot_emit_terminal_sequences(tmp_path, capsys): + source = create_git_forge( + tmp_path, + "official", + "calculator", + index_name="\x1b]52;c;clipboard\x07calculator\x1b[31m", + description="\x1b[31mred\x1b[0m\nsecond line", + ) + + assert assembler_main.list_tools([source]) == 0 + + output = capsys.readouterr().out + assert output == "official/calculator - red second line\n" + assert "\x1b" not in output + assert "\x07" not in output + + +def test_download_error_index_name_cannot_emit_terminal_sequences(tmp_path, capsys): + source = create_git_forge( + tmp_path, + "official", + "calculator", + index_name="\x1b]52;c;clipboard\x07calculator\x1b[31m", + ) + + assert assembler_main.download_tool("missing", str(tmp_path / "downloaded"), [source]) == 1 + + output = capsys.readouterr().err + assert "official/calculator" in output + assert "\x1b" not in output + assert "\x07" not in output + + +def test_download_tool_from_second_source(tmp_path, capsys): + official = create_git_forge(tmp_path, "official", "calculator") + team = create_git_forge(tmp_path, "team", "internal-search") + destination = tmp_path / "downloaded" + + assert assembler_main.download_tool("internal-search", str(destination), [official, team]) == 0 + + assert (destination / "tool" / "internal-search.py").is_file() + assert (destination / "pyproject.toml").is_file() + assert (destination / "requirements.txt").is_file() + assert "Downloaded team/internal-search" in capsys.readouterr().out + + +def test_download_tool_defaults_to_tool_directory_in_current_working_directory(tmp_path, monkeypatch, capsys): + source = create_git_forge(tmp_path, "official", "calculator") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(assembler_main, "load_sources", lambda: [source]) + monkeypatch.setattr("sys.argv", ["atomic", "download", "calculator"]) + + assert assembler_main.main() == 0 + + destination = tmp_path / "calculator" + assert (destination / "tool" / "calculator.py").is_file() + assert (destination / "pyproject.toml").is_file() + assert (destination / "requirements.txt").is_file() + assert "Downloaded official/calculator" in capsys.readouterr().out + + +def test_download_default_destination_and_success_output_hide_tool_path_controls(tmp_path, monkeypatch, capsys): + control_sequence = "\N{RIGHT-TO-LEFT OVERRIDE}" if os.name == "nt" else "\x1b]52;c;clipboard\x07" + tool_directory_name = f"calculator{control_sequence}clipboard" + source = create_git_forge(tmp_path, "official", tool_directory_name, index_name="calculator") + monkeypatch.chdir(tmp_path) + + assert assembler_main.download_tool("calculator", None, [source]) == 0 + + assert (tmp_path / tool_directory_name / "tool" / f"{tool_directory_name}.py").is_file() + output = capsys.readouterr().out + assert "Downloaded official/calculator" in output + assert control_sequence not in output + assert "\x1b" not in output + assert "\x07" not in output + + +@pytest.mark.parametrize("control", ["\x1b]52;c;clipboard\x07", "\x1b[31m", "\N{RIGHT-TO-LEFT OVERRIDE}"]) +def test_tool_source_qualification_cannot_emit_terminal_controls(tmp_path, capsys, control): + source = create_git_forge(tmp_path, "team", "calculator") + object.__setattr__(source, "name", f"team{control}") + + assert assembler_main.list_tools([source]) == 0 + + list_output = capsys.readouterr().out + assert control not in list_output + assert "\x1b" not in list_output + assert "\x07" not in list_output + assert "\N{RIGHT-TO-LEFT OVERRIDE}" not in list_output + + assert assembler_main.download_tool("calculator", str(tmp_path / "downloaded"), [source]) == 0 + + download_output = capsys.readouterr().out + assert control not in download_output + assert "\x1b" not in download_output + assert "\x07" not in download_output + assert "\N{RIGHT-TO-LEFT OVERRIDE}" not in download_output + + +def test_download_tool_requires_source_qualification_for_ambiguous_names(tmp_path, capsys): + official = create_git_forge(tmp_path, "official", "calculator") + team = create_git_forge(tmp_path, "team", "calculator") + + assert assembler_main.download_tool("calculator", str(tmp_path / "downloaded"), [official, team]) == 1 + + assert "Use /" in capsys.readouterr().err + + +def test_list_tools_returns_success_with_partial_source_failure(tmp_path, capsys): + official = create_git_forge(tmp_path, "official", "calculator") + missing = ForgeSource("missing", (tmp_path / "missing").as_uri(), "main", "atomic-forge/tools") + + assert assembler_main.list_tools([official, missing]) == 0 + + captured = capsys.readouterr() + assert "official/calculator" in captured.out + assert "Could not read source 'missing'" in captured.err + + +def test_list_tools_returns_failure_when_all_sources_fail(tmp_path, capsys): + first_missing = ForgeSource("first", (tmp_path / "first").as_uri(), "main", "atomic-forge/tools") + second_missing = ForgeSource("second", (tmp_path / "second").as_uri(), "main", "atomic-forge/tools") + + assert assembler_main.list_tools([first_missing, second_missing]) == 1 + + captured = capsys.readouterr() + assert captured.out == "" + assert "Could not read source 'first'" in captured.err + assert "Could not read source 'second'" in captured.err + + +def create_git_forge(tmp_path, name, tool_name, index_name=None, description=None): + forge_root = tmp_path / name + tool_path = forge_root / "atomic-forge" / "tools" / tool_name + (tool_path / "tool").mkdir(parents=True) + for filename in ("README.md", "pyproject.toml", "requirements.txt", "uv.lock", ".coveragerc"): + (tool_path / filename).write_text(filename) + (tool_path / "tool" / f"{tool_name}.py").write_text("") + index_path = forge_root / "atomic-forge" / "index.json" + index_path.write_text( + json.dumps( + { + "schema": 1, + "name": name, + "tools": [ + { + "name": index_name or tool_name, + "path": f"tools/{tool_name}", + "description": description or f"{tool_name} for tests", + } + ], + } + ) + ) + + repository = git.Repo.init(forge_root) + with repository.config_writer() as config: + config.set_value("user", "name", "Atomic Assembler Tests") + config.set_value("user", "email", "tests@example.com") + repository.index.add(["atomic-forge"]) + repository.index.commit("Add test forge") + return ForgeSource(name, forge_root.as_uri(), repository.active_branch.name, "atomic-forge/tools") diff --git a/atomic-forge/atomic_tools/tavily_search/pyproject.toml b/atomic-forge/atomic_tools/tavily_search/pyproject.toml deleted file mode 100644 index 379a5993..00000000 --- a/atomic-forge/atomic_tools/tavily_search/pyproject.toml +++ /dev/null @@ -1,16 +0,0 @@ -[project] -name = "atomic-tools-tavily-search" -version = "0.1.0" -description = "Tavily search tool for atomic-agents" -requires-python = ">=3.10" -dependencies = ["aiohttp", "pydantic"] - -[project.optional-dependencies] -dev = ["pytest", "pytest-asyncio"] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.pytest.ini_options] -asyncio_mode = "auto" diff --git a/atomic-forge/atomic_tools/tavily_search/requirements.txt b/atomic-forge/atomic_tools/tavily_search/requirements.txt deleted file mode 100644 index 74408d90..00000000 --- a/atomic-forge/atomic_tools/tavily_search/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -aiohttp>=3.9.0 -pydantic>=2.0.0 diff --git a/atomic-forge/atomic_tools/tavily_search/tests/test_tavily_search.py b/atomic-forge/atomic_tools/tavily_search/tests/test_tavily_search.py deleted file mode 100644 index 6dddb77b..00000000 --- a/atomic-forge/atomic_tools/tavily_search/tests/test_tavily_search.py +++ /dev/null @@ -1,136 +0,0 @@ -from unittest.mock import AsyncMock -import pytest -from tavily_search import ( - TavilySearchTool, - TavilySearchToolConfig, - TavilySearchToolInputSchema, -) - - -@pytest.fixture -def mock_config(): - return TavilySearchToolConfig(api_key="test_api_key", max_results=5) - - -@pytest.fixture -def tool(mock_config): - return TavilySearchTool(config=mock_config) - - -class TestTavilySearchTool: - @pytest.mark.asyncio - async def test_fetch_search_results_respects_max_results_override(self, tool): - """Regression test: max_results override must propagate to the API call, not be ignored.""" - mock_session = AsyncMock() - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.json = AsyncMock( - return_value={ - "results": [ - { - "title": f"Result {i}", - "url": f"http://example.com/{i}", - "content": "...", - "score": 1.0 - i * 0.1, - } - for i in range(5) - ], - "answer": "test answer", - } - ) - mock_response.reason = "OK" - mock_session.post = AsyncMock(return_value=mock_response) - mock_session.__aenter__ = AsyncMock(return_value=mock_session) - mock_session.__aexit__ = AsyncMock(return_value=None) - - await tool._fetch_search_results(mock_session, "test query", max_results=2) - - # The key assertion: the API must have been called with max_results=2, not self.max_results=5 - call_args = mock_session.post.call_args - json_data = call_args.kwargs["json"] - assert json_data["max_results"] == 2, "max_results override was ignored in API call" - - @pytest.mark.asyncio - async def test_run_async_respects_max_results_override(self, tool): - """Runtime max_results override must reach the API and limit result count.""" - mock_session = AsyncMock() - - async def mock_post(*args, **kwargs): - max_override = kwargs["json"]["max_results"] - return AsyncMock( - status=200, - reason="OK", - json=AsyncMock( - return_value={ - "results": [ - { - "title": f"Result {i}", - "url": f"http://example.com/{i}", - "content": "...", - "score": 1.0, - } - for i in range(max_override + 3) # Return MORE than requested - ], - "answer": "test answer", - } - ), - ) - - mock_session.post = mock_post - mock_session.__aenter__ = AsyncMock(return_value=mock_session) - mock_session.__aexit__ = AsyncMock(return_value=None) - - params = TavilySearchToolInputSchema(queries=["test query"]) - output = await tool.run_async(params, max_results=2) - - # Should have exactly 2 results since we requested 2, even though API returned 5 - assert len(output.results) == 2 - for call in mock_session.post.call_args_list: - assert call.kwargs["json"]["max_results"] == 2 - - @pytest.mark.asyncio - async def test_run_async_uses_default_when_no_override(self, tool): - """When no override is passed, self.max_results should be used.""" - mock_session = AsyncMock() - - async def mock_post(*args, **kwargs): - return AsyncMock( - status=200, - reason="OK", - json=AsyncMock( - return_value={ - "results": [ - { - "title": f"Result {i}", - "url": f"http://example.com/{i}", - "content": "...", - "score": 1.0, - } - for i in range(5) - ], - "answer": "test answer", - } - ), - ) - - mock_session.post = mock_post - mock_session.__aenter__ = AsyncMock(return_value=mock_session) - mock_session.__aexit__ = AsyncMock(return_value=None) - - params = TavilySearchToolInputSchema(queries=["test query"]) - await tool.run_async(params) # No max_results override - - for call in mock_session.post.call_args_list: - assert call.kwargs["json"]["max_results"] == 5 # Should use config default (5) - - def test_init_defaults(self): - """Default config should set max_results to 5.""" - tool = TavilySearchTool() - assert tool.max_results == 5 - assert tool.search_depth == "basic" - - def test_config_respects_custom_max_results(self): - """Config should accept and store custom max_results.""" - config = TavilySearchToolConfig(max_results=10) - tool = TavilySearchTool(config=config) - assert tool.max_results == 10 diff --git a/atomic-forge/atomic_tools/tavily_search/tool/__init__.py b/atomic-forge/atomic_tools/tavily_search/tool/__init__.py deleted file mode 100644 index 2742ab73..00000000 --- a/atomic-forge/atomic_tools/tavily_search/tool/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from tavily_search import ( - TavilySearchTool, - TavilySearchToolConfig, - TavilySearchToolInputSchema, - TavilySearchToolOutputSchema, -) - -__all__ = [ - "TavilySearchTool", - "TavilySearchToolConfig", - "TavilySearchToolInputSchema", - "TavilySearchToolOutputSchema", -] diff --git a/atomic-forge/atomic_tools/tavily_search/tool/tavily_search.py b/atomic-forge/atomic_tools/tavily_search/tool/tavily_search.py deleted file mode 100644 index ad00fdf5..00000000 --- a/atomic-forge/atomic_tools/tavily_search/tool/tavily_search.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -from typing import List, Literal, Optional -import asyncio -from concurrent.futures import ThreadPoolExecutor - -import aiohttp -from pydantic import Field - -from atomic_agents import BaseIOSchema, BaseTool, BaseToolConfig - - -################ -# INPUT SCHEMA # -################ -class TavilySearchToolInputSchema(BaseIOSchema): - """ - Schema for input to a tool for searching for information, news, references, and other content using Tavily. - Returns a list of search results with a short description or content snippet and URLs for further exploration - """ - - queries: List[str] = Field(..., description="List of search queries.") - - -#################### -# OUTPUT SCHEMA(S) # -#################### -class TavilySearchResultItemSchema(BaseIOSchema): - """This schema represents a single search result item""" - - title: str = Field(..., description="The title of the search result") - url: str = Field(..., description="The URL of the search result") - content: str = Field(None, description="The content snippet of the search result") - score: float = Field(..., description="The score of the search result") - raw_content: Optional[str] = Field(None, description="The raw content of the search result") - query: Optional[str] = Field(..., description="The query used to obtain this search result") - answer: Optional[str] = Field(..., description="The answer to the query provided by Tavily") - - -class TavilySearchToolOutputSchema(BaseIOSchema): - """This schema represents the output of the Tavily search tool.""" - - results: List[TavilySearchResultItemSchema] = Field(..., description="List of search result items") - - -############## -# TOOL LOGIC # -############## -class TavilySearchToolConfig(BaseToolConfig): - api_key: str = "" - max_results: int = 5 - search_depth: Literal["basic", "advanced"] = "basic" - include_domains: Optional[List[str]] = None - exclude_domains: Optional[List[str]] = None - - -class TavilySearchTool(BaseTool[TavilySearchToolInputSchema, TavilySearchToolOutputSchema]): - """ - Tool for performing searches using the Tavily search API. - - Attributes: - input_schema (TavilySearchToolInputSchema): The schema for the input data. - output_schema (TavilySearchToolOutputSchema): The schema for the output data. - max_results (int): The maximum number of search results to return. - api_key (str): The API key for the Tavily API. - """ - - def __init__(self, config: TavilySearchToolConfig = TavilySearchToolConfig()): - """ - Initializes the TavilySearchTool. - - Args: - config (TavilySearchToolConfig): - Configuration for the tool, including API key, max results, and optional title and description overrides. - """ - super().__init__(config) - self.api_key = config.api_key or os.getenv("TAVILY_API_KEY", "") - self.max_results = config.max_results - self.search_depth = config.search_depth - self.include_domains = config.include_domains - self.exclude_domains = config.exclude_domains - self.include_answer = False - - async def _fetch_search_results(self, session: aiohttp.ClientSession, query: str, max_results: int) -> dict: - headers = { - "accept": "/", - "content-type": "application/json", - "origin": "https://app.tavily.com/", - "referer": "https://app.tavily.com/", - "user-agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/131.0.0.0 Safari/537.36" - ), - } - - json_data = { - "query": query, - "api_key": self.api_key, - "search_depth": self.search_depth, - "include_domains": self.include_domains, - "exclude_domains": self.exclude_domains, - "max_results": max_results, - "include_answer": self.include_answer, - } - - async with session.post("https://api.tavily.com/search", headers=headers, json=json_data) as response: - if response.status != 200: - error_message = await response.text() - raise Exception( - f"Failed to fetch search results for query '{query}': " - f"{response.status} {response.reason}. " - f"Details: {error_message}" - ) - data = await response.json() - - results = data.get("results", []) - answer = data.get("answer", "") - - for result in results: - result["query"] = query - - return {"results": results, "answer": answer} - - async def run_async( - self, - params: TavilySearchToolInputSchema, - max_results: Optional[int] = None, - ) -> TavilySearchToolOutputSchema: - effective_max_results = self.max_results if max_results is None else max_results - - async with aiohttp.ClientSession() as session: - tasks = [self._fetch_search_results(session, query, effective_max_results) for query in params.queries] - raw_responses = await asyncio.gather(*tasks) - - processed_results = [] - for response in raw_responses: - query_results = response["results"] - answer = response["answer"] - - query_processed = [] - for result in query_results: - if all(key in result for key in ["title", "url", "content", "score"]): - query_processed.append( - TavilySearchResultItemSchema( - title=result["title"], - url=result["url"], - content=result.get("content", ""), - score=result.get("score", 0), - raw_content=result.get("raw_content"), - query=result.get("query"), - answer=answer, - ) - ) - else: - print(f"Skipping result due to missing keys: {result}") - - query_processed = query_processed[:effective_max_results] - processed_results.extend(query_processed) - - return TavilySearchToolOutputSchema(results=processed_results) - - def run( - self, - params: TavilySearchToolInputSchema, - max_results: Optional[int] = None, - ) -> TavilySearchToolOutputSchema: - """ - Runs the TavilyTool synchronously with the given parameters. - - Args: - params (TavilySearchToolInputSchema): The input parameters for the tool, adhering to the input schema. - max_results (Optional[int]): The maximum number of search results to return. - - Returns: - TavilySearchToolOutputSchema: The output of the tool, adhering to the output schema. - """ - with ThreadPoolExecutor() as executor: - result = executor.submit( - asyncio.run, - self.run_async(params, max_results), - ).result() - - return result diff --git a/atomic-forge/conformance/test_conformance.py b/atomic-forge/conformance/test_conformance.py new file mode 100644 index 00000000..3dfa26f9 --- /dev/null +++ b/atomic-forge/conformance/test_conformance.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import importlib.util +import inspect +import json +import re +import warnings +from functools import lru_cache +from pathlib import Path +from types import ModuleType +from typing import Any, get_args, get_origin + +import pytest + +from atomic_agents import BaseIOSchema, BaseTool, BaseToolConfig + + +TOOLS_DIR = Path(__file__).parents[1] / "tools" +INDEX_PATH = Path(__file__).parents[1] / "index.json" +REQUIRED_FILES = ("README.md", "pyproject.toml", "requirements.txt") + +# TODO: Remove these exceptions after the stale requirements files are corrected. +KNOWN_DEPENDENCY_MIGRATION = { + "searxng_search": ({"sympy"}, {"aiohttp"}), + "tavily_search": ({"sympy"}, set()), +} + + +def tool_directories() -> list[Path]: + return sorted(path for path in TOOLS_DIR.iterdir() if path.is_dir()) + + +TOOLS = tool_directories() + + +def tool_id(path: Path) -> str: + return path.name + + +def project_metadata(tool_dir: Path) -> dict[str, Any]: + import tomllib + + return tomllib.loads(tool_dir.joinpath("pyproject.toml").read_text(encoding="utf-8"))["project"] + + +def requirement_name(entry: str) -> str: + entry = entry.split("#", 1)[0].strip() + match = re.match(r"(?:-e\s+)?([A-Za-z0-9][A-Za-z0-9_.-]*)", entry) + if match is None: + raise ValueError(f"Unsupported requirements.txt entry: {entry!r}") + return normalize_name(match.group(1)) + + +def normalize_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def runtime_dependency_names(tool_dir: Path) -> set[str]: + return {normalize_name(requirement_name(entry)) for entry in project_metadata(tool_dir)["dependencies"]} + + +def requirements_names(tool_dir: Path) -> set[str]: + entries = tool_dir.joinpath("requirements.txt").read_text(encoding="utf-8").splitlines() + return {requirement_name(entry) for entry in entries if entry.strip() and not entry.lstrip().startswith("#")} + + +@lru_cache(maxsize=None) +def load_tool_module(tool_name: str) -> ModuleType: + tool_dir = TOOLS_DIR / tool_name + module_path = tool_dir / "tool" / f"{tool_name}.py" + if not module_path.is_file(): + raise AssertionError(f"{tool_name}: expected tool module at {module_path}") + + module_name = f"atomic_forge_conformance_{tool_name}" + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None or spec.loader is None: + raise AssertionError(f"{tool_name}: could not create an import spec for {module_path}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def local_classes(module: ModuleType, base_class: type) -> list[type]: + return [ + value + for value in vars(module).values() + if inspect.isclass(value) + and value.__module__ == module.__name__ + and value is not base_class + and issubclass(value, base_class) + ] + + +def main_tool_class(module: ModuleType, tool_name: str) -> type: + classes = local_classes(module, BaseTool) + assert len(classes) == 1, f"{tool_name}: expected one local BaseTool subclass, found {classes}" + return classes[0] + + +def tool_generic_parameters(tool_class: type, tool_name: str) -> tuple[type, type]: + generic_bases = [base for base in getattr(tool_class, "__orig_bases__", ()) if get_origin(base) is BaseTool] + assert ( + len(generic_bases) == 1 + ), f"{tool_name}: {tool_class.__name__} must declare BaseTool[InputSchema, OutputSchema] explicitly" + args = get_args(generic_bases[0]) + assert len(args) == 2, f"{tool_name}: {tool_class.__name__} must declare two BaseTool generic parameters" + return args[0], args[1] + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_required_files(tool_dir: Path) -> None: + missing = [name for name in REQUIRED_FILES if not tool_dir.joinpath(name).is_file()] + missing.extend(name for name in ("tool", "tests") if not tool_dir.joinpath(name).is_dir()) + assert not missing, f"{tool_id(tool_dir)}: missing required paths: {', '.join(missing)}" + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_optional_files_warn(tool_dir: Path) -> None: + missing = [name for name in ("uv.lock", ".coveragerc") if not tool_dir.joinpath(name).is_file()] + for name in missing: + warnings.warn(f"{tool_id(tool_dir)}: optional file is missing: {name}", UserWarning, stacklevel=1) + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_module_imports_cleanly(tool_dir: Path) -> None: + try: + load_tool_module(tool_id(tool_dir)) + except Exception as exc: + pytest.fail(f"{tool_id(tool_dir)}: tool module failed to import cleanly: {exc}") + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_schemas_follow_contract(tool_dir: Path) -> None: + name = tool_id(tool_dir) + module = load_tool_module(name) + tool_class = main_tool_class(module, name) + input_schema, output_schema = tool_generic_parameters(tool_class, name) + + for label, schema in (("input", input_schema), ("output", output_schema)): + assert inspect.isclass(schema) and issubclass( + schema, BaseIOSchema + ), f"{name}: {label} schema must subclass BaseIOSchema, got {schema!r}" + assert inspect.getdoc(schema), f"{name}: {label} schema must have a non-empty docstring" + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_config_follows_contract(tool_dir: Path) -> None: + name = tool_id(tool_dir) + module = load_tool_module(name) + configs = local_classes(module, BaseToolConfig) + assert len(configs) == 1, f"{name}: expected one local BaseToolConfig subclass, found {configs}" + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_main_tool_declares_explicit_generics(tool_dir: Path) -> None: + name = tool_id(tool_dir) + module = load_tool_module(name) + tool_class = main_tool_class(module, name) + tool_generic_parameters(tool_class, name) + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_requirements_match_pyproject(tool_dir: Path) -> None: + name = tool_id(tool_dir) + requirements = requirements_names(tool_dir) + dependencies = runtime_dependency_names(tool_dir) + extra_requirements = requirements - dependencies + missing_requirements = dependencies - requirements + + if extra_requirements or missing_requirements: + expected = KNOWN_DEPENDENCY_MIGRATION.get(name) + actual = (extra_requirements, missing_requirements) + if expected == actual: + warnings.warn( + f"{name}: dependency declaration mismatch is a known migration warning; " + f"requirements-only={sorted(extra_requirements)}, pyproject-only={sorted(missing_requirements)}", + UserWarning, + stacklevel=1, + ) + return + pytest.fail( + f"{name}: requirements.txt and pyproject.toml dependencies differ by name; " + f"requirements-only={sorted(extra_requirements)}, pyproject-only={sorted(missing_requirements)}" + ) + + +@pytest.mark.parametrize("tool_dir", TOOLS, ids=tool_id) +def test_index_entry_matches_project(tool_dir: Path) -> None: + name = tool_id(tool_dir) + metadata = project_metadata(tool_dir) + index = json.loads(INDEX_PATH.read_text(encoding="utf-8")) + matches = [entry for entry in index["tools"] if entry.get("path") == f"tools/{name}"] + assert len(matches) == 1, f"{name}: expected one index entry at tools/{name}, found {len(matches)}" + + entry = matches[0] + assert ( + entry.get("name") == metadata["name"] + ), f"{name}: index name {entry.get('name')!r} does not match pyproject name {metadata['name']!r}" + assert ( + entry.get("version") == metadata["version"] + ), f"{name}: index version {entry.get('version')!r} does not match pyproject version {metadata['version']!r}" diff --git a/atomic-forge/guides/tool_structure.md b/atomic-forge/guides/tool_structure.md index 5294e15d..4aa9f161 100644 --- a/atomic-forge/guides/tool_structure.md +++ b/atomic-forge/guides/tool_structure.md @@ -6,7 +6,7 @@ This guide aims to explain the anatomy of an **Atomic Tool** within the **Atomic ## Prerequisites -Set up the main Atomic Agents library for development by following the instructions in the [development setup guide](/guides/dev-setup.md). +Set up the main Atomic Agents library for development by following the instructions in the [development setup guide](../../guides/DEV_GUIDE.md). ## The Principles diff --git a/atomic-forge/index.json b/atomic-forge/index.json new file mode 100644 index 00000000..1fed9c18 --- /dev/null +++ b/atomic-forge/index.json @@ -0,0 +1,156 @@ +{ + "schema": 1, + "name": "atomic-forge", + "tools": [ + { + "name": "arxiv-search", + "path": "tools/arxiv_search", + "description": "arXiv search tool for Atomic Agents (free public API)", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "aiohttp>=3.9.0,<4.0.0" + ] + }, + { + "name": "bocha-search", + "path": "tools/bocha_search", + "description": "A tool for performing searches using BoCha", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "aiohttp>=3.9.0,<4.0.0" + ] + }, + { + "name": "calculator", + "path": "tools/calculator", + "description": "Calculator tool for Atomic Agents", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "sympy>=1.12,<2.0.0" + ] + }, + { + "name": "datetime-tool", + "path": "tools/datetime_tool", + "description": "Date and time operations (now, parse, convert, shift, diff) for Atomic Agents", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "tzdata>=2024.1" + ] + }, + { + "name": "fia-signals", + "path": "tools/fia_signals", + "description": "F\u00eda Signals crypto intelligence tool for Atomic Agents \u2014 market regime, trading signals, DeFi yields, wallet risk scoring", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "requests>=2.28.0,<3.0.0" + ] + }, + { + "name": "hackernews-search", + "path": "tools/hackernews_search", + "description": "Hacker News search tool for Atomic Agents (free Algolia API)", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "aiohttp>=3.9.0,<4.0.0" + ] + }, + { + "name": "pdf-reader", + "path": "tools/pdf_reader", + "description": "PDF reader tool for Atomic Agents (local files or URLs, page-range support)", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "pypdf>=5.1.0,<7.0.0", + "requests>=2.32.3,<3.0.0" + ] + }, + { + "name": "searxng-search", + "path": "tools/searxng_search", + "description": "A tool for performing searches using SearxNG", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "aiohttp>=3.9.0,<4.0.0" + ] + }, + { + "name": "tavily-search", + "path": "tools/tavily_search", + "description": "A tool for performing searches using Tavily", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "aiohttp>=3.9.0,<4.0.0" + ] + }, + { + "name": "weather-tool", + "path": "tools/weather", + "description": "Weather tool for Atomic Agents (Open-Meteo, no API key required)", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "requests>=2.32.3,<3.0.0" + ] + }, + { + "name": "webpage-scraper", + "path": "tools/webpage_scraper", + "description": "Tool for scraping webpage content and converting it to markdown format", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "beautifulsoup4>=4.12.0,<5.0.0", + "markdownify>=0.11.0,<1.0.0", + "readability-lxml>=0.8.1,<1.0.0", + "requests>=2.31.0,<3.0.0", + "lxml>=5.1.0,<6.0.0", + "lxml-html-clean>=0.3.1,<1.0.0" + ] + }, + { + "name": "wikipedia-search", + "path": "tools/wikipedia_search", + "description": "Wikipedia search tool for Atomic Agents (free, no API key required)", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "aiohttp>=3.9.0,<4.0.0" + ] + }, + { + "name": "youtube-transcript-scraper", + "path": "tools/youtube_transcript_scraper", + "description": "A tool for fetching and processing YouTube video transcripts", + "version": "1.0.0", + "dependencies": [ + "atomic-agents", + "pydantic>=2.10.3,<3.0.0", + "google-api-python-client>=2.118.0,<3.0.0", + "youtube-transcript-api>=1.1.1,<2.0.0" + ] + } + ] +} diff --git a/atomic-forge/scripts/generate_index.py b/atomic-forge/scripts/generate_index.py new file mode 100644 index 00000000..809b8094 --- /dev/null +++ b/atomic-forge/scripts/generate_index.py @@ -0,0 +1,57 @@ +import json +import tomllib +from pathlib import Path + +FORGE_ROOT = Path(__file__).resolve().parents[1] +TOOLS_ROOT = FORGE_ROOT / "tools" +INDEX_PATH = FORGE_ROOT / "index.json" + + +def first_readme_paragraph(readme_path: Path) -> str: + paragraphs: list[str] = [] + lines: list[str] = [] + + for line in readme_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + lines.append(line.strip()) + elif lines: + paragraphs.append(" ".join(lines)) + lines = [] + + if lines: + paragraphs.append(" ".join(lines)) + + return paragraphs[0] if paragraphs else "" + + +def tool_record(tool_path: Path) -> dict[str, object]: + with (tool_path / "pyproject.toml").open("rb") as pyproject_file: + project = tomllib.load(pyproject_file)["project"] + + description = project.get("description", "").strip() + if not description: + description = first_readme_paragraph(tool_path / "README.md") + + return { + "name": project["name"], + "path": tool_path.relative_to(FORGE_ROOT).as_posix(), + "description": description, + "version": project["version"], + "dependencies": project.get("dependencies", []), + } + + +def build_index() -> dict[str, object]: + tools = sorted( + (tool_record(tool_path) for tool_path in TOOLS_ROOT.iterdir() if (tool_path / "pyproject.toml").is_file()), + key=lambda tool: str(tool["name"]), + ) + return {"schema": 1, "name": "atomic-forge", "tools": tools} + + +def main() -> None: + INDEX_PATH.write_text(json.dumps(build_index(), indent=2) + "\n", encoding="utf-8", newline="\n") + + +if __name__ == "__main__": + main() diff --git a/atomic-forge/tools/bocha_search/tests/test_bocha_search.py b/atomic-forge/tools/bocha_search/tests/test_bocha_search.py index ded74052..8848e039 100644 --- a/atomic-forge/tools/bocha_search/tests/test_bocha_search.py +++ b/atomic-forge/tools/bocha_search/tests/test_bocha_search.py @@ -197,7 +197,7 @@ async def test_bocha_search_tool_concurrent_queries(mock_aiohttp_session): @pytest.mark.asyncio async def test_bocha_search_tool_config_params_real_case_cn(mock_aiohttp_session): """Test that config params freshness/include/exclude are included in request payload with Chinese websites""" - mock_api_key = os.getenv("BOCHA_API_KEY") + mock_api_key = "KEY" # Tool configuration with domestic websites tool = BoChaSearchTool( diff --git a/atomic-forge/tools/searxng_search/requirements.txt b/atomic-forge/tools/searxng_search/requirements.txt index 06f7f357..03fbc081 100644 --- a/atomic-forge/tools/searxng_search/requirements.txt +++ b/atomic-forge/tools/searxng_search/requirements.txt @@ -1,3 +1,3 @@ atomic-agents>=1.0.0,<2.0.0 pydantic>=2.8.2,<3.0.0 -sympy>=1.12,<2.0.0 +aiohttp>=3.9.0,<4.0.0 diff --git a/atomic-forge/tools/tavily_search/tests/test_tavily_seach.py b/atomic-forge/tools/tavily_search/tests/test_tavily_search.py similarity index 100% rename from atomic-forge/tools/tavily_search/tests/test_tavily_seach.py rename to atomic-forge/tools/tavily_search/tests/test_tavily_search.py diff --git a/atomic-forge/tools/webpage_scraper/.coveragerc b/atomic-forge/tools/webpage_scraper/.coveragerc new file mode 100644 index 00000000..c45136ef --- /dev/null +++ b/atomic-forge/tools/webpage_scraper/.coveragerc @@ -0,0 +1,8 @@ +[run] +source = tool +omit = */tests/* + +[report] +exclude_lines = + if __name__ == "__main__": +show_missing = True diff --git a/atomic-forge/tools/youtube_transcript_scraper/tests/test_youtube_transcript_scraper.py b/atomic-forge/tools/youtube_transcript_scraper/tests/test_youtube_transcript_scraper.py index 0259a07b..ff513048 100644 --- a/atomic-forge/tools/youtube_transcript_scraper/tests/test_youtube_transcript_scraper.py +++ b/atomic-forge/tools/youtube_transcript_scraper/tests/test_youtube_transcript_scraper.py @@ -1,30 +1,63 @@ import os import sys -import pytest -from unittest.mock import patch, MagicMock +from contextlib import contextmanager from datetime import datetime +from unittest.mock import MagicMock, patch -from youtube_transcript_api import NoTranscriptFound, TranscriptsDisabled +import pytest +from youtube_transcript_api import ( + FetchedTranscript, + FetchedTranscriptSnippet, + NoTranscriptFound, + TranscriptsDisabled, + YouTubeTranscriptApi, +) # Add the parent directory of 'tests' to the Python path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from tool.youtube_transcript_scraper import ( # noqa: E402 YouTubeTranscriptTool, + YouTubeTranscriptToolConfig, YouTubeTranscriptToolInputSchema, YouTubeTranscriptToolOutputSchema, - YouTubeTranscriptToolConfig, ) +def make_fetched_transcript(): + return FetchedTranscript( + snippets=[ + FetchedTranscriptSnippet(text="Never gonna give you up", start=0.0, duration=5.0), + FetchedTranscriptSnippet(text="Never gonna let you down", start=5.0, duration=5.0), + ], + video_id="dQw4w9WgXcQ", + language="English", + language_code="en", + is_generated=False, + ) + + +@contextmanager +def mock_transcript_api(fetched_transcript=None, side_effect=None): + api = MagicMock(spec=YouTubeTranscriptApi) + api.fetch = MagicMock(return_value=fetched_transcript, side_effect=side_effect) + + def fetch_raw_transcript(video_id, languages=None): + if languages is None: + fetched = api.fetch(video_id) + else: + fetched = api.fetch(video_id, languages=languages) + return fetched.to_raw_data() + + api.get_transcript = fetch_raw_transcript + with patch("tool.youtube_transcript_scraper.YouTubeTranscriptApi", api): + yield api + + def test_youtube_transcript_tool(): mock_api_key = "mock_api_key" mock_video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" mock_video_id = "dQw4w9WgXcQ" - mock_transcript = [ - {"text": "Never gonna give you up", "duration": 5.0}, - {"text": "Never gonna let you down", "duration": 5.0}, - ] mock_metadata = { "title": "Rick Astley - Never Gonna Give You Up", "channelTitle": "Rick Astley", @@ -32,10 +65,7 @@ def test_youtube_transcript_tool(): } with ( - patch( - "tool.youtube_transcript_scraper.YouTubeTranscriptApi.get_transcript", - return_value=mock_transcript, - ), + mock_transcript_api(make_fetched_transcript()) as mock_transcript_api_instance, patch("tool.youtube_transcript_scraper.build") as mock_build, ): mock_youtube = MagicMock() @@ -56,15 +86,13 @@ def test_youtube_transcript_tool(): "channel": mock_metadata["channelTitle"], "published_at": datetime.fromisoformat(mock_metadata["publishedAt"].replace("Z", "")), } + mock_transcript_api_instance.fetch.assert_called_once_with(mock_video_id) def test_youtube_transcript_tool_with_language(): mock_api_key = "mock_api_key" mock_video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - mock_transcript = [ - {"text": "Never gonna give you up", "duration": 5.0}, - {"text": "Never gonna let you down", "duration": 5.0}, - ] + mock_video_id = "dQw4w9WgXcQ" mock_metadata = { "title": "Rick Astley - Never Gonna Give You Up", "channelTitle": "Rick Astley", @@ -72,10 +100,7 @@ def test_youtube_transcript_tool_with_language(): } with ( - patch( - "tool.youtube_transcript_scraper.YouTubeTranscriptApi.get_transcript", - return_value=mock_transcript, - ), + mock_transcript_api(make_fetched_transcript()) as mock_transcript_api_instance, patch("tool.youtube_transcript_scraper.build") as mock_build, ): mock_youtube = MagicMock() @@ -88,19 +113,19 @@ def test_youtube_transcript_tool_with_language(): assert isinstance(result, YouTubeTranscriptToolOutputSchema) assert result.transcript == "Never gonna give you up Never gonna let you down" + mock_transcript_api_instance.fetch.assert_called_once_with(mock_video_id, languages=["en"]) def test_youtube_transcript_tool_no_transcript(): mock_api_key = "mock_api_key" mock_video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - with patch( - "tool.youtube_transcript_scraper.YouTubeTranscriptApi.get_transcript", + with mock_transcript_api( side_effect=NoTranscriptFound( video_id="dQw4w9WgXcQ", requested_language_codes=["en"], transcript_data=None, - ), + ) ): youtube_transcript_tool = YouTubeTranscriptTool(YouTubeTranscriptToolConfig(api_key=mock_api_key)) input_schema = YouTubeTranscriptToolInputSchema(video_url=mock_video_url) @@ -115,10 +140,7 @@ def test_youtube_transcript_tool_transcripts_disabled(): mock_api_key = "mock_api_key" mock_video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - with patch( - "tool.youtube_transcript_scraper.YouTubeTranscriptApi.get_transcript", - side_effect=TranscriptsDisabled("Transcripts are disabled"), - ): + with mock_transcript_api(side_effect=TranscriptsDisabled("Transcripts are disabled")): youtube_transcript_tool = YouTubeTranscriptTool(YouTubeTranscriptToolConfig(api_key=mock_api_key)) input_schema = YouTubeTranscriptToolInputSchema(video_url=mock_video_url) diff --git a/claude-plugin/atomic-agents/.claude-plugin/plugin.json b/claude-plugin/atomic-agents/.claude-plugin/plugin.json index 7638345f..6764666f 100644 --- a/claude-plugin/atomic-agents/.claude-plugin/plugin.json +++ b/claude-plugin/atomic-agents/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "atomic-agents", "version": "2.2.0", - "description": "Skills plus explorer and reviewer subagents for building, scaffolding, debugging, understanding, and auditing applications with the Atomic Agents Python framework. Includes the umbrella `framework` skill, action-oriented `create-atomic-schema` / `create-atomic-agent` / `create-atomic-tool` / `create-atomic-context-provider` skills, the `troubleshoot` diagnostic skill, the `new-app` scaffolder (emits AGENTS.md + CLAUDE.md), progressive-disclosure reference material for prompts, orchestration, memory, hooks, providers, project structure, and testing, and isolated-context subagents for codebase mapping and code review.", + "description": "Skills plus explorer and reviewer subagents for building, scaffolding, debugging, understanding, and auditing applications with the Atomic Agents Python framework. Includes the umbrella `framework` skill, action-oriented `create-atomic-schema` / `create-atomic-agent` / `create-atomic-tool` / `create-atomic-context-provider` skills, the `find-forge-tools` discovery skill, the `troubleshoot` diagnostic skill, the `new-app` scaffolder (emits AGENTS.md + CLAUDE.md), progressive-disclosure reference material for prompts, orchestration, memory, hooks, providers, project structure, and testing, and isolated-context subagents for codebase mapping and code review.", "author": { "name": "Kenny Vaneetvelde", "email": "kenny@eigenwise.io" diff --git a/claude-plugin/atomic-agents/skills/create-atomic-tool/SKILL.md b/claude-plugin/atomic-agents/skills/create-atomic-tool/SKILL.md index 3e6e9cef..a727072d 100644 --- a/claude-plugin/atomic-agents/skills/create-atomic-tool/SKILL.md +++ b/claude-plugin/atomic-agents/skills/create-atomic-tool/SKILL.md @@ -13,6 +13,7 @@ For deep material (MCP interop, distributing as a standalone package, advanced e - **This skill**: the user is creating a specific tool — wrapping an API, building a calculator, scraping a page, querying a DB. - **`framework` skill**: questions about Atomic Agents in general, or the user is doing something other than authoring a tool. +- **`find-forge-tools` skill**: the user needs a capability and should check configured Forge sources for an existing tool first. ## Phase 1 — Clarify @@ -153,7 +154,44 @@ class WeatherTool(BaseTool[WeatherInput, WeatherOutput]): - The async hook is `async def run_async`, not `arun` — the framework calls `run_async`. - Convert routine failures (rate limits, 404s, validation rejects from the upstream API) into a typed failure output. Reserve `raise` for programmer error. -## Phase 4 — Wire it into an agent +## Phase 4 — Package it for the forge + +When the user wants a reusable or downloadable tool, emit a complete forge package instead of loose project files. Before creating a new tool, check `../find-forge-tools/SKILL.md` when that skill is available, then inspect the forge index and existing tools so the name and capability are not duplicated. + +Create this directory tree, file for file: + +```text +tool_name/ +│ .coveragerc +│ pyproject.toml +│ README.md +│ requirements.txt +│ uv.lock +│ +├── tool/ +│ │ tool_name.py +│ │ some_util_file.py +│ │ another_util_file.py +│ +└── tests/ + │ test_tool_name.py + │ test_some_util_file.py + │ test_another_util_file.py +``` + +The package must include: + +- `.coveragerc`, copied from the standard forge configuration. +- `pyproject.toml`, with the tool name, version, description, `README.md` readme, Python requirement, runtime dependencies, and the standard development dependency group. Keep runtime dependencies under `[project].dependencies` and development-only dependencies under `[dependency-groups].dev`. +- `requirements.txt`, containing exactly the runtime dependencies from `pyproject.toml`, with the Python version constraint omitted. Do not copy development dependencies into this file. +- `tool/`, containing the implementation and any small helper modules. Keep the public tool class, input/output schemas, and config in importable modules. +- `tests/`, containing tests for the schemas, configuration, success path, and routine failure paths. Add focused tests for HTTP or async behavior when those paths exist. +- `README.md`, with these standard sections: Overview, Prerequisites and Dependencies, Installation, Configuration when the tool needs it, Input & Output Structure, Usage, Contributing, and License. +- `uv.lock`, generated by running `uv sync` from the package directory after metadata and dependencies are complete. + +Run `uv sync` before handing off the package. The generated package must be self-contained and importable from its own directory, while still following the `BaseIOSchema`, `BaseToolConfig`, and explicit `BaseTool[Input, Output]` contracts above. + +## Phase 5 — Wire it into an agent Two integration shapes (see `../framework/references/tools.md` for more): @@ -169,15 +207,29 @@ result = tool.run(call) **Router agent** — agent picks among tools via a discriminated union of tool-call schemas. Use this when the agent has 2–10 tools to choose from. For dozens, see the search+execute pattern in `../framework/references/orchestration.md`. -## Phase 5 — Verify +## Phase 6 — Verify + +For an in-project tool, run a self-contained smoke test from the project root: ```bash uv run python -c "from .tools._tool import Tool, Input; t = Tool(); print(t.run(Input(...)))" ``` -If imports fail with the docstring error, add the docstring on the schema. If `self.input_schema` is `None`, the generic parameters are missing — write `class FooTool(BaseTool[FooInput, FooOutput]):`, not `class FooTool(BaseTool):`. +For a standalone Forge package, keep the package contract from `../framework/references/tools.md`: `pyproject.toml`, `README.md`, `requirements.txt`, `tool/`, and `tests/`. Run its own tests from the package root so the smoke test uses the package's declared dependencies: + +```bash +uv run pytest tests +``` + +Only run the repository conformance suite after contributing the package to Forge. First place it under `atomic-forge/tools/` and regenerate `atomic-forge/index.json`, then run: + +```bash +uv run pytest atomic-forge/conformance +``` + +The conformance suite checks the required forge layout, clean imports, schema and config contracts, explicit `BaseTool` generics, runtime dependency parity, and index metadata. If imports fail with the docstring error, add the docstring on the schema. If `self.input_schema` is `None`, the generic parameters are missing — write `class FooTool(BaseTool[FooInput, FooOutput]):`, not `class FooTool(BaseTool)`. -## Phase 6 — Hand off +## Phase 7 — Hand off Tell the user: @@ -186,7 +238,7 @@ Tell the user: - Optional next steps: - The agent that calls it → `create-atomic-agent` skill. - Multi-agent wiring around the tool → `../framework/references/orchestration.md`. - - MCP interop or packaging the tool for distribution → `../framework/references/tools.md`. + - MCP interop, Forge package structure, or discovering/downloading Forge tools → `../framework/references/tools.md`. ## Anti-patterns diff --git a/claude-plugin/atomic-agents/skills/find-forge-tools/SKILL.md b/claude-plugin/atomic-agents/skills/find-forge-tools/SKILL.md new file mode 100644 index 00000000..91fc4de8 --- /dev/null +++ b/claude-plugin/atomic-agents/skills/find-forge-tools/SKILL.md @@ -0,0 +1,46 @@ +--- +name: find-forge-tools +description: Find and download an existing Atomic Forge tool before writing one from scratch. Use when an Atomic Agents project needs a tool capability, such as search, scraping, weather, PDF, or date/time support. +--- + +# Find Forge Tools + +Check Atomic Forge before creating a tool from scratch. A Forge is a Git repository with an index of downloadable tools, so configured sources can include private forges through the user's existing Git credentials. + +## 1. Search the configured forges + +From the project root, run: + +```bash +atomic list +``` + +This searches every configured source. If the CLI is unavailable, read `atomic-forge/index.json` in the repository instead. Match the requested capability against the tool name and description. If names overlap, keep the `source/name` form shown by `atomic list`. + +## 2. Download a match + +Download the matching tool into the project: + +```bash +atomic download +``` + +Use the source-qualified name when needed: + +```bash +atomic download / +``` + +If the project needs a specific destination, pass the supported destination flag: + +```bash +atomic download --dest +``` + +Read the downloaded README and wire the tool into the agent's tool list. Confirm its dependencies and configuration before running it. + +## 3. Hand off when there is no match + +When the forge index has no suitable tool, continue with the `create-atomic-tool` skill at `../create-atomic-tool/SKILL.md`. It covers the schemas, `BaseTool` implementation, configuration, failure outputs, and verification for a new in-project tool. + +If a new tool would be useful to other projects, package it for the Forge format after implementing it and add it to the relevant source index. diff --git a/docs/contributing.md b/docs/contributing.md index 1822809f..228830ed 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -54,38 +54,43 @@ We follow these coding standards: ## Creating Tools -When creating new tools: +When creating a new tool, follow the [tool structure guide](../atomic-forge/guides/tool_structure.md), which covers the package layout, schemas, configuration, implementation, tests, and documentation. The `create-atomic-tool` skill in the Atomic Agents Claude plugin can guide you through the same process as an assisted path. -1. Use the tool template: - ```bash - atomic-assembler create-tool my-tool - ``` +The core implementation uses typed `BaseIOSchema` contracts and a `BaseToolConfig` configuration class: -2. Implement the required interfaces: - ```python - from pydantic import BaseModel - from atomic_agents import BaseTool - - class MyToolInputs(BaseModel): - # Define input schema - pass - - class MyToolOutputs(BaseModel): - # Define output schema - pass - - class MyTool(BaseTool[MyToolInputs, MyToolOutputs]): - name = "my_tool" - description = "Tool description" - inputs_schema = MyToolInputs - outputs_schema = MyToolOutputs - - def run(self, inputs: MyToolInputs) -> MyToolOutputs: - # Implement tool logic - pass - ``` +```python +from pydantic import Field + +from atomic_agents import BaseIOSchema, BaseTool, BaseToolConfig + + +class MyToolInputSchema(BaseIOSchema): + """Input for MyTool.""" + + value: str = Field(..., description="Value to process.") + + +class MyToolOutputSchema(BaseIOSchema): + """Result returned by MyTool.""" + + result: str = Field(..., description="Processed value.") + + +class MyToolConfig(BaseToolConfig): + """Configuration for MyTool.""" + + +class MyTool(BaseTool[MyToolInputSchema, MyToolOutputSchema]): + """Process a value.""" + + def __init__(self, config: MyToolConfig = MyToolConfig()): + super().__init__(config) + + def run(self, params: MyToolInputSchema) -> MyToolOutputSchema: + return MyToolOutputSchema(result=params.value) +``` -3. Add comprehensive tests: +1. Add comprehensive tests: ```python def test_my_tool(): tool = MyTool() diff --git a/docs/guides/atomic_forge.md b/docs/guides/atomic_forge.md new file mode 100644 index 00000000..585b4a61 --- /dev/null +++ b/docs/guides/atomic_forge.md @@ -0,0 +1,91 @@ +# Atomic Forge and Assembler + +The **Atomic Forge** is a collection of standalone tool packages. You download a tool into your project, review the source, and own the copy. Tools become part of your codebase, so you can change them for your use case and install only the dependencies you need. + +The **Atomic Assembler** is the `atomic` CLI for browsing and downloading Forge tools. + +## Quick workflow + +List the tools in your configured Forge sources: + +```bash +atomic list +``` + +Download a tool by name: + +```bash +atomic download +``` + +If two sources contain the same tool, use `source/name` as the name. Choose a destination with `--dest`: + +```bash +atomic download --dest ./tools/ +``` + +The downloaded package includes its source, tests, `pyproject.toml`, and `requirements.txt`. Install its runtime dependencies, then use the tool as Python code in your project: + +```bash +pip install -r .//requirements.txt +``` + +See the downloaded tool's `README.md` for its import and usage example. `atomic download --help` shows the current command options. + +## Private and additional Forges + +A Forge is any Git repository with a `tools/` directory and an `index.json` file. To make your own Forge, add tool packages under `tools/`, then generate the index from the repository root: + +```bash +python atomic-forge/scripts/generate_index.py +``` + +The generator reads each tool's `pyproject.toml` and writes the Forge `index.json`. Commit both the tools and the generated index. A private Forge uses the same layout and does not need a registry service. + +Add it to your local source list with a name and Git URL. For the standalone layout above, pass the root `tools/` directory explicitly: + +```bash +atomic sources add company https://git.example.com/company/atomic-forge.git \ + --tools-path tools +``` + +Omitting `--tools-path` uses the official repository layout, where tools live under `atomic-forge/tools`. + +For a non-default branch or tools directory, pass the shipped options: + +```bash +atomic sources add company https://git.example.com/company/atomic-forge.git \ + --branch main --tools-path tools +``` + +The Assembler clones sources with your normal Git setup. Configure authentication through Git credentials, SSH keys, or your usual credential helper. The Forge does not handle credentials itself. + +Use these commands to inspect or remove configured sources: + +```bash +atomic sources list +atomic sources remove company +``` + +Run `atomic sources add --help` for all available flags. + +## Forge tool conformance + +Every Forge tool follows the same package contract: + +- It has one focused responsibility and can run independently. +- Its input and output schemas inherit from `BaseIOSchema`. +- Its configuration inherits from `BaseToolConfig`. +- Its main tool class inherits from `BaseTool`. +- It keeps implementation code in `tool/` and tests in `tests/`. +- It includes `pyproject.toml`, `README.md`, and a runtime-only `requirements.txt`. +- Its README documents purpose, configuration, and usage. + +The complete standard, including the expected package layout and authoring details, is in the [Atomic Tool structure guide](https://github.com/eigenwise/atomic-agents/blob/main/atomic-forge/guides/tool_structure.md). + +When you want an assisted authoring workflow, use the `create-atomic-tool` skill from the Atomic Agents Claude plugin. It walks through the schemas, configuration, implementation, tests, verification, and hand-off for a distributable tool. + +## Related guides + +- [Tools Guide](tools.md): compose tools with agents and application code. +- [Atomic Forge directory](https://github.com/eigenwise/atomic-agents/tree/main/atomic-forge): browse the shipped tools and Forge files. diff --git a/docs/guides/index.md b/docs/guides/index.md index ae40e05f..2e8db171 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -9,6 +9,7 @@ This section contains detailed guides for working with Atomic Agents. quickstart memory tools +atomic_forge hooks orchestration cookbook diff --git a/docs/guides/tools.md b/docs/guides/tools.md index 9b5d9cbe..856461e8 100644 --- a/docs/guides/tools.md +++ b/docs/guides/tools.md @@ -135,6 +135,8 @@ When in doubt, start with Pattern 1. Add a choice agent only when you actually h ## The Atomic Forge: where tools live +For the download workflow, private Forge sources, and the package conformance standard, see the [Atomic Forge and Assembler guide](atomic_forge.md). + Tools themselves are distributed via the **Atomic Forge** — a registry of standalone, modular tool packages that you download into your project. The Forge approach gives you: 1. **Full Control**: You own the tool's source. Modify behavior locally without forking the framework.