diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b5e659..967aafb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,31 @@ jobs: working-directory: agents/langgraph-reference run: pytest -q + schema: + name: schema conformance + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + + - name: JSON Schema stays in sync with the Go types + run: go test ./trajectory/ -run "TestSchema" -v + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install the schema validator + working-directory: agents/langgraph-reference + run: pip install -r requirements.txt + + - name: Fixtures and emitter output conform to the published JSON Schema + working-directory: agents/langgraph-reference + run: pytest -q tests/test_schema.py + trazo-gate: name: trazo gate demo runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4032b27 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: Release + +# Cut a release by pushing a semver tag, e.g. `git tag v0.2.0 && git push --tags`. +# The tag must match the compiled-in const Version in cmd/trazo/version.go, or the +# guard below fails the build before anything is published. +on: + push: + tags: ["v*"] + +permissions: + contents: write # required to create the GitHub release and upload artifacts + +jobs: + release: + name: build and publish binaries + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # tags and full history so build info is stamped correctly + + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + + - name: Tag matches the compiled-in version + run: | + tag="${GITHUB_REF_NAME#v}" + code="$(sed -n 's/.*const Version = "\([^"]*\)".*/\1/p' cmd/trazo/version.go)" + echo "tag=$tag code=$code" + if [ "$tag" != "$code" ]; then + echo "::error::tag v$tag does not match const Version $code in cmd/trazo/version.go" + exit 1 + fi + + - name: Build, vet, test before releasing + run: | + go vet ./... + go test ./... + + - name: Cross-compile release archives + run: make dist VERSION="${GITHUB_REF_NAME}" + + - name: Create the GitHub release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "${GITHUB_REF_NAME}" \ + --title "${GITHUB_REF_NAME}" \ + --generate-notes \ + dist/* diff --git a/.gitignore b/.gitignore index c4123ba..a98ef3f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ # Test binary, built with `go test -c` *.test +# trazo build artifacts (Makefile `build`/`install` and ad-hoc `go build -o trazo`) +/bin/ +/trazo + # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..fd0b0b9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,93 @@ +# Changelog + +All notable changes to trazo are documented here. The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project aims +to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html) for the +product version. The trace **schema** version is tracked separately; see +[docs/versioning.md](docs/versioning.md). + +## [Unreleased] + +### Added + +- `trazo version` subcommand and `-version` flag, reporting the product version, + the supported trace schema version, the git commit, and the build time. Commit + and build metadata come from the Go toolchain's build info (no ldflags needed). +- [docs/versioning.md](docs/versioning.md): the product and schema versioning + policy, the schema compatibility contract, and the release process. +- Versioned, self-describing JSON output: `-format json` now emits an envelope + with `outputVersion`, `trazoVersion`, `traceSchemaVersion`, `generatedAt`, an + aggregate `summary`, `results`, and structured `errors`. Documented in + [docs/output.md](docs/output.md). +- Evaluator policy file: `-config ` loads a versioned, stdlib-only JSON + policy (the new `config` package) that pins which evaluators run and their + thresholds, for reproducibility across dev, CI, and teams. Precedence is + defaults < config < explicitly-set flags. Documented in + [docs/config.md](docs/config.md), with an example config. +- Black-box CLI tests (`cmd/trazo/cli_test.go`) that build the binary and assert + exit codes (0/1/2), each output format, invalid flags, missing paths, empty + directories, and config/flag precedence. +- A dedicated `schema conformance` CI job that runs the Go schema-sync tests and + validates fixtures and emitter output against the published JSON Schema, plus a + test that the shipped example config always loads. +- `-verbose` flag: prints operational metrics to stderr after a run + (`loaded=N valid=V invalid=I evaluated=E duration=Xms`), so batch runs are + observable without parsing the results. +- Typed file errors: `runner.FileError` now carries an `ErrorKind` + (`read_file`, `invalid_json`, `invalid_trace`, `evaluator`, `canceled`), + surfaced as `errors[].kind` in the JSON output (bumped to `outputVersion` 1.1, + additive) and tagged in the text output. +- A realistic integration fixture (`testdata/complex_run.json`) and test that + runs the full structural evaluator set over one interleaved trace, asserting + how the work divides across judgments; it is also covered by the Python schema + conformance suite. +- [docs/llm-judge.md](docs/llm-judge.md): the opt-in LLM-as-judge evaluator, its + determinism (temperature 0, capped tokens), timeout and cancellation model, and + the split between a malformed verdict (neutral finding) and a transport failure + (a run error; structural evaluators still run). +- [docs/security.md](docs/security.md): what a trace holds, the single component + that sends data off-machine (the judge, final output only), how to disable + network egress, redaction at emit time, and not committing private traces. +- Packaging and release: `LICENSE` (MIT), a `Makefile` (build, check, gate, + install, cross-compiled `dist` archives with checksums), a tag-triggered GitHub + Actions release workflow that guards the tag against the compiled-in version and + publishes binaries for Linux, macOS, and Windows (amd64/arm64), and a README + with badges, `go install` instructions, and links to the core docs. +- This changelog. + +### Changed + +- BREAKING (JSON output): the top-level keys `evaluations` and `fileErrors` are + now `results` and `errors`, nested under the new envelope. Consumers should + read `outputVersion` and ignore unknown fields. + +## [0.1.0] + +The hardening round: formalize the trace contract and round the core out from a +working MVP toward production-grade. + +### Added + +- Formal JSON Schema (`trajectory/trace.schema.json`, draft 2020-12) as the + strict external contract, kept in sync with the Go types by a dependency-free + test and validated against fixtures and emitter output from Python. +- Trace schema versioning: `version` is required and semver-gated by major (see + docs/versioning.md). +- Explicit `toolCallId` for authoritative tool call/result pairing, with + name/FIFO as a documented fallback. +- CLI: single-file input, `-recursive`, `-validate` (structure-only), up-front + `-format` validation, and a proper `-help`. +- Golden tests for the text, JSON, and Markdown output formats. + +### Changed + +- `Evaluator` interface takes a `context.Context`, so Ctrl+C or a CI timeout + aborts in-flight work (notably the LLM judge). +- `Run.Validate` deepened: non-negative quantities, monotonic step timestamps, + steps within the run interval, per-type required fields. +- Text output regrouped by run with a summary footer. +- Output formatting centralized in the `report` package; `Evaluator.go` renamed + to `evaluator.go`. + +[Unreleased]: https://github.com/Cro22/trazo/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/Cro22/trazo/releases/tag/v0.1.0 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ecdab74 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Jesús Núñez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..70e4ea5 --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +# trazo build and release tasks. +# +# The Go core is standard-library only, so these targets need nothing but the Go +# toolchain (plus tar/zip/sha256sum for `dist`, which are present on CI runners +# and most Unix hosts). On Windows, run under Git Bash or WSL. + +BIN := trazo +CMD := ./cmd/trazo +BINDIR := bin +DISTDIR := dist + +# Version stamped into archive names. The binary itself reports the compiled-in +# const Version plus git metadata from the Go toolchain (see cmd/trazo/version.go); +# this is only for naming the release artifacts. +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) + +# Release target matrix (GOOS/GOARCH). +PLATFORMS := linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 + +.DEFAULT_GOAL := build +.PHONY: build check test vet fmt install gate agent-test dist clean + +build: ## Build the CLI into ./bin + go build -trimpath -o $(BINDIR)/$(BIN) $(CMD) + +check: vet test ## Full Go gate: vet then test + go build ./... + +test: ## Run the Go test suite + go test ./... + +vet: ## Run go vet + go vet ./... + +fmt: ## Format all Go sources + go fmt ./... + +install: ## Install the CLI into GOBIN (go install) + go install $(CMD) + +# Mirror the CI trazo-gate job locally: clean traces pass, bad traces must fail. +gate: build ## Demonstrate the evaluator gate on the CI fixtures + $(BINDIR)/$(BIN) -dir testdata/ci/clean + @echo "clean traces passed (exit 0)" + @if $(BINDIR)/$(BIN) -dir testdata/ci/failing; then \ + echo "gate did not fail on a bad finding" >&2; exit 1; \ + fi + @echo "bad traces correctly failed the gate (exit non-zero)" + +# Run the Python reference-agent tests. Requires the venv from +# agents/langgraph-reference (see the README quickstart). +agent-test: ## Run the reference-agent pytest suite + cd agents/langgraph-reference && pytest -q + +# Cross-compile release archives plus a checksums file into ./dist. Each archive +# holds one static binary; unix targets are .tar.gz, windows is .zip. +dist: ## Build release archives for every target platform + rm -rf $(DISTDIR) + mkdir -p $(DISTDIR) + @for p in $(PLATFORMS); do \ + os=$${p%/*}; arch=$${p#*/}; \ + ext=; [ $$os = windows ] && ext=.exe; \ + echo "building $$os/$$arch"; \ + CGO_ENABLED=0 GOOS=$$os GOARCH=$$arch \ + go build -trimpath -o $(DISTDIR)/$(BIN)$$ext $(CMD) || exit 1; \ + base=$(BIN)_$(VERSION)_$$os_$$arch; \ + if [ $$os = windows ]; then \ + (cd $(DISTDIR) && zip -q $$base.zip $(BIN)$$ext && rm $(BIN)$$ext); \ + else \ + (cd $(DISTDIR) && tar czf $$base.tar.gz $(BIN)$$ext && rm $(BIN)$$ext); \ + fi; \ + done + @(cd $(DISTDIR) && sha256sum * > checksums.txt) + @echo "artifacts in $(DISTDIR):" + @ls -1 $(DISTDIR) + +clean: ## Remove build and release artifacts + rm -rf $(BINDIR) $(DISTDIR) $(BIN) $(BIN).exe diff --git a/README.md b/README.md index 1eb3697..fe1b76f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,35 @@ # trazo +[![CI](https://github.com/Cro22/trazo/actions/workflows/ci.yml/badge.svg)](https://github.com/Cro22/trazo/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/Cro22/trazo?sort=semver)](https://github.com/Cro22/trazo/releases) +[![Go version](https://img.shields.io/github/go-mod/go-version/Cro22/trazo)](go.mod) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) + Trajectory evaluation for LLM agents. trazo ingests agent run traces as JSON and runs evaluators over them, producing findings with a severity taxonomy. The core is written in Go; a Python LangGraph reference agent proves the framework end to end: real agent produces real traces, and the unmodified Go core evaluates them. +## Install + +The CLI is a single self-contained binary with no runtime dependencies. + +```bash +# With the Go toolchain (installs into $GOBIN): +go install github.com/Cro22/trazo/cmd/trazo@latest + +# Or from source: +git clone https://github.com/Cro22/trazo && cd trazo +make build # -> ./bin/trazo (or: go build -o trazo ./cmd/trazo) +``` + +Prebuilt binaries for Linux, macOS, and Windows (amd64 and arm64) are attached to +each [GitHub release](https://github.com/Cro22/trazo/releases). Verify a download +against the release's `checksums.txt`, then put the binary on your `PATH`. + +`trazo version` reports the product version, the supported trace schema version, +and the build's git commit and time. + ## The severity taxonomy Every finding carries one judgment (`evaluator/Evaluator.go`): @@ -62,6 +87,18 @@ directory with a bad finding (`testdata/ci/failing`) makes trazo exit non-zero. The Markdown report is written to the job summary. This is how trazo fails a build on a `JudgmentBad`. +## Core documentation + +- [docs/config.md](docs/config.md) — the `-config` evaluator policy file and + the defaults < config < flags precedence. +- [docs/output.md](docs/output.md) — the versioned JSON output envelope. +- [docs/versioning.md](docs/versioning.md) — product versus trace-schema + versioning and the release process. +- [docs/llm-judge.md](docs/llm-judge.md) — the opt-in LLM-as-judge evaluator: + determinism, timeouts, cost, and failure handling. +- [docs/security.md](docs/security.md) — what a trace holds, the one component + that sends data off-machine, and how to keep private traces private. + ## Reference agent (LangGraph) `agents/langgraph-reference/` is a GitHub-issue triage agent built with @@ -148,3 +185,7 @@ Model tier is a cheap flash model (`gemini-2.5-flash` by default) with temperature 0 and a low output-token cap. A full triage of one repo is about 2,400 tokens and costs roughly **$0.002** per run. Override the model with `--model` and bound the loop with `--iteration-cap`. + +## License + +MIT. See [LICENSE](LICENSE). diff --git a/agents/langgraph-reference/docs/trace-schema.md b/agents/langgraph-reference/docs/trace-schema.md index 6079121..edfef7e 100644 --- a/agents/langgraph-reference/docs/trace-schema.md +++ b/agents/langgraph-reference/docs/trace-schema.md @@ -148,6 +148,9 @@ stamps it on every trace by default. Keep the two constants in sync; a Go test (`trajectory/schema_test.go`) checks the JSON Schema requires `version` and that `SchemaVersion` passes the gate. +The full compatibility contract, the product-vs-schema version distinction, and +the release process live in [`docs/versioning.md`](../../../docs/versioning.md). + ## Tool call / result pairing (what the emitter must respect) `ToolCallEvaluator` (`evaluator/toolcalls.go`) walks the steps in order and pairs diff --git a/agents/langgraph-reference/tests/test_agent.py b/agents/langgraph-reference/tests/test_agent.py index 7bc841b..868cb53 100644 --- a/agents/langgraph-reference/tests/test_agent.py +++ b/agents/langgraph-reference/tests/test_agent.py @@ -85,7 +85,7 @@ def test_agent_trace_evaluates_clean_in_go(tmp_path) -> None: ) assert proc.stdout, proc.stderr out = json.loads(proc.stdout) - assert out["fileErrors"] == [] + assert out["errors"] == [] # Clean means every evaluator (tool_calls, loops, cost_latency, node) is clean. - findings = [f for e in out["evaluations"] if e["runId"] == result.run_id for f in e["findings"]] + findings = [f for e in out["results"] if e["runId"] == result.run_id for f in e["findings"]] assert findings == [], findings diff --git a/agents/langgraph-reference/tests/test_cross_language.py b/agents/langgraph-reference/tests/test_cross_language.py index 0806488..9ed6872 100644 --- a/agents/langgraph-reference/tests/test_cross_language.py +++ b/agents/langgraph-reference/tests/test_cross_language.py @@ -40,10 +40,10 @@ def _run_trazo(traces_dir: Path) -> dict: def _findings_for(result: dict, run_id: str) -> list[dict]: - if not any(ev["runId"] == run_id for ev in result["evaluations"]): + if not any(ev["runId"] == run_id for ev in result["results"]): raise AssertionError(f"run {run_id} not found in {result}") # Aggregate findings across every evaluator for this run. - return [f for ev in result["evaluations"] if ev["runId"] == run_id for f in ev["findings"]] + return [f for ev in result["results"] if ev["runId"] == run_id for f in ev["findings"]] def test_clean_run_evaluates_with_no_findings(tmp_path) -> None: @@ -55,7 +55,7 @@ def test_clean_run_evaluates_with_no_findings(tmp_path) -> None: rec.flush(tmp_path, end_time=_dt(4)) result = _run_trazo(tmp_path) - assert result["fileErrors"] == [] + assert result["errors"] == [] assert _findings_for(result, "run-clean") == [] diff --git a/agents/langgraph-reference/tests/test_scenarios.py b/agents/langgraph-reference/tests/test_scenarios.py index c09ee15..4f65b12 100644 --- a/agents/langgraph-reference/tests/test_scenarios.py +++ b/agents/langgraph-reference/tests/test_scenarios.py @@ -85,5 +85,5 @@ def test_scenarios_produce_expected_severity_in_go(tmp_path, name, judgment) -> assert proc.stdout, proc.stderr out = json.loads(proc.stdout) # Aggregate findings across every evaluator for this run. - findings = [f for e in out["evaluations"] if e["runId"] == result.run_id for f in e["findings"]] + findings = [f for e in out["results"] if e["runId"] == result.run_id for f in e["findings"]] assert any(f["judgment"] == judgment for f in findings), findings diff --git a/agents/langgraph-reference/tests/test_schema.py b/agents/langgraph-reference/tests/test_schema.py index 15f2259..a0be645 100644 --- a/agents/langgraph-reference/tests/test_schema.py +++ b/agents/langgraph-reference/tests/test_schema.py @@ -32,6 +32,7 @@ "testdata/sample_run_missing_result.json", "testdata/ci/clean/triage_clean.json", "testdata/ci/failing/tool_error.json", + "testdata/complex_run.json", ] diff --git a/cmd/trazo/cli_test.go b/cmd/trazo/cli_test.go new file mode 100644 index 0000000..6754abe --- /dev/null +++ b/cmd/trazo/cli_test.go @@ -0,0 +1,228 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// trazoBin is the freshly built binary under test, shared across the black-box +// tests below. Building once in TestMain keeps the suite fast. +var trazoBin string + +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "trazo-cli-test") + if err != nil { + fmt.Fprintln(os.Stderr, "mktemp:", err) + os.Exit(1) + } + bin := filepath.Join(dir, "trazo") + if runtime.GOOS == "windows" { + bin += ".exe" + } + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + fmt.Fprintln(os.Stderr, "building trazo:", err) + os.RemoveAll(dir) + os.Exit(1) + } + trazoBin = bin + + code := m.Run() + os.RemoveAll(dir) + os.Exit(code) +} + +// Fixture paths, relative to this package directory (cmd/trazo). +const ( + cleanDir = "../../testdata/ci/clean" + cleanFile = "../../testdata/ci/clean/triage_clean.json" + failingFile = "../../testdata/ci/failing/tool_error.json" + brokenFile = "../../testdata/runs/broken.json" +) + +type cliResult struct { + stdout string + stderr string + code int +} + +func runCLI(t *testing.T, args ...string) cliResult { + t.Helper() + cmd := exec.Command(trazoBin, args...) + var out, errb bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &errb + err := cmd.Run() + + code := 0 + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + code = ee.ExitCode() + } else { + t.Fatalf("running %v: %v", args, err) + } + } + return cliResult{stdout: out.String(), stderr: errb.String(), code: code} +} + +func TestCLI_TextCleanExit0(t *testing.T) { + r := runCLI(t, "-dir", cleanDir) + if r.code != 0 { + t.Fatalf("expected exit 0, got %d (stderr: %s)", r.code, r.stderr) + } + if !strings.Contains(r.stdout, "clean") || !strings.Contains(r.stdout, "Summary:") { + t.Errorf("expected a clean summary, got:\n%s", r.stdout) + } +} + +func TestCLI_BadExit1(t *testing.T) { + r := runCLI(t, failingFile) + if r.code != 1 { + t.Fatalf("expected exit 1 on a bad finding, got %d (stdout: %s)", r.code, r.stdout) + } + if !strings.Contains(r.stdout, "[BAD]") { + t.Errorf("expected a BAD finding in output, got:\n%s", r.stdout) + } +} + +func TestCLI_FileErrorExit2(t *testing.T) { + r := runCLI(t, brokenFile) + if r.code != 2 { + t.Fatalf("expected exit 2 on a file error, got %d", r.code) + } + if !strings.Contains(r.stdout, "File errors:") { + t.Errorf("expected a file errors section, got:\n%s", r.stdout) + } +} + +func TestCLI_JSONBadExit1(t *testing.T) { + r := runCLI(t, "-json", failingFile) + if r.code != 1 { + t.Fatalf("expected exit 1, got %d", r.code) + } + var env struct { + OutputVersion string `json:"outputVersion"` + Summary struct { + Bad int `json:"bad"` + } `json:"summary"` + Results []json.RawMessage `json:"results"` + } + if err := json.Unmarshal([]byte(r.stdout), &env); err != nil { + t.Fatalf("output is not valid JSON: %v\n%s", err, r.stdout) + } + if env.OutputVersion == "" { + t.Error("JSON envelope missing outputVersion") + } + if env.Summary.Bad < 1 { + t.Errorf("expected at least one bad finding in summary, got %d", env.Summary.Bad) + } + if len(env.Results) == 0 { + t.Error("expected non-empty results") + } +} + +func TestCLI_MarkdownFileErrorExit2(t *testing.T) { + r := runCLI(t, "-format", "md", brokenFile) + if r.code != 2 { + t.Fatalf("expected exit 2, got %d", r.code) + } + if !strings.Contains(r.stdout, "**Verdict: FAIL**") || !strings.Contains(r.stdout, "File errors") { + t.Errorf("expected a FAIL markdown report with file errors, got:\n%s", r.stdout) + } +} + +func TestCLI_DirectoryDoesNotExist(t *testing.T) { + r := runCLI(t, "./does-not-exist-xyz") + if r.code == 0 { + t.Fatalf("expected a non-zero exit for a missing path, got 0") + } + if !strings.Contains(r.stderr, "trazo:") { + t.Errorf("expected a prefixed error on stderr, got:\n%s", r.stderr) + } +} + +func TestCLI_InvalidFormat(t *testing.T) { + r := runCLI(t, "-format", "xml", cleanFile) + if r.code == 0 { + t.Fatalf("expected a non-zero exit for an invalid format") + } + if !strings.Contains(r.stderr, "unknown -format") { + t.Errorf("expected an unknown-format error, got:\n%s", r.stderr) + } +} + +func TestCLI_TwoPathsError(t *testing.T) { + r := runCLI(t, cleanFile, failingFile) + if r.code != 2 { + t.Fatalf("expected exit 2 for two PATH args, got %d", r.code) + } + if !strings.Contains(r.stderr, "at most one PATH") { + t.Errorf("expected a one-PATH error, got:\n%s", r.stderr) + } +} + +func TestCLI_Version(t *testing.T) { + r := runCLI(t, "version") + if r.code != 0 { + t.Fatalf("expected exit 0, got %d", r.code) + } + if !strings.Contains(r.stdout, "trazo "+Version) || !strings.Contains(r.stdout, "trace schema ") { + t.Errorf("unexpected version output:\n%s", r.stdout) + } +} + +func TestCLI_EmptyDirectory(t *testing.T) { + empty := t.TempDir() + r := runCLI(t, "-dir", empty) + if r.code != 0 { + t.Fatalf("expected exit 0 for an empty directory, got %d", r.code) + } + if !strings.Contains(r.stderr, "no .json traces") { + t.Errorf("expected a no-traces notice on stderr, got:\n%s", r.stderr) + } +} + +func TestCLI_Verbose(t *testing.T) { + r := runCLI(t, "-verbose", "-dir", "../../testdata/runs") + if !strings.Contains(r.stderr, "loaded=") || !strings.Contains(r.stderr, "duration=") { + t.Errorf("expected operational metrics on stderr, got:\n%s", r.stderr) + } + // testdata/runs has 2 valid and 2 invalid fixtures. + if !strings.Contains(r.stderr, "valid=2") || !strings.Contains(r.stderr, "invalid=2") { + t.Errorf("unexpected metric counts, got:\n%s", r.stderr) + } +} + +// TestCLI_FlagOverridesConfig pins the precedence rule: a config sets a lax cost +// threshold (no finding), and an explicit -max-step-cost flag overrides it (a +// finding appears). Both runs stay exit 0 because a cost finding is neutral. +func TestCLI_FlagOverridesConfig(t *testing.T) { + cfgPath := filepath.Join(t.TempDir(), "policy.json") + cfg := `{"version":1,"evaluators":{"cost_latency":{"enabled":true,"maxStepCost":1.0,"maxRunCost":1.0,"maxStepLatencyMs":600000,"maxRunLatencyMs":600000}}}` + if err := os.WriteFile(cfgPath, []byte(cfg), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + withConfig := runCLI(t, "-config", cfgPath, cleanFile) + if withConfig.code != 0 { + t.Fatalf("config-only run should be clean (exit 0), got %d:\n%s", withConfig.code, withConfig.stdout) + } + if strings.Contains(withConfig.stdout, "cost_latency") { + t.Errorf("lax config should produce no cost finding, got:\n%s", withConfig.stdout) + } + + overridden := runCLI(t, "-config", cfgPath, "-max-step-cost", "0.00001", cleanFile) + if !strings.Contains(overridden.stdout, "cost_latency") { + t.Errorf("flag should override config and produce a cost finding, got:\n%s", overridden.stdout) + } +} diff --git a/cmd/trazo/main.go b/cmd/trazo/main.go index 8a4c387..6f14639 100644 --- a/cmd/trazo/main.go +++ b/cmd/trazo/main.go @@ -8,10 +8,13 @@ import ( "os" "os/signal" "strings" + "time" + "github.com/Cro22/trazo/config" "github.com/Cro22/trazo/evaluator" "github.com/Cro22/trazo/report" "github.com/Cro22/trazo/runner" + "github.com/Cro22/trazo/trajectory" ) // Exit codes: 0 clean, 1 at least one JudgmentBad finding, 2 at least one @@ -27,8 +30,18 @@ func main() { log.SetFlags(0) log.SetPrefix("trazo: ") + // `trazo version` is a subcommand, handled before flag parsing so it works + // without any other arguments. + if len(os.Args) > 1 && os.Args[1] == "version" { + fmt.Print(versionReport(readBuildDetails())) + return + } + + showVersion := flag.Bool("version", false, "print version information and exit") + configPath := flag.String("config", "", "path to a JSON evaluator policy file (see docs/config.md)") dir := flag.String("dir", "./testdata/runs", "directory of traces to scan when no PATH is given") recursive := flag.Bool("recursive", false, "descend into subdirectories when PATH is a directory") + verbose := flag.Bool("verbose", false, "print operational metrics (loaded/valid/invalid/evaluated/duration) to stderr") validate := flag.Bool("validate", false, "only check that traces load and pass structural validation; skip evaluators") asJSON := flag.Bool("json", false, "print results as JSON (alias for -format json)") format := flag.String("format", "text", "output format: text, json, or md") @@ -45,6 +58,11 @@ func main() { flag.Usage = usage flag.Parse() + if *showVersion { + fmt.Print(versionReport(readBuildDetails())) + return + } + if flag.NArg() > 1 { log.Printf("at most one PATH may be given, got %d", flag.NArg()) flag.Usage() @@ -73,32 +91,84 @@ func main() { log.Printf("no .json traces found under %q", path) } - evaluators := buildEvaluators(*validate, evaluatorConfig{ - maxRepeats: *maxRepeats, - maxStepCost: *maxStepCost, - maxStepLatencyMs: *maxStepLatencyMs, - maxRunCost: *maxRunCost, - maxRunLatencyMs: *maxRunLatencyMs, - terminalNodes: splitCSV(*terminalNodes), - llmJudge: *llmJudge, - judgeModel: *judgeModel, + // Policy precedence: built-in defaults < config file < explicitly-set flags. + // The config file pins a reproducible policy; a flag the user actually passed + // still wins over it (flag.Visit reports only the flags that were set). + cfg := config.Default() + if *configPath != "" { + loaded, err := config.Load(*configPath) + if err != nil { + log.Fatalf("%v", err) + } + cfg = *loaded + } + flag.Visit(func(f *flag.Flag) { + switch f.Name { + case "max-repeats": + cfg.Evaluators.Loops.MaxRepeats = *maxRepeats + case "max-step-cost": + cfg.Evaluators.CostLatency.MaxStepCost = *maxStepCost + case "max-step-latency-ms": + cfg.Evaluators.CostLatency.MaxStepLatencyMs = *maxStepLatencyMs + case "max-run-cost": + cfg.Evaluators.CostLatency.MaxRunCost = *maxRunCost + case "max-run-latency-ms": + cfg.Evaluators.CostLatency.MaxRunLatencyMs = *maxRunLatencyMs + case "terminal-nodes": + cfg.Evaluators.NodeTransitions.TerminalNodes = splitCSV(*terminalNodes) + case "llm-judge": + cfg.Evaluators.LLMJudge.Enabled = *llmJudge + case "judge-model": + cfg.Evaluators.LLMJudge.Model = *judgeModel + } }) + // In validate-only mode the runner just loads and structurally validates each + // file, so no evaluators are built regardless of the policy. + var evaluators []evaluator.Evaluator + if !*validate { + evaluators, err = cfg.Build(func(model string) (evaluator.Evaluator, error) { + client, cerr := evaluator.NewGeminiClient(model) + if cerr != nil { + return nil, cerr + } + return &evaluator.LLMJudgeEvaluator{Client: client}, nil + }) + if err != nil { + log.Fatalf("llm-judge: %v", err) + } + } + // Cancel in-flight evaluation on Ctrl+C (SIGINT) or SIGTERM so a long run, // notably one using the network-bound LLM judge, stops promptly. ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() + start := time.Now() resp := runner.NewRunner(evaluators).RunFiles(ctx, files) + elapsed := time.Since(start) + + if *verbose { + invalid := distinctInvalidFiles(resp) + evaluated := report.Summarize(resp, len(files)).Runs + log.Printf("loaded=%d valid=%d invalid=%d evaluated=%d duration=%s", + len(files), len(files)-invalid, invalid, evaluated, elapsed.Round(time.Millisecond)) + } - render(out, *validate, resp, len(files)) + meta := report.Meta{ + TrazoVersion: Version, + TraceSchemaVersion: trajectory.SchemaVersion, + Files: len(files), + GeneratedAt: time.Now(), + } + render(out, *validate, resp, meta) os.Exit(exitCode(resp)) } func usage() { w := flag.CommandLine.Output() fmt.Fprintf(w, "trazo evaluates agent trace files (trazo JSON format) and reports findings.\n\n") - fmt.Fprintf(w, "Usage:\n trazo [flags] [PATH]\n\n") + fmt.Fprintf(w, "Usage:\n trazo [flags] [PATH]\n trazo version\n\n") fmt.Fprintf(w, "PATH is a single trace file or a directory of .json traces. If omitted, -dir is scanned.\n\n") fmt.Fprintf(w, "Flags:\n") flag.PrintDefaults() @@ -118,10 +188,10 @@ func resolveFiles(path string, recursive bool) ([]string, error) { return []string{path}, nil } -func render(out string, validate bool, resp *runner.Response, total int) { +func render(out string, validate bool, resp *runner.Response, meta report.Meta) { switch out { case "json": - s, err := report.JSON(resp) + s, err := report.JSON(resp, meta) if err != nil { log.Fatalf("encoding JSON: %v", err) } @@ -130,50 +200,21 @@ func render(out string, validate bool, resp *runner.Response, total int) { fmt.Print(report.Markdown(resp)) default: // text if validate { - fmt.Print(report.ValidateSummary(resp, total)) + fmt.Print(report.ValidateSummary(resp, meta.Files)) } else { fmt.Print(report.Text(resp)) } } } -type evaluatorConfig struct { - maxRepeats int - maxStepCost float64 - maxStepLatencyMs int64 - maxRunCost float64 - maxRunLatencyMs int64 - terminalNodes []string - llmJudge bool - judgeModel string -} - -// buildEvaluators assembles the evaluator set. In validate-only mode it returns -// none, so the runner just loads and structurally validates each file. The LLM -// judge is opt-in and constructed last because it can fail (missing API key). -func buildEvaluators(validate bool, cfg evaluatorConfig) []evaluator.Evaluator { - if validate { - return nil - } - evaluators := []evaluator.Evaluator{ - &evaluator.ToolCallEvaluator{}, - &evaluator.LoopEvaluator{MaxRepeats: cfg.maxRepeats}, - &evaluator.CostLatencyEvaluator{ - MaxStepCost: cfg.maxStepCost, - MaxStepLatencyMs: cfg.maxStepLatencyMs, - MaxRunCost: cfg.maxRunCost, - MaxRunLatencyMs: cfg.maxRunLatencyMs, - }, - &evaluator.NodeTransitionEvaluator{TerminalNodes: cfg.terminalNodes}, - } - if cfg.llmJudge { - client, err := evaluator.NewGeminiClient(cfg.judgeModel) - if err != nil { - log.Fatalf("llm-judge: %v", err) - } - evaluators = append(evaluators, &evaluator.LLMJudgeEvaluator{Client: client}) +// distinctInvalidFiles counts the unique files that produced at least one error, +// so a file with several evaluator errors is still counted once. +func distinctInvalidFiles(resp *runner.Response) int { + seen := map[string]bool{} + for _, fe := range resp.FileErrors { + seen[fe.File] = true } - return evaluators + return len(seen) } // splitCSV parses a comma-separated flag value into a trimmed, non-empty slice, diff --git a/cmd/trazo/version.go b/cmd/trazo/version.go new file mode 100644 index 0000000..252cba2 --- /dev/null +++ b/cmd/trazo/version.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "runtime" + "runtime/debug" + "strings" + + "github.com/Cro22/trazo/trajectory" +) + +// Version is the trazo product (binary) version. It is distinct from the trace +// schema version (trajectory.SchemaVersion): the product version tracks the CLI +// and library, the schema version tracks the trace format. See docs/versioning.md +// for the policy on bumping each. +const Version = "0.2.0" + +// buildDetails is the VCS/build metadata embedded by the Go toolchain. It is +// populated from runtime/debug build info, which `go build` fills in +// automatically from the enclosing git repo (no ldflags needed); under `go run` +// or `go test` it may be absent, in which case commit stays "unknown". +type buildDetails struct { + commit string + modified bool + buildTime string + goVersion string +} + +func readBuildDetails() buildDetails { + d := buildDetails{commit: "unknown", goVersion: runtime.Version()} + info, ok := debug.ReadBuildInfo() + if !ok { + return d + } + for _, s := range info.Settings { + switch s.Key { + case "vcs.revision": + d.commit = s.Value + case "vcs.time": + d.buildTime = s.Value + case "vcs.modified": + d.modified = s.Value == "true" + } + } + return d +} + +// versionReport renders the multi-line output of `trazo version`. It is kept pure +// (details passed in) so it can be tested without depending on build metadata. +func versionReport(d buildDetails) string { + commit := d.commit + if commit != "unknown" && len(commit) > 12 { + commit = commit[:12] + } + if d.modified { + commit += "-dirty" + } + + var b strings.Builder + fmt.Fprintf(&b, "trazo %s\n", Version) + fmt.Fprintf(&b, "trace schema %s\n", trajectory.SchemaVersion) + fmt.Fprintf(&b, "commit %s\n", commit) + if d.buildTime != "" { + fmt.Fprintf(&b, "built %s\n", d.buildTime) + } + fmt.Fprintf(&b, "%s\n", d.goVersion) + return b.String() +} diff --git a/cmd/trazo/version_test.go b/cmd/trazo/version_test.go new file mode 100644 index 0000000..aa09bd2 --- /dev/null +++ b/cmd/trazo/version_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "strings" + "testing" + + "github.com/Cro22/trazo/trajectory" +) + +func TestVersionReport_Fields(t *testing.T) { + got := versionReport(buildDetails{ + commit: "abcdef1234567890", + buildTime: "2026-08-07T12:00:00Z", + goVersion: "go1.25.0", + }) + + wantLines := []string{ + "trazo " + Version, + "trace schema " + trajectory.SchemaVersion, + "commit abcdef123456", // truncated to 12 chars + "built 2026-08-07T12:00:00Z", + "go1.25.0", + } + for _, line := range wantLines { + if !strings.Contains(got, line) { + t.Errorf("version report missing %q, got:\n%s", line, got) + } + } +} + +func TestVersionReport_DirtyAndUnknown(t *testing.T) { + dirty := versionReport(buildDetails{commit: "deadbeefcafebabe", modified: true, goVersion: "go1.25.0"}) + if !strings.Contains(dirty, "commit deadbeefcafe-dirty") { + t.Errorf("expected truncated dirty commit, got:\n%s", dirty) + } + + unknown := versionReport(buildDetails{commit: "unknown", goVersion: "go1.25.0"}) + if !strings.Contains(unknown, "commit unknown\n") { + t.Errorf("expected literal unknown commit, got:\n%s", unknown) + } + // With no build time, the "built" line is omitted entirely. + if strings.Contains(unknown, "built ") { + t.Errorf("did not expect a built line without build time, got:\n%s", unknown) + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..90e1ac8 --- /dev/null +++ b/config/config.go @@ -0,0 +1,167 @@ +// Package config defines trazo's evaluator policy file: a small, versioned JSON +// document that pins which evaluators run and with what thresholds, so a policy +// is reproducible across a developer's machine, CI, and a team. It is +// stdlib-only, like the rest of the core, and independent of the CLI: another Go +// program can Load a policy and build the same evaluator set. +package config + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + + "github.com/Cro22/trazo/evaluator" +) + +// SupportedVersion is the config file format version this build understands. The +// file must declare it explicitly; a different value is rejected rather than +// guessed at. +const SupportedVersion = 1 + +// Config is the whole policy file. Version is the config format version (not the +// product or trace-schema version). +type Config struct { + Version int `json:"version"` + Evaluators Evaluators `json:"evaluators"` +} + +type Evaluators struct { + ToolCalls ToolCalls `json:"tool_calls"` + Loops Loops `json:"loops"` + CostLatency CostLatency `json:"cost_latency"` + NodeTransitions NodeTransitions `json:"node_transitions"` + LLMJudge LLMJudge `json:"llm_judge"` +} + +type ToolCalls struct { + Enabled bool `json:"enabled"` +} + +type Loops struct { + Enabled bool `json:"enabled"` + MaxRepeats int `json:"maxRepeats"` +} + +type CostLatency struct { + Enabled bool `json:"enabled"` + MaxStepCost float64 `json:"maxStepCost"` + MaxStepLatencyMs int64 `json:"maxStepLatencyMs"` + MaxRunCost float64 `json:"maxRunCost"` + MaxRunLatencyMs int64 `json:"maxRunLatencyMs"` +} + +type NodeTransitions struct { + Enabled bool `json:"enabled"` + TerminalNodes []string `json:"terminalNodes"` +} + +type LLMJudge struct { + Enabled bool `json:"enabled"` + Model string `json:"model"` +} + +// Default is the built-in policy, matching the CLI's flag defaults: the four +// structural evaluators enabled, the LLM judge off. TerminalNodes is left nil so +// the node evaluator falls back to its own default set. +func Default() Config { + return Config{ + Version: SupportedVersion, + Evaluators: Evaluators{ + ToolCalls: ToolCalls{Enabled: true}, + Loops: Loops{Enabled: true, MaxRepeats: evaluator.DefaultMaxRepeats}, + CostLatency: CostLatency{ + Enabled: true, + MaxStepCost: evaluator.DefaultMaxStepCost, + MaxStepLatencyMs: evaluator.DefaultMaxStepLatencyMs, + MaxRunCost: evaluator.DefaultMaxRunCost, + MaxRunLatencyMs: evaluator.DefaultMaxRunLatencyMs, + }, + NodeTransitions: NodeTransitions{Enabled: true}, + LLMJudge: LLMJudge{Enabled: false, Model: evaluator.DefaultJudgeModel}, + }, + } +} + +// Load reads a policy file, layering it over Default so an omitted field keeps +// its default (partial configs are valid). Unknown fields are rejected, so a +// typo like "maxRepeat" is an error instead of being silently ignored. The file +// must declare version == SupportedVersion. +func Load(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + cfg := Default() + // Clear Version so we can tell whether the file actually declared it: an + // omitted version leaves it 0 and fails the check below. + cfg.Version = 0 + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&cfg); err != nil { + return nil, fmt.Errorf("config %s: %w", path, err) + } + + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("config %s: %w", path, err) + } + return &cfg, nil +} + +func (c *Config) validate() error { + if c.Version != SupportedVersion { + return fmt.Errorf("version %d is unsupported (this build accepts version %d)", c.Version, SupportedVersion) + } + if c.Evaluators.Loops.MaxRepeats < 0 { + return fmt.Errorf("loops.maxRepeats is negative (%d)", c.Evaluators.Loops.MaxRepeats) + } + cl := c.Evaluators.CostLatency + if cl.MaxStepCost < 0 || cl.MaxRunCost < 0 { + return fmt.Errorf("cost_latency costs must be non-negative") + } + if cl.MaxStepLatencyMs < 0 || cl.MaxRunLatencyMs < 0 { + return fmt.Errorf("cost_latency latencies must be non-negative") + } + for i, n := range c.Evaluators.NodeTransitions.TerminalNodes { + if n == "" { + return fmt.Errorf("node_transitions.terminalNodes[%d] is empty", i) + } + } + return nil +} + +// Build turns the policy into the evaluator set to run. The LLM judge is +// constructed via newJudge only when enabled; newJudge may fail (missing API +// key), so the caller supplies it and handles that error. +func (c *Config) Build(newJudge func(model string) (evaluator.Evaluator, error)) ([]evaluator.Evaluator, error) { + var evals []evaluator.Evaluator + e := c.Evaluators + + if e.ToolCalls.Enabled { + evals = append(evals, &evaluator.ToolCallEvaluator{}) + } + if e.Loops.Enabled { + evals = append(evals, &evaluator.LoopEvaluator{MaxRepeats: e.Loops.MaxRepeats}) + } + if e.CostLatency.Enabled { + evals = append(evals, &evaluator.CostLatencyEvaluator{ + MaxStepCost: e.CostLatency.MaxStepCost, + MaxStepLatencyMs: e.CostLatency.MaxStepLatencyMs, + MaxRunCost: e.CostLatency.MaxRunCost, + MaxRunLatencyMs: e.CostLatency.MaxRunLatencyMs, + }) + } + if e.NodeTransitions.Enabled { + evals = append(evals, &evaluator.NodeTransitionEvaluator{TerminalNodes: e.NodeTransitions.TerminalNodes}) + } + if e.LLMJudge.Enabled { + judge, err := newJudge(e.LLMJudge.Model) + if err != nil { + return nil, err + } + evals = append(evals, judge) + } + return evals, nil +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..df4a1b5 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,141 @@ +package config + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Cro22/trazo/evaluator" + "github.com/Cro22/trazo/trajectory" +) + +func writeConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "trazo.config.json") + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +// stubEvaluator stands in for a real evaluator so Build can be tested without a +// network-backed judge. +type stubEvaluator struct{} + +func (stubEvaluator) EvaluateRun(context.Context, *trajectory.Run) (*evaluator.Evaluation, error) { + return &evaluator.Evaluation{}, nil +} + +func noJudge(string) (evaluator.Evaluator, error) { + return nil, errors.New("newJudge should not be called when the judge is disabled") +} + +func TestDefault_BuildsFourEvaluators(t *testing.T) { + cfg := Default() + evals, err := cfg.Build(noJudge) + if err != nil { + t.Fatalf("Build: %v", err) + } + if len(evals) != 4 { + t.Fatalf("default policy should build 4 evaluators (judge off), got %d", len(evals)) + } +} + +func TestLoad_PartialLayersOverDefaults(t *testing.T) { + // Only override one nested field; everything else must keep its default. + path := writeConfig(t, `{"version": 1, "evaluators": {"loops": {"maxRepeats": 9}}}`) + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Evaluators.Loops.MaxRepeats != 9 { + t.Errorf("maxRepeats override lost: got %d", cfg.Evaluators.Loops.MaxRepeats) + } + if !cfg.Evaluators.Loops.Enabled { + t.Error("omitted loops.enabled should keep the default (true)") + } + if cfg.Evaluators.CostLatency.MaxStepCost != evaluator.DefaultMaxStepCost { + t.Errorf("omitted cost_latency should keep defaults, got %v", cfg.Evaluators.CostLatency.MaxStepCost) + } +} + +func TestLoad_DisableEvaluator(t *testing.T) { + path := writeConfig(t, `{"version": 1, "evaluators": {"cost_latency": {"enabled": false}, "loops": {"enabled": false}}}`) + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + evals, err := cfg.Build(noJudge) + if err != nil { + t.Fatalf("Build: %v", err) + } + // tool_calls and node_transitions remain enabled by default => 2. + if len(evals) != 2 { + t.Fatalf("expected 2 evaluators after disabling two, got %d", len(evals)) + } +} + +func TestLoad_RejectsUnknownField(t *testing.T) { + path := writeConfig(t, `{"version": 1, "evaluators": {"loops": {"maxRepeat": 3}}}`) + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("expected unknown-field error, got: %v", err) + } +} + +func TestLoad_RejectsBadVersion(t *testing.T) { + for _, body := range []string{ + `{"evaluators": {}}`, // missing version + `{"version": 2, "evaluators": {}}`, // unsupported version + } { + path := writeConfig(t, body) + if _, err := Load(path); err == nil { + t.Errorf("expected version error for %s", body) + } + } +} + +func TestLoad_RejectsNegativeThresholds(t *testing.T) { + path := writeConfig(t, `{"version": 1, "evaluators": {"loops": {"maxRepeats": -1}}}`) + if _, err := Load(path); err == nil { + t.Fatal("expected negative-threshold error") + } +} + +// TestExampleConfigLoads keeps the shipped example policy honest: it must always +// load and build under the current loader, so the docs never drift from the code. +func TestExampleConfigLoads(t *testing.T) { + cfg, err := Load(filepath.Join("..", "docs", "trazo.config.example.json")) + if err != nil { + t.Fatalf("example config failed to load: %v", err) + } + if _, err := cfg.Build(noJudge); err != nil { + t.Fatalf("example config failed to build: %v", err) + } +} + +func TestBuild_JudgeEnabledCallsFactory(t *testing.T) { + cfg := Default() + cfg.Evaluators.LLMJudge.Enabled = true + + called := false + evals, err := cfg.Build(func(model string) (evaluator.Evaluator, error) { + called = true + if model != evaluator.DefaultJudgeModel { + t.Errorf("expected default judge model, got %q", model) + } + return stubEvaluator{}, nil + }) + if err != nil { + t.Fatalf("Build: %v", err) + } + if !called { + t.Error("judge factory was not called though the judge is enabled") + } + if len(evals) != 5 { + t.Fatalf("expected 5 evaluators with judge on, got %d", len(evals)) + } +} diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000..8cb5822 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,74 @@ +# Evaluator policy file (`-config`) + +Passing `-config ` loads a JSON policy that pins which evaluators run and +with what thresholds. Commit it to a repo and every developer, CI job, and +teammate evaluates traces the same way, instead of remembering a string of flags. + +The format is stdlib-only JSON (no new dependency) and is versioned in its own +right. A full example is [trazo.config.example.json](trazo.config.example.json). + +```json +{ + "version": 1, + "evaluators": { + "tool_calls": { "enabled": true }, + "loops": { "enabled": true, "maxRepeats": 3 }, + "cost_latency": { + "enabled": true, + "maxStepCost": 0.05, + "maxStepLatencyMs": 30000, + "maxRunCost": 0.2, + "maxRunLatencyMs": 120000 + }, + "node_transitions": { + "enabled": true, + "terminalNodes": ["end", "__end__", "finish", "done"] + }, + "llm_judge": { "enabled": false, "model": "gemini-2.5-flash" } + } +} +``` + +## Rules + +- `version` is required and must be `1` (the format version this build accepts). +- Every field is optional beyond `version`: an omitted field keeps its built-in + default, so a partial config is valid. `{"version": 1, "evaluators": {}}` is the + default policy. +- Unknown fields are rejected. A typo like `maxRepeat` is an error, not a silent + no-op. +- Thresholds must be non-negative; `terminalNodes` entries must be non-empty. +- Omitting `node_transitions.terminalNodes` falls back to the evaluator's default + terminal set. + +## Precedence + +From lowest to highest: + +1. Built-in defaults. +2. The `-config` file. +3. Explicitly-set command-line flags. + +A flag only overrides the config when you actually pass it, so the config pins a +reproducible baseline while a one-off flag (`-max-step-cost 0.01`) still wins for +a single run. Example: + +```bash +# Reproducible team policy, committed to the repo: +trazo -config trazo.config.json ./traces + +# Same policy, but tighten one threshold for this run only: +trazo -config trazo.config.json -max-step-cost 0.01 ./traces +``` + +## Using it as a library + +The policy is a normal Go package, independent of the CLI: + +```go +cfg, err := config.Load("trazo.config.json") +evals, err := cfg.Build(func(model string) (evaluator.Evaluator, error) { + // construct your judge, or return an error to disable it +}) +resp := runner.NewRunner(evals).RunFiles(ctx, files) +``` diff --git a/docs/llm-judge.md b/docs/llm-judge.md new file mode 100644 index 0000000..0e9c12e --- /dev/null +++ b/docs/llm-judge.md @@ -0,0 +1,98 @@ +# LLM-as-judge evaluator (`-llm-judge`) + +Most trazo evaluators are structural: they reason about the shape of a trace +(orphan tool calls, loops, cost, latency) without ever reading the semantics of +what the agent produced. The LLM judge is the one exception. It asks a model to +grade the agent's final answer for correctness and usefulness, turning a +subjective "was this a good run?" into a `good` / `neutral` / `bad` judgment +alongside the structural findings. + +Because it makes a network call and costs money, it is **opt-in** and is not part +of the default evaluator set. + +## Enabling it + +```sh +export GEMINI_API_KEY=... # never commit this +go run ./cmd/trazo -llm-judge testdata/complex_run.json +``` + +Flags: + +| Flag | Default | Meaning | +| --- | --- | --- | +| `-llm-judge` | off | enable the judge (requires `GEMINI_API_KEY`) | +| `-judge-model` | `gemini-2.5-flash` | model passed to the Gemini API | + +The same two settings exist in the config file under `evaluators.llm_judge` +(`enabled`, `model`), so a policy file can pin the judge on for a whole team. As +everywhere else, an explicit `-llm-judge` flag overrides the config value. + +## What it judges, and how + +The judge looks at the run's **last `llm_call` output only**, once per run. If a +trace has no `llm_call` step there is nothing to grade and the evaluator returns +no findings. It does not re-grade intermediate reasoning or tool output; the +final answer is the thing a user sees, so it is the thing that gets graded. + +The prompt asks the model to reply with a single JSON object: + +```json +{"judgment": "good|neutral|bad", "score": 0.0-1.0, "comment": "short reason"} +``` + +- `good` maps to `JudgmentGood`, `bad` to `JudgmentBad`, `neutral` to + `JudgmentNeutral` (trazo's own taxonomy). +- Surrounding prose or code fences are tolerated: the parser extracts the first + `{ ... }` span from the reply. + +## Determinism and cost discipline + +- **Temperature 0.** The request pins `temperature: 0`, so for a given model and + input the verdict is as stable as the provider allows. It is still an LLM, so + treat verdicts as a strong signal, not a hard oracle. +- **Capped output.** `maxOutputTokens: 256`. The verdict is tiny; there is no + reason to pay for more. +- **One call per run.** The judge never loops or retries internally. One trace is + one completion. Cost scales linearly with the number of runs you judge, not + with trace size. + +## Timeouts, cancellation, and retries + +- **Timeout.** Each judge call is bounded by a timeout (default **30s**, + overridable on the evaluator). The bound is layered on top of the caller's + context, so a run cancelled with Ctrl+C or by a CI timeout also aborts the + in-flight network call promptly. +- **No retries.** A failed call is not retried. This is deliberate: retries hide + provider instability and multiply cost. If you need retry semantics, wrap the + provider at the HTTP layer. + +## What happens when the judge fails + +The two failure modes are treated differently on purpose. + +1. **A malformed but returned verdict** (the model replied, but the JSON is + missing, unparseable, or carries an unknown judgment word) becomes a single + `JudgmentNeutral` finding whose comment quotes the offending reply. The run is + still evaluated; you just get "the judge could not make up its mind" instead + of a grade. + +2. **A transport-level failure** (no API key, network error, non-200 status, + timeout, empty candidate list) is surfaced as a run **error**, not a finding. + In the CLI's JSON output it appears in `errors[]` with `kind: "evaluator"`; + in text output it is reported as a failed evaluation. Crucially, the failure + is isolated to the judge: the structural evaluators for that same file still + run and still produce their findings. A judge outage degrades the report, it + does not abort it. + +So: an ambiguous answer is neutral; an unreachable provider is an error. Neither +one silently passes a bad run. + +## Testing without a network + +The judge depends only on a small `Completion` interface +(`Complete(ctx, prompt) (string, error)`), so tests inject a fake instead of +calling Gemini. See `evaluator/judge_test.go`: `fakeJudge` returns a canned reply +(or a canned error) and lets the tests exercise every branch above, including the +Gemini HTTP client against an `httptest` server. Running `go test ./evaluator/` +never touches the network and needs no API key. diff --git a/docs/output.md b/docs/output.md new file mode 100644 index 0000000..86f19a0 --- /dev/null +++ b/docs/output.md @@ -0,0 +1,95 @@ +# Machine-readable output (`-format json`) + +`trazo -format json` (or `-json`) prints a single self-describing envelope meant +for CI and downstream tools. Its shape is a contract with its own version, +independent of the product and trace-schema versions. + +## Envelope + +```json +{ + "outputVersion": "1.1", + "trazoVersion": "0.2.0", + "traceSchemaVersion": "0.1.0", + "generatedAt": "2026-08-07T12:00:00Z", + "summary": { + "files": 4, + "runs": 2, + "evaluations": 3, + "findings": 3, + "good": 0, + "neutral": 2, + "bad": 1, + "fileErrors": 1 + }, + "results": [ + { + "evaluatorName": "tool_calls", + "runId": "run-1", + "findings": [ + { "stepIndex": 3, "judgment": "bad", "comment": "..." } + ] + } + ], + "errors": [ + { "file": "broken.json", "kind": "invalid_json", "error": "unexpected end of JSON input" } + ] +} +``` + +## Fields + +| Field | Type | Notes | +|-------|------|-------| +| `outputVersion` | string | Version of this envelope contract. See [Stability](#stability). | +| `trazoVersion` | string | Product version that produced the output. | +| `traceSchemaVersion` | string | Trace schema version this build supports. | +| `generatedAt` | RFC3339 string | UTC generation time. | +| `summary` | object | Aggregate counts, see below. | +| `results` | array | One entry per (evaluator, run); `[]` when there is nothing to report. | +| `errors` | array | One entry per file that failed to read, parse, or validate; `[]` when none. | + +`summary`: + +| Field | Meaning | +|-------|---------| +| `files` | Trace files considered. | +| `runs` | Distinct runs evaluated. | +| `evaluations` | Entries in `results` (evaluator x run). | +| `findings` | Total findings across all results. | +| `good` / `neutral` / `bad` | Findings by severity. | +| `fileErrors` | Entries in `errors`. | + +Each `results[i]` object: `evaluatorName` (string), `runId` (string), `findings` +(array). Each finding: `stepIndex` (int; `-1` is run-level), `judgment` (`good` | +`neutral` | `bad`), `comment` (string), and `score` (number, omitted when zero). + +Each `errors[i]` object: `file` (path as given to trazo), `kind` (category, see +below), and `error` (message). `kind` lets a consumer react by category instead +of matching message strings: + +| `kind` | Meaning | +|--------|---------| +| `read_file` | The file could not be read. | +| `invalid_json` | The bytes are not valid JSON. | +| `invalid_trace` | JSON parsed but failed structural validation (`Run.Validate`). | +| `evaluator` | An evaluator returned an error. | +| `canceled` | The context was canceled (Ctrl+C, timeout) before processing. | + +## Stability + +`outputVersion` follows semver-like rules for the envelope: + +- Additive, backward-compatible changes (a new field) bump the **minor**. +- Renaming or removing a field, or changing a field's meaning, bumps the + **major**. + +Consumers should ignore unknown fields so a minor bump does not break them. +Empty collections are always `[]`, never `null`, so indexing is safe without a +nil check. See [versioning.md](versioning.md) for how this relates to the product +and trace-schema versions. + +## Exit codes + +The envelope is independent of the process exit code, which CI can gate on +directly: `0` clean, `1` at least one `bad` finding, `2` at least one file error. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..a7cf038 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,86 @@ +# Privacy and data handling + +Trazo evaluates agent traces. Those traces are a faithful recording of what an +agent did, which means they can contain whatever the agent read, wrote, or was +told. This document describes exactly what a trace holds, where that data can +leave your machine, and how to keep private data private. + +## What a trace file contains + +A trace is a JSON `Run` with a list of `Step`s. The fields that can carry +sensitive content are: + +| Field | Present on | Can contain | +| --- | --- | --- | +| `input` | any step | the LLM prompt, tool arguments, user-supplied text | +| `output` | any step | the LLM completion, tool results, fetched documents | +| `error` | any step | error strings, which sometimes echo inputs or paths | +| `agent`, `llm`, `tool`, `node` | identifiers | model names, internal tool and node names | + +`input` and `output` are opaque payloads (`json.RawMessage`): trazo does not +inspect or constrain their contents, so anything the agent handled can end up +there verbatim. Prompts, retrieved passages, API responses, file contents, PII +in a user request: if the agent saw it, the trace can hold it. + +The remaining fields (`cost`, `inputTokens`, `outputTokens`, `durationMs`, +timestamps, IDs, step `type`) are operational metadata and are not sensitive on +their own. + +## Where data goes + +By default, trazo is **fully local**. The core loads trace files from disk, runs +the structural evaluators (tool calls, loops, cost, latency, nodes) entirely +in-process, and writes findings to stdout. No trace data leaves the machine. The +core is standard-library only and opens no network connections in this mode. + +There is exactly **one** component that sends trace data off the machine: the +[LLM judge](llm-judge.md). When you pass `-llm-judge`, the agent's final +`llm_call` **output** is placed in a prompt and sent to the Gemini API for +grading. That output text leaves your machine and is subject to the provider's +data-handling terms. + +Nothing else is transmitted: not tool inputs, not intermediate steps, not the +whole trace. Only the final answer, and only when the judge is explicitly +enabled. + +## How to disable network egress + +- **Do not pass `-llm-judge`** (and leave `evaluators.llm_judge.enabled` false or + absent in any config file). This is the default. With the judge off, trazo + makes no outbound calls at all. +- If you want a hard guarantee, run trazo on a host with no network access, or + omit `GEMINI_API_KEY` from the environment. Without the key the judge cannot be + constructed and the run fails fast rather than sending anything. + +## Redaction + +Trazo does not redact for you; it evaluates whatever it is given. Redaction +belongs at trace-creation time, in whatever emits the trace (for the reference +agent, that is the Python `trazo_emitter`). Strip or mask secrets, credentials, +and PII from `input`, `output`, and `error` payloads before writing the file. +Because payloads are opaque to the core, a redacted trace evaluates exactly like +an unredacted one; the structural evaluators care about shape, not content. + +If you enable the judge, remember that redaction of the final output directly +changes what is sent to Gemini. A masked answer is a masked prompt. + +## Do not commit private traces + +Real agent runs make excellent test fixtures, which is precisely why they leak. +Treat trace files from real workloads as you would logs or a database dump: + +- Keep them out of the repository. Add your traces directory to `.gitignore`. +- The trace fixtures that **are** committed under `testdata/` and + `agents/langgraph-reference/docs/` are hand-authored or produced against public + repositories with synthetic content. Keep it that way: do not replace them with + captures from private runs. +- Before sharing a trace for a bug report, open it and check `input` / `output` / + `error` on every step. A trace is human-readable JSON; there is no excuse for + not looking. + +## API keys + +`GEMINI_API_KEY` is read from the environment and is the only secret trazo +consumes. It is never written to a trace, never logged, and never printed. Never +hardcode it, never commit it, and never paste it into a config file (the config +schema has no field for it, by design). diff --git a/docs/trazo.config.example.json b/docs/trazo.config.example.json new file mode 100644 index 0000000..0e948e2 --- /dev/null +++ b/docs/trazo.config.example.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "evaluators": { + "tool_calls": { + "enabled": true + }, + "loops": { + "enabled": true, + "maxRepeats": 3 + }, + "cost_latency": { + "enabled": true, + "maxStepCost": 0.05, + "maxStepLatencyMs": 30000, + "maxRunCost": 0.2, + "maxRunLatencyMs": 120000 + }, + "node_transitions": { + "enabled": true, + "terminalNodes": ["end", "__end__", "finish", "done"] + }, + "llm_judge": { + "enabled": false, + "model": "gemini-2.5-flash" + } + } +} diff --git a/docs/versioning.md b/docs/versioning.md new file mode 100644 index 0000000..b2e823c --- /dev/null +++ b/docs/versioning.md @@ -0,0 +1,82 @@ +# Versioning and compatibility + +Trazo carries two independent version numbers. Keeping them separate is +deliberate: the tool and the data format evolve at different rates. + +| Version | What it tracks | Where it lives | +|---------|----------------|----------------| +| Product version | The CLI and the Go library (behavior, flags, output shape) | `main.Version`, reported by `trazo version` | +| Trace schema version | The on-disk trace JSON format | `trajectory.SchemaVersion`, stamped in each trace's `version` field | + +Both follow [Semantic Versioning](https://semver.org/): `MAJOR.MINOR.PATCH`. + +## Trace schema compatibility + +Trazo accepts a trace whose **major** schema version matches the binary's +supported major. Within that major, minor and patch differences are compatible. + +- **Patch** (`0.1.0` -> `0.1.1`): clarifications or fixes with no field changes. +- **Minor** (`0.1.0` -> `0.2.0`): backward-compatible additions, typically a new + optional field. Older traces still validate; newer traces still load on an + older binary of the same major, because unknown fields are ignored by the Go + loader. (Adding the optional `toolCallId` field was such a minor bump.) +- **Major** (`0.x` -> `1.0.0`): a breaking change: a removed or renamed field, or + a changed meaning. Traces across a major boundary are **rejected**. + +Current supported schema major: **0**. +Current schema version: **0.1.0**. + +### What happens to an incompatible trace + +The gate is `trajectory.checkVersion`, run as part of `Run.Validate`: + +- Missing or non-semver `version` -> rejected as an invalid trace. +- A `2.0.0` trace on a major-0 binary -> rejected with a message naming the + supported version, for example: + + ``` + run: schema version "2.0.0" (major 2) is unsupported; this build accepts major 0 (0.1.0) + ``` + +Rejected traces surface as file errors (CLI exit code 2); they do not abort the +rest of a batch. + +### Migrating older traces + +Within a major there is nothing to migrate: an older minor validates as-is. A +future major bump will ship with either a documented migration for the changed +fields or an explicit decision to drop support for the old major, recorded in the +[changelog](../CHANGELOG.md). We do not silently coerce across a major. + +## Product versioning + +The product version bumps on user-visible changes to the CLI or library: + +- **Patch**: bug fixes, no interface change. +- **Minor**: backward-compatible features (a new flag, a new evaluator, an + additive field in the JSON output). +- **Major**: breaking changes to flags, exit codes, the library API, or the + machine-readable output contract. + +The machine-readable JSON output is a contract in its own right and carries its +own `outputVersion` field; see [output.md](output.md). + +## Release process + +1. Move the `[Unreleased]` entries in [CHANGELOG.md](../CHANGELOG.md) under a new + `[X.Y.Z]` heading with the date. +2. Bump `main.Version` (and `trajectory.SchemaVersion` plus `supportedMajor` if + the trace format changed). +3. Tag the commit `vX.Y.Z`. The build embeds the commit automatically, so + `trazo version` reports the tagged revision. +4. `go build ./... && go test ./...` must be green, and the Python suite too. + +`trazo version` prints all of this at a glance: + +``` +trazo 0.2.0 +trace schema 0.1.0 +commit 891c879de1ad +built 2026-08-07T21:01:56Z +go1.25.0 +``` diff --git a/evaluator/integration_test.go b/evaluator/integration_test.go new file mode 100644 index 0000000..5ec003b --- /dev/null +++ b/evaluator/integration_test.go @@ -0,0 +1,74 @@ +package evaluator + +import ( + "context" + "os" + "testing" + + "github.com/Cro22/trazo/trajectory" +) + +// TestComplexRun_AllEvaluators exercises a single realistic trace against the +// full structural evaluator set: interleaved tool calls paired by id (no +// orphans), a tool error, a high-cost step, and a node visited enough times to +// look like a loop. It documents how the evaluators divide the work on one trace. +func TestComplexRun_AllEvaluators(t *testing.T) { + data, err := os.ReadFile("../testdata/complex_run.json") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + run, err := trajectory.LoadRun(data) + if err != nil { + t.Fatalf("load run: %v", err) + } + if err := run.Validate(); err != nil { + t.Fatalf("fixture must be a valid trace: %v", err) + } + + ctx := context.Background() + + // tool_calls: exactly one bad (the fetch 429), and no orphans, because the + // interleaved search/fetch calls each pair with their result by id. + tc, _ := (&ToolCallEvaluator{}).EvaluateRun(ctx, run) + if got := countBy(tc, JudgmentBad); got != 1 { + t.Errorf("tool_calls: expected 1 bad, got %d (%+v)", got, tc.Findings) + } + if got := countBy(tc, JudgmentNeutral); got != 0 { + t.Errorf("tool_calls: expected 0 neutral (no orphans), got %d (%+v)", got, tc.Findings) + } + + // loops: the node "retry" is visited three times, hitting the default limit. + lp, _ := (&LoopEvaluator{}).EvaluateRun(ctx, run) + if got := countBy(lp, JudgmentBad); got != 1 { + t.Errorf("loops: expected 1 bad, got %d (%+v)", got, lp.Findings) + } + + // cost_latency: the llm step costs 0.10, over the default per-step budget. A + // cost overrun is neutral, not bad: it needs review but is not necessarily the + // agent's fault. + cl, _ := (&CostLatencyEvaluator{ + MaxStepCost: DefaultMaxStepCost, + MaxStepLatencyMs: DefaultMaxStepLatencyMs, + MaxRunCost: DefaultMaxRunCost, + MaxRunLatencyMs: DefaultMaxRunLatencyMs, + }).EvaluateRun(ctx, run) + if countBy(cl, JudgmentNeutral) < 1 { + t.Errorf("cost_latency: expected at least 1 neutral, got %+v", cl.Findings) + } + + // node_transitions: the run ends at the terminal node "end", so nothing here. + nt, _ := (&NodeTransitionEvaluator{}).EvaluateRun(ctx, run) + if len(nt.Findings) != 0 { + t.Errorf("node_transitions: expected 0 findings (ends at terminal), got %+v", nt.Findings) + } +} + +func countBy(e *Evaluation, j Judgment) int { + n := 0 + for _, f := range e.Findings { + if f.Judgment == j { + n++ + } + } + return n +} diff --git a/report/golden_test.go b/report/golden_test.go index 2cc9a44..3a96f76 100644 --- a/report/golden_test.go +++ b/report/golden_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/Cro22/trazo/evaluator" "github.com/Cro22/trazo/runner" @@ -51,7 +52,7 @@ func goldenFixture() *runner.Response { }, }, FileErrors: []runner.FileError{ - {File: "broken.json", Err: errors.New("unexpected end of JSON input")}, + {File: "broken.json", Kind: runner.ErrorKindInvalidJSON, Err: errors.New("unexpected end of JSON input")}, }, } } @@ -59,7 +60,13 @@ func goldenFixture() *runner.Response { func TestGolden(t *testing.T) { resp := goldenFixture() - jsonOut, err := JSON(resp) + meta := Meta{ + TrazoVersion: "0.2.0", + TraceSchemaVersion: "0.1.0", + Files: 4, + GeneratedAt: time.Date(2026, 8, 7, 12, 0, 0, 0, time.UTC), + } + jsonOut, err := JSON(resp, meta) if err != nil { t.Fatalf("JSON: %v", err) } diff --git a/report/json.go b/report/json.go index af3c306..e0360c9 100644 --- a/report/json.go +++ b/report/json.go @@ -2,37 +2,66 @@ package report import ( "encoding/json" + "time" "github.com/Cro22/trazo/evaluator" "github.com/Cro22/trazo/runner" ) +// OutputVersion is the version of the machine-readable JSON output contract (the +// envelope shape below), independent of the product and trace-schema versions. +// Bump the minor for additive fields, the major for a breaking change. See +// docs/output.md. +const OutputVersion = "1.1" + +// Meta is the context the caller supplies for the JSON envelope: the versions to +// stamp, the number of files considered, and the generation time. GeneratedAt is +// injected (not read from the clock here) so output is deterministic in tests. +type Meta struct { + TrazoVersion string + TraceSchemaVersion string + Files int + GeneratedAt time.Time +} + type fileErrorJSON struct { File string `json:"file"` + Kind string `json:"kind"` Error string `json:"error"` } -type responseJSON struct { - Evaluations []*evaluator.Evaluation `json:"evaluations"` - FileErrors []fileErrorJSON `json:"fileErrors"` +type jsonEnvelope struct { + OutputVersion string `json:"outputVersion"` + TrazoVersion string `json:"trazoVersion"` + TraceSchemaVersion string `json:"traceSchemaVersion"` + GeneratedAt string `json:"generatedAt"` + Summary Summary `json:"summary"` + Results []*evaluator.Evaluation `json:"results"` + Errors []fileErrorJSON `json:"errors"` } -// JSON renders the evaluation results as indented JSON. Empty slices serialize as -// [] rather than null so consumers can index without a nil check, and errors are -// flattened to strings. -func JSON(resp *runner.Response) (string, error) { - out := responseJSON{ - Evaluations: resp.Evaluations, - FileErrors: []fileErrorJSON{}, +// JSON renders the results as a versioned, self-describing envelope: metadata, an +// aggregate summary, the per-run evaluations, and structured file errors. Empty +// slices serialize as [] (never null) so consumers can index without a nil +// check. The shape is a stable contract; see docs/output.md. +func JSON(resp *runner.Response, meta Meta) (string, error) { + env := jsonEnvelope{ + OutputVersion: OutputVersion, + TrazoVersion: meta.TrazoVersion, + TraceSchemaVersion: meta.TraceSchemaVersion, + GeneratedAt: meta.GeneratedAt.UTC().Format(time.RFC3339), + Summary: Summarize(resp, meta.Files), + Results: resp.Evaluations, + Errors: []fileErrorJSON{}, } - if out.Evaluations == nil { - out.Evaluations = []*evaluator.Evaluation{} + if env.Results == nil { + env.Results = []*evaluator.Evaluation{} } for _, fe := range resp.FileErrors { - out.FileErrors = append(out.FileErrors, fileErrorJSON{File: fe.File, Error: fe.Err.Error()}) + env.Errors = append(env.Errors, fileErrorJSON{File: fe.File, Kind: string(fe.Kind), Error: fe.Err.Error()}) } - data, err := json.MarshalIndent(out, "", " ") + data, err := json.MarshalIndent(env, "", " ") if err != nil { return "", err } diff --git a/report/summary.go b/report/summary.go new file mode 100644 index 0000000..7a89f43 --- /dev/null +++ b/report/summary.go @@ -0,0 +1,49 @@ +package report + +import ( + "github.com/Cro22/trazo/evaluator" + "github.com/Cro22/trazo/runner" +) + +// Summary is the aggregate outcome of a run batch, shared by the text footer and +// the JSON envelope so the two never disagree. +type Summary struct { + Files int `json:"files"` + Runs int `json:"runs"` + Evaluations int `json:"evaluations"` + Findings int `json:"findings"` + Good int `json:"good"` + Neutral int `json:"neutral"` + Bad int `json:"bad"` + FileErrors int `json:"fileErrors"` +} + +// Summarize aggregates a Response. files is the number of trace files considered +// (which the Response alone does not carry, since valid files with no findings +// still count); pass 0 when it is not known. +func Summarize(resp *runner.Response, files int) Summary { + s := Summary{ + Files: files, + Evaluations: len(resp.Evaluations), + FileErrors: len(resp.FileErrors), + } + seen := map[string]bool{} + for _, e := range resp.Evaluations { + if !seen[e.RunID] { + seen[e.RunID] = true + s.Runs++ + } + for _, f := range e.Findings { + s.Findings++ + switch f.Judgment { + case evaluator.JudgmentBad: + s.Bad++ + case evaluator.JudgmentNeutral: + s.Neutral++ + case evaluator.JudgmentGood: + s.Good++ + } + } + } + return s +} diff --git a/report/testdata/json.golden b/report/testdata/json.golden index a064bf2..e32f601 100644 --- a/report/testdata/json.golden +++ b/report/testdata/json.golden @@ -1,5 +1,19 @@ { - "evaluations": [ + "outputVersion": "1.1", + "trazoVersion": "0.2.0", + "traceSchemaVersion": "0.1.0", + "generatedAt": "2026-08-07T12:00:00Z", + "summary": { + "files": 4, + "runs": 2, + "evaluations": 3, + "findings": 3, + "good": 0, + "neutral": 2, + "bad": 1, + "fileErrors": 1 + }, + "results": [ { "evaluatorName": "tool_calls", "runId": "run-1", @@ -34,9 +48,10 @@ "findings": [] } ], - "fileErrors": [ + "errors": [ { "file": "broken.json", + "kind": "invalid_json", "error": "unexpected end of JSON input" } ] diff --git a/report/testdata/text.golden b/report/testdata/text.golden index 2ef862c..dc6a153 100644 --- a/report/testdata/text.golden +++ b/report/testdata/text.golden @@ -8,4 +8,4 @@ run-ok (data_extractor) clean Summary: 2 runs, 1 bad, 2 neutral, 0 good, 1 file error File errors: - broken.json: unexpected end of JSON input + broken.json [invalid_json]: unexpected end of JSON input diff --git a/report/text.go b/report/text.go index e8f4987..37af7d2 100644 --- a/report/text.go +++ b/report/text.go @@ -34,11 +34,11 @@ func Text(resp *runner.Response) string { b.WriteString("\n") } - b.WriteString(summaryLine(groups, order, len(resp.FileErrors))) + b.WriteString(summaryLine(resp)) if len(resp.FileErrors) > 0 { b.WriteString("\nFile errors:\n") for _, fe := range resp.FileErrors { - fmt.Fprintf(&b, " %s: %s\n", fe.File, oneline(fe.Err.Error())) + b.WriteString(fileErrorLine(fe)) } } return b.String() @@ -128,22 +128,10 @@ func stepText(idx int) string { return fmt.Sprintf("step %d", idx) } -func summaryLine(groups map[string]*runGroup, order []string, fileErrors int) string { - var bad, neutral, good int - for _, runID := range order { - for _, r := range groups[runID].rows { - switch r.severity { - case "[BAD]": - bad++ - case "[NEUTRAL]": - neutral++ - case "[GOOD]": - good++ - } - } - } +func summaryLine(resp *runner.Response) string { + s := Summarize(resp, 0) return fmt.Sprintf("Summary: %s, %d bad, %d neutral, %d good, %s\n", - plural(len(order), "run"), bad, neutral, good, plural(fileErrors, "file error")) + plural(s.Runs, "run"), s.Bad, s.Neutral, s.Good, plural(s.FileErrors, "file error")) } // oneline flattens a possibly multi-line message onto a single line so table @@ -152,6 +140,16 @@ func oneline(s string) string { return strings.ReplaceAll(s, "\n", "; ") } +// fileErrorLine renders one file error, tagging it with its kind when known so +// the reader can tell a parse error from an invalid trace at a glance. +func fileErrorLine(fe runner.FileError) string { + msg := oneline(fe.Err.Error()) + if fe.Kind != "" { + return fmt.Sprintf(" %s [%s]: %s\n", fe.File, fe.Kind, msg) + } + return fmt.Sprintf(" %s: %s\n", fe.File, msg) +} + func plural(n int, unit string) string { if n == 1 { return fmt.Sprintf("%d %s", n, unit) diff --git a/report/validate.go b/report/validate.go index ba44479..49709a7 100644 --- a/report/validate.go +++ b/report/validate.go @@ -22,7 +22,7 @@ func ValidateSummary(resp *runner.Response, total int) string { var b strings.Builder fmt.Fprintf(&b, "Validated %d file(s): %d valid, %d invalid.\n", total, valid, invalid) for _, fe := range resp.FileErrors { - fmt.Fprintf(&b, " %s: %s\n", fe.File, cell(fe.Err.Error())) + b.WriteString(fileErrorLine(fe)) } return b.String() } diff --git a/runner/runner.go b/runner/runner.go index d490d98..4053b57 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -16,8 +16,22 @@ type Runner struct { evals []evaluator.Evaluator } +// ErrorKind classifies why a file could not be evaluated, so callers (the CLI, +// the JSON output, CI) can react by category instead of matching on message +// strings. +type ErrorKind string + +const ( + ErrorKindReadFile ErrorKind = "read_file" // the file could not be read + ErrorKindInvalidJSON ErrorKind = "invalid_json" // the bytes are not valid JSON + ErrorKindInvalidTrace ErrorKind = "invalid_trace" // JSON parsed but failed Run.Validate + ErrorKindEvaluator ErrorKind = "evaluator" // an evaluator returned an error + ErrorKindCanceled ErrorKind = "canceled" // the context was canceled before processing +) + type FileError struct { File string + Kind ErrorKind Err error } @@ -132,28 +146,28 @@ func (r *Runner) processFile(ctx context.Context, path string) fileResult { var res fileResult if err := ctx.Err(); err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindCanceled, Err: err}) return res } fileBytes, err := os.ReadFile(path) if err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindReadFile, Err: err}) return res } run, err := trajectory.LoadRun(fileBytes) if err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindInvalidJSON, Err: err}) return res } if err := run.Validate(); err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindInvalidTrace, Err: err}) return res } for _, judge := range r.evals { eval, err := judge.EvaluateRun(ctx, run) if err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindEvaluator, Err: err}) continue } // Carry the agent name for human-facing output; evaluators only set RunID. diff --git a/runner/runner_test.go b/runner/runner_test.go index 938a92b..5900d4a 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -37,9 +37,15 @@ func TestRunner_Run(t *testing.T) { if resp.FileErrors[0].File != wantBroken { t.Errorf("expected first file error on %s, got %s", wantBroken, resp.FileErrors[0].File) } + if resp.FileErrors[0].Kind != ErrorKindInvalidJSON { + t.Errorf("broken.json: expected kind %q, got %q", ErrorKindInvalidJSON, resp.FileErrors[0].Kind) + } if resp.FileErrors[1].File != wantInvalid { t.Errorf("expected second file error on %s, got %s", wantInvalid, resp.FileErrors[1].File) } + if resp.FileErrors[1].Kind != ErrorKindInvalidTrace { + t.Errorf("invalid_run.json: expected kind %q, got %q", ErrorKindInvalidTrace, resp.FileErrors[1].Kind) + } } func TestCollectFiles_NonRecursiveSkipsSubdirs(t *testing.T) { diff --git a/testdata/complex_run.json b/testdata/complex_run.json new file mode 100644 index 0000000..f21b860 --- /dev/null +++ b/testdata/complex_run.json @@ -0,0 +1,82 @@ +{ + "id": "run-complex-01", + "agent": "researcher", + "version": "0.1.0", + "startTime": "2026-08-07T09:00:00Z", + "endTime": "2026-08-07T09:00:12Z", + "steps": [ + { + "node": "start", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:00Z", + "durationMs": 3 + }, + { + "llm": "gemini-2.5-flash", + "type": "llm_call", + "timestamp": "2026-08-07T09:00:01Z", + "input": {"system": "plan the research", "user": "compare two libraries"}, + "output": "I will search and fetch in parallel.", + "cost": 0.10, + "inputTokens": 900, + "outputTokens": 120, + "durationMs": 1200 + }, + { + "tool": "search", + "toolCallId": "call-search-1", + "type": "tool_call", + "timestamp": "2026-08-07T09:00:02Z", + "input": {"q": "library A benchmarks"}, + "durationMs": 400 + }, + { + "tool": "fetch", + "toolCallId": "call-fetch-1", + "type": "tool_call", + "timestamp": "2026-08-07T09:00:03Z", + "input": {"url": "https://example.com/a"}, + "durationMs": 500 + }, + { + "tool": "fetch", + "toolCallId": "call-fetch-1", + "type": "tool_result", + "timestamp": "2026-08-07T09:00:04Z", + "durationMs": 500, + "error": "429 Too Many Requests" + }, + { + "tool": "search", + "toolCallId": "call-search-1", + "type": "tool_result", + "timestamp": "2026-08-07T09:00:05Z", + "output": {"hits": 3}, + "durationMs": 400 + }, + { + "node": "retry", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:06Z", + "durationMs": 2 + }, + { + "node": "retry", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:07Z", + "durationMs": 2 + }, + { + "node": "retry", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:08Z", + "durationMs": 2 + }, + { + "node": "end", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:10Z", + "durationMs": 3 + } + ] +}