From 1cf5b9efb5e6f8c1581a0ba954a4118501284cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 18:24:50 -0400 Subject: [PATCH 01/11] feat(cli): add version command and versioning policy (0.2.0 M1) Introduce the product versioning contract, distinct from the trace schema version: - `trazo version` subcommand and `-version` flag report the product version (main.Version = 0.2.0), the supported trace schema version, and the git commit/build time read from the Go toolchain's build info (no ldflags). - docs/versioning.md documents the product-vs-schema version split, the schema compatibility contract (accept same-major, reject cross-major, what a 2.0.0 trace does), migration stance, and the release process. - CHANGELOG.md (Keep a Changelog) records this round and the 0.1.0 hardening. - Link the schema doc's Versioning section to docs/versioning.md. versionReport is kept pure and unit-tested for field rendering, commit truncation, the -dirty suffix, and the unknown-commit fallback. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 49 +++++++++++ .../langgraph-reference/docs/trace-schema.md | 3 + cmd/trazo/main.go | 15 +++- cmd/trazo/version.go | 68 +++++++++++++++ cmd/trazo/version_test.go | 45 ++++++++++ docs/versioning.md | 82 +++++++++++++++++++ 6 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md create mode 100644 cmd/trazo/version.go create mode 100644 cmd/trazo/version_test.go create mode 100644 docs/versioning.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5bb0248 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,49 @@ +# Changelog + +All notable changes to trazo are documented here. The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project aims +to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html) for the +product version. The trace **schema** version is tracked separately; see +[docs/versioning.md](docs/versioning.md). + +## [Unreleased] + +### Added + +- `trazo version` subcommand and `-version` flag, reporting the product version, + the supported trace schema version, the git commit, and the build time. Commit + and build metadata come from the Go toolchain's build info (no ldflags needed). +- [docs/versioning.md](docs/versioning.md): the product and schema versioning + policy, the schema compatibility contract, and the release process. +- This changelog. + +## [0.1.0] + +The hardening round: formalize the trace contract and round the core out from a +working MVP toward production-grade. + +### Added + +- Formal JSON Schema (`trajectory/trace.schema.json`, draft 2020-12) as the + strict external contract, kept in sync with the Go types by a dependency-free + test and validated against fixtures and emitter output from Python. +- Trace schema versioning: `version` is required and semver-gated by major (see + docs/versioning.md). +- Explicit `toolCallId` for authoritative tool call/result pairing, with + name/FIFO as a documented fallback. +- CLI: single-file input, `-recursive`, `-validate` (structure-only), up-front + `-format` validation, and a proper `-help`. +- Golden tests for the text, JSON, and Markdown output formats. + +### Changed + +- `Evaluator` interface takes a `context.Context`, so Ctrl+C or a CI timeout + aborts in-flight work (notably the LLM judge). +- `Run.Validate` deepened: non-negative quantities, monotonic step timestamps, + steps within the run interval, per-type required fields. +- Text output regrouped by run with a summary footer. +- Output formatting centralized in the `report` package; `Evaluator.go` renamed + to `evaluator.go`. + +[Unreleased]: https://github.com/Cro22/trazo/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/Cro22/trazo/releases/tag/v0.1.0 diff --git a/agents/langgraph-reference/docs/trace-schema.md b/agents/langgraph-reference/docs/trace-schema.md index 6079121..edfef7e 100644 --- a/agents/langgraph-reference/docs/trace-schema.md +++ b/agents/langgraph-reference/docs/trace-schema.md @@ -148,6 +148,9 @@ stamps it on every trace by default. Keep the two constants in sync; a Go test (`trajectory/schema_test.go`) checks the JSON Schema requires `version` and that `SchemaVersion` passes the gate. +The full compatibility contract, the product-vs-schema version distinction, and +the release process live in [`docs/versioning.md`](../../../docs/versioning.md). + ## Tool call / result pairing (what the emitter must respect) `ToolCallEvaluator` (`evaluator/toolcalls.go`) walks the steps in order and pairs diff --git a/cmd/trazo/main.go b/cmd/trazo/main.go index 8a4c387..eaf17e0 100644 --- a/cmd/trazo/main.go +++ b/cmd/trazo/main.go @@ -27,6 +27,14 @@ func main() { log.SetFlags(0) log.SetPrefix("trazo: ") + // `trazo version` is a subcommand, handled before flag parsing so it works + // without any other arguments. + if len(os.Args) > 1 && os.Args[1] == "version" { + fmt.Print(versionReport(readBuildDetails())) + return + } + + showVersion := flag.Bool("version", false, "print version information and exit") dir := flag.String("dir", "./testdata/runs", "directory of traces to scan when no PATH is given") recursive := flag.Bool("recursive", false, "descend into subdirectories when PATH is a directory") validate := flag.Bool("validate", false, "only check that traces load and pass structural validation; skip evaluators") @@ -45,6 +53,11 @@ func main() { flag.Usage = usage flag.Parse() + if *showVersion { + fmt.Print(versionReport(readBuildDetails())) + return + } + if flag.NArg() > 1 { log.Printf("at most one PATH may be given, got %d", flag.NArg()) flag.Usage() @@ -98,7 +111,7 @@ func main() { func usage() { w := flag.CommandLine.Output() fmt.Fprintf(w, "trazo evaluates agent trace files (trazo JSON format) and reports findings.\n\n") - fmt.Fprintf(w, "Usage:\n trazo [flags] [PATH]\n\n") + fmt.Fprintf(w, "Usage:\n trazo [flags] [PATH]\n trazo version\n\n") fmt.Fprintf(w, "PATH is a single trace file or a directory of .json traces. If omitted, -dir is scanned.\n\n") fmt.Fprintf(w, "Flags:\n") flag.PrintDefaults() diff --git a/cmd/trazo/version.go b/cmd/trazo/version.go new file mode 100644 index 0000000..252cba2 --- /dev/null +++ b/cmd/trazo/version.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "runtime" + "runtime/debug" + "strings" + + "github.com/Cro22/trazo/trajectory" +) + +// Version is the trazo product (binary) version. It is distinct from the trace +// schema version (trajectory.SchemaVersion): the product version tracks the CLI +// and library, the schema version tracks the trace format. See docs/versioning.md +// for the policy on bumping each. +const Version = "0.2.0" + +// buildDetails is the VCS/build metadata embedded by the Go toolchain. It is +// populated from runtime/debug build info, which `go build` fills in +// automatically from the enclosing git repo (no ldflags needed); under `go run` +// or `go test` it may be absent, in which case commit stays "unknown". +type buildDetails struct { + commit string + modified bool + buildTime string + goVersion string +} + +func readBuildDetails() buildDetails { + d := buildDetails{commit: "unknown", goVersion: runtime.Version()} + info, ok := debug.ReadBuildInfo() + if !ok { + return d + } + for _, s := range info.Settings { + switch s.Key { + case "vcs.revision": + d.commit = s.Value + case "vcs.time": + d.buildTime = s.Value + case "vcs.modified": + d.modified = s.Value == "true" + } + } + return d +} + +// versionReport renders the multi-line output of `trazo version`. It is kept pure +// (details passed in) so it can be tested without depending on build metadata. +func versionReport(d buildDetails) string { + commit := d.commit + if commit != "unknown" && len(commit) > 12 { + commit = commit[:12] + } + if d.modified { + commit += "-dirty" + } + + var b strings.Builder + fmt.Fprintf(&b, "trazo %s\n", Version) + fmt.Fprintf(&b, "trace schema %s\n", trajectory.SchemaVersion) + fmt.Fprintf(&b, "commit %s\n", commit) + if d.buildTime != "" { + fmt.Fprintf(&b, "built %s\n", d.buildTime) + } + fmt.Fprintf(&b, "%s\n", d.goVersion) + return b.String() +} diff --git a/cmd/trazo/version_test.go b/cmd/trazo/version_test.go new file mode 100644 index 0000000..aa09bd2 --- /dev/null +++ b/cmd/trazo/version_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "strings" + "testing" + + "github.com/Cro22/trazo/trajectory" +) + +func TestVersionReport_Fields(t *testing.T) { + got := versionReport(buildDetails{ + commit: "abcdef1234567890", + buildTime: "2026-08-07T12:00:00Z", + goVersion: "go1.25.0", + }) + + wantLines := []string{ + "trazo " + Version, + "trace schema " + trajectory.SchemaVersion, + "commit abcdef123456", // truncated to 12 chars + "built 2026-08-07T12:00:00Z", + "go1.25.0", + } + for _, line := range wantLines { + if !strings.Contains(got, line) { + t.Errorf("version report missing %q, got:\n%s", line, got) + } + } +} + +func TestVersionReport_DirtyAndUnknown(t *testing.T) { + dirty := versionReport(buildDetails{commit: "deadbeefcafebabe", modified: true, goVersion: "go1.25.0"}) + if !strings.Contains(dirty, "commit deadbeefcafe-dirty") { + t.Errorf("expected truncated dirty commit, got:\n%s", dirty) + } + + unknown := versionReport(buildDetails{commit: "unknown", goVersion: "go1.25.0"}) + if !strings.Contains(unknown, "commit unknown\n") { + t.Errorf("expected literal unknown commit, got:\n%s", unknown) + } + // With no build time, the "built" line is omitted entirely. + if strings.Contains(unknown, "built ") { + t.Errorf("did not expect a built line without build time, got:\n%s", unknown) + } +} diff --git a/docs/versioning.md b/docs/versioning.md new file mode 100644 index 0000000..7a108a7 --- /dev/null +++ b/docs/versioning.md @@ -0,0 +1,82 @@ +# Versioning and compatibility + +Trazo carries two independent version numbers. Keeping them separate is +deliberate: the tool and the data format evolve at different rates. + +| Version | What it tracks | Where it lives | +|---------|----------------|----------------| +| Product version | The CLI and the Go library (behavior, flags, output shape) | `main.Version`, reported by `trazo version` | +| Trace schema version | The on-disk trace JSON format | `trajectory.SchemaVersion`, stamped in each trace's `version` field | + +Both follow [Semantic Versioning](https://semver.org/): `MAJOR.MINOR.PATCH`. + +## Trace schema compatibility + +Trazo accepts a trace whose **major** schema version matches the binary's +supported major. Within that major, minor and patch differences are compatible. + +- **Patch** (`0.1.0` -> `0.1.1`): clarifications or fixes with no field changes. +- **Minor** (`0.1.0` -> `0.2.0`): backward-compatible additions, typically a new + optional field. Older traces still validate; newer traces still load on an + older binary of the same major, because unknown fields are ignored by the Go + loader. (Adding the optional `toolCallId` field was such a minor bump.) +- **Major** (`0.x` -> `1.0.0`): a breaking change: a removed or renamed field, or + a changed meaning. Traces across a major boundary are **rejected**. + +Current supported schema major: **0**. +Current schema version: **0.1.0**. + +### What happens to an incompatible trace + +The gate is `trajectory.checkVersion`, run as part of `Run.Validate`: + +- Missing or non-semver `version` -> rejected as an invalid trace. +- A `2.0.0` trace on a major-0 binary -> rejected with a message naming the + supported version, for example: + + ``` + run: schema version "2.0.0" (major 2) is unsupported; this build accepts major 0 (0.1.0) + ``` + +Rejected traces surface as file errors (CLI exit code 2); they do not abort the +rest of a batch. + +### Migrating older traces + +Within a major there is nothing to migrate: an older minor validates as-is. A +future major bump will ship with either a documented migration for the changed +fields or an explicit decision to drop support for the old major, recorded in the +[changelog](../CHANGELOG.md). We do not silently coerce across a major. + +## Product versioning + +The product version bumps on user-visible changes to the CLI or library: + +- **Patch**: bug fixes, no interface change. +- **Minor**: backward-compatible features (a new flag, a new evaluator, an + additive field in the JSON output). +- **Major**: breaking changes to flags, exit codes, the library API, or the + machine-readable output contract. + +The machine-readable JSON output is a contract in its own right and carries its +own `schemaVersion` field; see the output section of the README once published. + +## Release process + +1. Move the `[Unreleased]` entries in [CHANGELOG.md](../CHANGELOG.md) under a new + `[X.Y.Z]` heading with the date. +2. Bump `main.Version` (and `trajectory.SchemaVersion` plus `supportedMajor` if + the trace format changed). +3. Tag the commit `vX.Y.Z`. The build embeds the commit automatically, so + `trazo version` reports the tagged revision. +4. `go build ./... && go test ./...` must be green, and the Python suite too. + +`trazo version` prints all of this at a glance: + +``` +trazo 0.2.0 +trace schema 0.1.0 +commit 891c879de1ad +built 2026-08-07T21:01:56Z +go1.25.0 +``` From 19fc67259b91e8231ee90a5174f1ceae319bd78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 18:31:13 -0400 Subject: [PATCH 02/11] feat(output): versioned, self-describing JSON envelope (0.2.0 M2) Replace the bare {evaluations, fileErrors} JSON output with a versioned envelope: outputVersion, trazoVersion, traceSchemaVersion, generatedAt, an aggregate summary, results, and structured errors. The shape is a documented contract (docs/output.md) with its own version, so CI and downstream tools can depend on it and ignore unknown fields across minor bumps. Introduce report.Summarize as the single source of the aggregate counts, shared by the JSON summary and the text footer so they cannot disagree. GeneratedAt is injected via report.Meta, keeping output deterministic in tests. BREAKING (JSON output): top-level keys evaluations/fileErrors are now results/errors under the envelope. Update the Python tests that parse the CLI output and the golden file accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 10 +++ .../langgraph-reference/tests/test_agent.py | 4 +- .../tests/test_cross_language.py | 6 +- .../tests/test_scenarios.py | 2 +- cmd/trazo/main.go | 16 +++- docs/output.md | 85 +++++++++++++++++++ docs/versioning.md | 2 +- report/golden_test.go | 9 +- report/json.go | 56 +++++++++--- report/summary.go | 49 +++++++++++ report/testdata/json.golden | 18 +++- report/text.go | 20 +---- 12 files changed, 233 insertions(+), 44 deletions(-) create mode 100644 docs/output.md create mode 100644 report/summary.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bb0248..6abcc5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,18 @@ product version. The trace **schema** version is tracked separately; see and build metadata come from the Go toolchain's build info (no ldflags needed). - [docs/versioning.md](docs/versioning.md): the product and schema versioning policy, the schema compatibility contract, and the release process. +- Versioned, self-describing JSON output: `-format json` now emits an envelope + with `outputVersion`, `trazoVersion`, `traceSchemaVersion`, `generatedAt`, an + aggregate `summary`, `results`, and structured `errors`. Documented in + [docs/output.md](docs/output.md). - This changelog. +### Changed + +- BREAKING (JSON output): the top-level keys `evaluations` and `fileErrors` are + now `results` and `errors`, nested under the new envelope. Consumers should + read `outputVersion` and ignore unknown fields. + ## [0.1.0] The hardening round: formalize the trace contract and round the core out from a diff --git a/agents/langgraph-reference/tests/test_agent.py b/agents/langgraph-reference/tests/test_agent.py index 7bc841b..868cb53 100644 --- a/agents/langgraph-reference/tests/test_agent.py +++ b/agents/langgraph-reference/tests/test_agent.py @@ -85,7 +85,7 @@ def test_agent_trace_evaluates_clean_in_go(tmp_path) -> None: ) assert proc.stdout, proc.stderr out = json.loads(proc.stdout) - assert out["fileErrors"] == [] + assert out["errors"] == [] # Clean means every evaluator (tool_calls, loops, cost_latency, node) is clean. - findings = [f for e in out["evaluations"] if e["runId"] == result.run_id for f in e["findings"]] + findings = [f for e in out["results"] if e["runId"] == result.run_id for f in e["findings"]] assert findings == [], findings diff --git a/agents/langgraph-reference/tests/test_cross_language.py b/agents/langgraph-reference/tests/test_cross_language.py index 0806488..9ed6872 100644 --- a/agents/langgraph-reference/tests/test_cross_language.py +++ b/agents/langgraph-reference/tests/test_cross_language.py @@ -40,10 +40,10 @@ def _run_trazo(traces_dir: Path) -> dict: def _findings_for(result: dict, run_id: str) -> list[dict]: - if not any(ev["runId"] == run_id for ev in result["evaluations"]): + if not any(ev["runId"] == run_id for ev in result["results"]): raise AssertionError(f"run {run_id} not found in {result}") # Aggregate findings across every evaluator for this run. - return [f for ev in result["evaluations"] if ev["runId"] == run_id for f in ev["findings"]] + return [f for ev in result["results"] if ev["runId"] == run_id for f in ev["findings"]] def test_clean_run_evaluates_with_no_findings(tmp_path) -> None: @@ -55,7 +55,7 @@ def test_clean_run_evaluates_with_no_findings(tmp_path) -> None: rec.flush(tmp_path, end_time=_dt(4)) result = _run_trazo(tmp_path) - assert result["fileErrors"] == [] + assert result["errors"] == [] assert _findings_for(result, "run-clean") == [] diff --git a/agents/langgraph-reference/tests/test_scenarios.py b/agents/langgraph-reference/tests/test_scenarios.py index c09ee15..4f65b12 100644 --- a/agents/langgraph-reference/tests/test_scenarios.py +++ b/agents/langgraph-reference/tests/test_scenarios.py @@ -85,5 +85,5 @@ def test_scenarios_produce_expected_severity_in_go(tmp_path, name, judgment) -> assert proc.stdout, proc.stderr out = json.loads(proc.stdout) # Aggregate findings across every evaluator for this run. - findings = [f for e in out["evaluations"] if e["runId"] == result.run_id for f in e["findings"]] + findings = [f for e in out["results"] if e["runId"] == result.run_id for f in e["findings"]] assert any(f["judgment"] == judgment for f in findings), findings diff --git a/cmd/trazo/main.go b/cmd/trazo/main.go index eaf17e0..3015bd6 100644 --- a/cmd/trazo/main.go +++ b/cmd/trazo/main.go @@ -8,10 +8,12 @@ import ( "os" "os/signal" "strings" + "time" "github.com/Cro22/trazo/evaluator" "github.com/Cro22/trazo/report" "github.com/Cro22/trazo/runner" + "github.com/Cro22/trazo/trajectory" ) // Exit codes: 0 clean, 1 at least one JudgmentBad finding, 2 at least one @@ -104,7 +106,13 @@ func main() { resp := runner.NewRunner(evaluators).RunFiles(ctx, files) - render(out, *validate, resp, len(files)) + meta := report.Meta{ + TrazoVersion: Version, + TraceSchemaVersion: trajectory.SchemaVersion, + Files: len(files), + GeneratedAt: time.Now(), + } + render(out, *validate, resp, meta) os.Exit(exitCode(resp)) } @@ -131,10 +139,10 @@ func resolveFiles(path string, recursive bool) ([]string, error) { return []string{path}, nil } -func render(out string, validate bool, resp *runner.Response, total int) { +func render(out string, validate bool, resp *runner.Response, meta report.Meta) { switch out { case "json": - s, err := report.JSON(resp) + s, err := report.JSON(resp, meta) if err != nil { log.Fatalf("encoding JSON: %v", err) } @@ -143,7 +151,7 @@ func render(out string, validate bool, resp *runner.Response, total int) { fmt.Print(report.Markdown(resp)) default: // text if validate { - fmt.Print(report.ValidateSummary(resp, total)) + fmt.Print(report.ValidateSummary(resp, meta.Files)) } else { fmt.Print(report.Text(resp)) } diff --git a/docs/output.md b/docs/output.md new file mode 100644 index 0000000..26d33a4 --- /dev/null +++ b/docs/output.md @@ -0,0 +1,85 @@ +# Machine-readable output (`-format json`) + +`trazo -format json` (or `-json`) prints a single self-describing envelope meant +for CI and downstream tools. Its shape is a contract with its own version, +independent of the product and trace-schema versions. + +## Envelope + +```json +{ + "outputVersion": "1.0", + "trazoVersion": "0.2.0", + "traceSchemaVersion": "0.1.0", + "generatedAt": "2026-08-07T12:00:00Z", + "summary": { + "files": 4, + "runs": 2, + "evaluations": 3, + "findings": 3, + "good": 0, + "neutral": 2, + "bad": 1, + "fileErrors": 1 + }, + "results": [ + { + "evaluatorName": "tool_calls", + "runId": "run-1", + "findings": [ + { "stepIndex": 3, "judgment": "bad", "comment": "..." } + ] + } + ], + "errors": [ + { "file": "broken.json", "error": "unexpected end of JSON input" } + ] +} +``` + +## Fields + +| Field | Type | Notes | +|-------|------|-------| +| `outputVersion` | string | Version of this envelope contract. See [Stability](#stability). | +| `trazoVersion` | string | Product version that produced the output. | +| `traceSchemaVersion` | string | Trace schema version this build supports. | +| `generatedAt` | RFC3339 string | UTC generation time. | +| `summary` | object | Aggregate counts, see below. | +| `results` | array | One entry per (evaluator, run); `[]` when there is nothing to report. | +| `errors` | array | One entry per file that failed to read, parse, or validate; `[]` when none. | + +`summary`: + +| Field | Meaning | +|-------|---------| +| `files` | Trace files considered. | +| `runs` | Distinct runs evaluated. | +| `evaluations` | Entries in `results` (evaluator x run). | +| `findings` | Total findings across all results. | +| `good` / `neutral` / `bad` | Findings by severity. | +| `fileErrors` | Entries in `errors`. | + +Each `results[i]` object: `evaluatorName` (string), `runId` (string), `findings` +(array). Each finding: `stepIndex` (int; `-1` is run-level), `judgment` (`good` | +`neutral` | `bad`), `comment` (string), and `score` (number, omitted when zero). + +Each `errors[i]` object: `file` (path as given to trazo) and `error` (message). + +## Stability + +`outputVersion` follows semver-like rules for the envelope: + +- Additive, backward-compatible changes (a new field) bump the **minor**. +- Renaming or removing a field, or changing a field's meaning, bumps the + **major**. + +Consumers should ignore unknown fields so a minor bump does not break them. +Empty collections are always `[]`, never `null`, so indexing is safe without a +nil check. See [versioning.md](versioning.md) for how this relates to the product +and trace-schema versions. + +## Exit codes + +The envelope is independent of the process exit code, which CI can gate on +directly: `0` clean, `1` at least one `bad` finding, `2` at least one file error. diff --git a/docs/versioning.md b/docs/versioning.md index 7a108a7..b2e823c 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -59,7 +59,7 @@ The product version bumps on user-visible changes to the CLI or library: machine-readable output contract. The machine-readable JSON output is a contract in its own right and carries its -own `schemaVersion` field; see the output section of the README once published. +own `outputVersion` field; see [output.md](output.md). ## Release process diff --git a/report/golden_test.go b/report/golden_test.go index 2cc9a44..3a68a66 100644 --- a/report/golden_test.go +++ b/report/golden_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/Cro22/trazo/evaluator" "github.com/Cro22/trazo/runner" @@ -59,7 +60,13 @@ func goldenFixture() *runner.Response { func TestGolden(t *testing.T) { resp := goldenFixture() - jsonOut, err := JSON(resp) + meta := Meta{ + TrazoVersion: "0.2.0", + TraceSchemaVersion: "0.1.0", + Files: 4, + GeneratedAt: time.Date(2026, 8, 7, 12, 0, 0, 0, time.UTC), + } + jsonOut, err := JSON(resp, meta) if err != nil { t.Fatalf("JSON: %v", err) } diff --git a/report/json.go b/report/json.go index af3c306..9cdf3a2 100644 --- a/report/json.go +++ b/report/json.go @@ -2,37 +2,65 @@ package report import ( "encoding/json" + "time" "github.com/Cro22/trazo/evaluator" "github.com/Cro22/trazo/runner" ) +// OutputVersion is the version of the machine-readable JSON output contract (the +// envelope shape below), independent of the product and trace-schema versions. +// Bump the minor for additive fields, the major for a breaking change. See +// docs/output.md. +const OutputVersion = "1.0" + +// Meta is the context the caller supplies for the JSON envelope: the versions to +// stamp, the number of files considered, and the generation time. GeneratedAt is +// injected (not read from the clock here) so output is deterministic in tests. +type Meta struct { + TrazoVersion string + TraceSchemaVersion string + Files int + GeneratedAt time.Time +} + type fileErrorJSON struct { File string `json:"file"` Error string `json:"error"` } -type responseJSON struct { - Evaluations []*evaluator.Evaluation `json:"evaluations"` - FileErrors []fileErrorJSON `json:"fileErrors"` +type jsonEnvelope struct { + OutputVersion string `json:"outputVersion"` + TrazoVersion string `json:"trazoVersion"` + TraceSchemaVersion string `json:"traceSchemaVersion"` + GeneratedAt string `json:"generatedAt"` + Summary Summary `json:"summary"` + Results []*evaluator.Evaluation `json:"results"` + Errors []fileErrorJSON `json:"errors"` } -// JSON renders the evaluation results as indented JSON. Empty slices serialize as -// [] rather than null so consumers can index without a nil check, and errors are -// flattened to strings. -func JSON(resp *runner.Response) (string, error) { - out := responseJSON{ - Evaluations: resp.Evaluations, - FileErrors: []fileErrorJSON{}, +// JSON renders the results as a versioned, self-describing envelope: metadata, an +// aggregate summary, the per-run evaluations, and structured file errors. Empty +// slices serialize as [] (never null) so consumers can index without a nil +// check. The shape is a stable contract; see docs/output.md. +func JSON(resp *runner.Response, meta Meta) (string, error) { + env := jsonEnvelope{ + OutputVersion: OutputVersion, + TrazoVersion: meta.TrazoVersion, + TraceSchemaVersion: meta.TraceSchemaVersion, + GeneratedAt: meta.GeneratedAt.UTC().Format(time.RFC3339), + Summary: Summarize(resp, meta.Files), + Results: resp.Evaluations, + Errors: []fileErrorJSON{}, } - if out.Evaluations == nil { - out.Evaluations = []*evaluator.Evaluation{} + if env.Results == nil { + env.Results = []*evaluator.Evaluation{} } for _, fe := range resp.FileErrors { - out.FileErrors = append(out.FileErrors, fileErrorJSON{File: fe.File, Error: fe.Err.Error()}) + env.Errors = append(env.Errors, fileErrorJSON{File: fe.File, Error: fe.Err.Error()}) } - data, err := json.MarshalIndent(out, "", " ") + data, err := json.MarshalIndent(env, "", " ") if err != nil { return "", err } diff --git a/report/summary.go b/report/summary.go new file mode 100644 index 0000000..7a89f43 --- /dev/null +++ b/report/summary.go @@ -0,0 +1,49 @@ +package report + +import ( + "github.com/Cro22/trazo/evaluator" + "github.com/Cro22/trazo/runner" +) + +// Summary is the aggregate outcome of a run batch, shared by the text footer and +// the JSON envelope so the two never disagree. +type Summary struct { + Files int `json:"files"` + Runs int `json:"runs"` + Evaluations int `json:"evaluations"` + Findings int `json:"findings"` + Good int `json:"good"` + Neutral int `json:"neutral"` + Bad int `json:"bad"` + FileErrors int `json:"fileErrors"` +} + +// Summarize aggregates a Response. files is the number of trace files considered +// (which the Response alone does not carry, since valid files with no findings +// still count); pass 0 when it is not known. +func Summarize(resp *runner.Response, files int) Summary { + s := Summary{ + Files: files, + Evaluations: len(resp.Evaluations), + FileErrors: len(resp.FileErrors), + } + seen := map[string]bool{} + for _, e := range resp.Evaluations { + if !seen[e.RunID] { + seen[e.RunID] = true + s.Runs++ + } + for _, f := range e.Findings { + s.Findings++ + switch f.Judgment { + case evaluator.JudgmentBad: + s.Bad++ + case evaluator.JudgmentNeutral: + s.Neutral++ + case evaluator.JudgmentGood: + s.Good++ + } + } + } + return s +} diff --git a/report/testdata/json.golden b/report/testdata/json.golden index a064bf2..6c23ed9 100644 --- a/report/testdata/json.golden +++ b/report/testdata/json.golden @@ -1,5 +1,19 @@ { - "evaluations": [ + "outputVersion": "1.0", + "trazoVersion": "0.2.0", + "traceSchemaVersion": "0.1.0", + "generatedAt": "2026-08-07T12:00:00Z", + "summary": { + "files": 4, + "runs": 2, + "evaluations": 3, + "findings": 3, + "good": 0, + "neutral": 2, + "bad": 1, + "fileErrors": 1 + }, + "results": [ { "evaluatorName": "tool_calls", "runId": "run-1", @@ -34,7 +48,7 @@ "findings": [] } ], - "fileErrors": [ + "errors": [ { "file": "broken.json", "error": "unexpected end of JSON input" diff --git a/report/text.go b/report/text.go index e8f4987..0d622b3 100644 --- a/report/text.go +++ b/report/text.go @@ -34,7 +34,7 @@ func Text(resp *runner.Response) string { b.WriteString("\n") } - b.WriteString(summaryLine(groups, order, len(resp.FileErrors))) + b.WriteString(summaryLine(resp)) if len(resp.FileErrors) > 0 { b.WriteString("\nFile errors:\n") for _, fe := range resp.FileErrors { @@ -128,22 +128,10 @@ func stepText(idx int) string { return fmt.Sprintf("step %d", idx) } -func summaryLine(groups map[string]*runGroup, order []string, fileErrors int) string { - var bad, neutral, good int - for _, runID := range order { - for _, r := range groups[runID].rows { - switch r.severity { - case "[BAD]": - bad++ - case "[NEUTRAL]": - neutral++ - case "[GOOD]": - good++ - } - } - } +func summaryLine(resp *runner.Response) string { + s := Summarize(resp, 0) return fmt.Sprintf("Summary: %s, %d bad, %d neutral, %d good, %s\n", - plural(len(order), "run"), bad, neutral, good, plural(fileErrors, "file error")) + plural(s.Runs, "run"), s.Bad, s.Neutral, s.Good, plural(s.FileErrors, "file error")) } // oneline flattens a possibly multi-line message onto a single line so table From e9f8d7b5606679e945f1bac0bb60950d5766ade7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 19:44:44 -0400 Subject: [PATCH 03/11] feat(config): reproducible JSON evaluator policy file (0.2.0 M3) Add a stdlib-only `config` package and a `-config ` flag: a versioned JSON policy that pins which evaluators run and their thresholds, so a team can commit one file and get identical evaluation across dev and CI. No new dependency; the core stays stdlib-only. - Partial configs are valid (omitted fields keep defaults); unknown fields are rejected (a typo is an error, not a silent no-op); version is required. - Precedence: defaults < config file < explicitly-set flags (via flag.Visit), so the config is a reproducible baseline a one-off flag can still override. - The package is CLI-independent: config.Load + Config.Build return the evaluator set for use from another Go program. Document it in docs/config.md with an example policy, and record it in the changelog. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 + cmd/trazo/main.go | 96 +++++++++---------- config/config.go | 167 +++++++++++++++++++++++++++++++++ config/config_test.go | 129 +++++++++++++++++++++++++ docs/config.md | 74 +++++++++++++++ docs/trazo.config.example.json | 27 ++++++ 6 files changed, 450 insertions(+), 48 deletions(-) create mode 100644 config/config.go create mode 100644 config/config_test.go create mode 100644 docs/config.md create mode 100644 docs/trazo.config.example.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 6abcc5e..3cca563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ product version. The trace **schema** version is tracked separately; see with `outputVersion`, `trazoVersion`, `traceSchemaVersion`, `generatedAt`, an aggregate `summary`, `results`, and structured `errors`. Documented in [docs/output.md](docs/output.md). +- Evaluator policy file: `-config ` loads a versioned, stdlib-only JSON + policy (the new `config` package) that pins which evaluators run and their + thresholds, for reproducibility across dev, CI, and teams. Precedence is + defaults < config < explicitly-set flags. Documented in + [docs/config.md](docs/config.md), with an example config. - This changelog. ### Changed diff --git a/cmd/trazo/main.go b/cmd/trazo/main.go index 3015bd6..994430f 100644 --- a/cmd/trazo/main.go +++ b/cmd/trazo/main.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/Cro22/trazo/config" "github.com/Cro22/trazo/evaluator" "github.com/Cro22/trazo/report" "github.com/Cro22/trazo/runner" @@ -37,6 +38,7 @@ func main() { } showVersion := flag.Bool("version", false, "print version information and exit") + configPath := flag.String("config", "", "path to a JSON evaluator policy file (see docs/config.md)") dir := flag.String("dir", "./testdata/runs", "directory of traces to scan when no PATH is given") recursive := flag.Bool("recursive", false, "descend into subdirectories when PATH is a directory") validate := flag.Bool("validate", false, "only check that traces load and pass structural validation; skip evaluators") @@ -88,17 +90,54 @@ func main() { log.Printf("no .json traces found under %q", path) } - evaluators := buildEvaluators(*validate, evaluatorConfig{ - maxRepeats: *maxRepeats, - maxStepCost: *maxStepCost, - maxStepLatencyMs: *maxStepLatencyMs, - maxRunCost: *maxRunCost, - maxRunLatencyMs: *maxRunLatencyMs, - terminalNodes: splitCSV(*terminalNodes), - llmJudge: *llmJudge, - judgeModel: *judgeModel, + // Policy precedence: built-in defaults < config file < explicitly-set flags. + // The config file pins a reproducible policy; a flag the user actually passed + // still wins over it (flag.Visit reports only the flags that were set). + cfg := config.Default() + if *configPath != "" { + loaded, err := config.Load(*configPath) + if err != nil { + log.Fatalf("%v", err) + } + cfg = *loaded + } + flag.Visit(func(f *flag.Flag) { + switch f.Name { + case "max-repeats": + cfg.Evaluators.Loops.MaxRepeats = *maxRepeats + case "max-step-cost": + cfg.Evaluators.CostLatency.MaxStepCost = *maxStepCost + case "max-step-latency-ms": + cfg.Evaluators.CostLatency.MaxStepLatencyMs = *maxStepLatencyMs + case "max-run-cost": + cfg.Evaluators.CostLatency.MaxRunCost = *maxRunCost + case "max-run-latency-ms": + cfg.Evaluators.CostLatency.MaxRunLatencyMs = *maxRunLatencyMs + case "terminal-nodes": + cfg.Evaluators.NodeTransitions.TerminalNodes = splitCSV(*terminalNodes) + case "llm-judge": + cfg.Evaluators.LLMJudge.Enabled = *llmJudge + case "judge-model": + cfg.Evaluators.LLMJudge.Model = *judgeModel + } }) + // In validate-only mode the runner just loads and structurally validates each + // file, so no evaluators are built regardless of the policy. + var evaluators []evaluator.Evaluator + if !*validate { + evaluators, err = cfg.Build(func(model string) (evaluator.Evaluator, error) { + client, cerr := evaluator.NewGeminiClient(model) + if cerr != nil { + return nil, cerr + } + return &evaluator.LLMJudgeEvaluator{Client: client}, nil + }) + if err != nil { + log.Fatalf("llm-judge: %v", err) + } + } + // Cancel in-flight evaluation on Ctrl+C (SIGINT) or SIGTERM so a long run, // notably one using the network-bound LLM judge, stops promptly. ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) @@ -158,45 +197,6 @@ func render(out string, validate bool, resp *runner.Response, meta report.Meta) } } -type evaluatorConfig struct { - maxRepeats int - maxStepCost float64 - maxStepLatencyMs int64 - maxRunCost float64 - maxRunLatencyMs int64 - terminalNodes []string - llmJudge bool - judgeModel string -} - -// buildEvaluators assembles the evaluator set. In validate-only mode it returns -// none, so the runner just loads and structurally validates each file. The LLM -// judge is opt-in and constructed last because it can fail (missing API key). -func buildEvaluators(validate bool, cfg evaluatorConfig) []evaluator.Evaluator { - if validate { - return nil - } - evaluators := []evaluator.Evaluator{ - &evaluator.ToolCallEvaluator{}, - &evaluator.LoopEvaluator{MaxRepeats: cfg.maxRepeats}, - &evaluator.CostLatencyEvaluator{ - MaxStepCost: cfg.maxStepCost, - MaxStepLatencyMs: cfg.maxStepLatencyMs, - MaxRunCost: cfg.maxRunCost, - MaxRunLatencyMs: cfg.maxRunLatencyMs, - }, - &evaluator.NodeTransitionEvaluator{TerminalNodes: cfg.terminalNodes}, - } - if cfg.llmJudge { - client, err := evaluator.NewGeminiClient(cfg.judgeModel) - if err != nil { - log.Fatalf("llm-judge: %v", err) - } - evaluators = append(evaluators, &evaluator.LLMJudgeEvaluator{Client: client}) - } - return evaluators -} - // splitCSV parses a comma-separated flag value into a trimmed, non-empty slice, // returning nil for an empty value so the evaluator falls back to its default. func splitCSV(s string) []string { diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..90e1ac8 --- /dev/null +++ b/config/config.go @@ -0,0 +1,167 @@ +// Package config defines trazo's evaluator policy file: a small, versioned JSON +// document that pins which evaluators run and with what thresholds, so a policy +// is reproducible across a developer's machine, CI, and a team. It is +// stdlib-only, like the rest of the core, and independent of the CLI: another Go +// program can Load a policy and build the same evaluator set. +package config + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + + "github.com/Cro22/trazo/evaluator" +) + +// SupportedVersion is the config file format version this build understands. The +// file must declare it explicitly; a different value is rejected rather than +// guessed at. +const SupportedVersion = 1 + +// Config is the whole policy file. Version is the config format version (not the +// product or trace-schema version). +type Config struct { + Version int `json:"version"` + Evaluators Evaluators `json:"evaluators"` +} + +type Evaluators struct { + ToolCalls ToolCalls `json:"tool_calls"` + Loops Loops `json:"loops"` + CostLatency CostLatency `json:"cost_latency"` + NodeTransitions NodeTransitions `json:"node_transitions"` + LLMJudge LLMJudge `json:"llm_judge"` +} + +type ToolCalls struct { + Enabled bool `json:"enabled"` +} + +type Loops struct { + Enabled bool `json:"enabled"` + MaxRepeats int `json:"maxRepeats"` +} + +type CostLatency struct { + Enabled bool `json:"enabled"` + MaxStepCost float64 `json:"maxStepCost"` + MaxStepLatencyMs int64 `json:"maxStepLatencyMs"` + MaxRunCost float64 `json:"maxRunCost"` + MaxRunLatencyMs int64 `json:"maxRunLatencyMs"` +} + +type NodeTransitions struct { + Enabled bool `json:"enabled"` + TerminalNodes []string `json:"terminalNodes"` +} + +type LLMJudge struct { + Enabled bool `json:"enabled"` + Model string `json:"model"` +} + +// Default is the built-in policy, matching the CLI's flag defaults: the four +// structural evaluators enabled, the LLM judge off. TerminalNodes is left nil so +// the node evaluator falls back to its own default set. +func Default() Config { + return Config{ + Version: SupportedVersion, + Evaluators: Evaluators{ + ToolCalls: ToolCalls{Enabled: true}, + Loops: Loops{Enabled: true, MaxRepeats: evaluator.DefaultMaxRepeats}, + CostLatency: CostLatency{ + Enabled: true, + MaxStepCost: evaluator.DefaultMaxStepCost, + MaxStepLatencyMs: evaluator.DefaultMaxStepLatencyMs, + MaxRunCost: evaluator.DefaultMaxRunCost, + MaxRunLatencyMs: evaluator.DefaultMaxRunLatencyMs, + }, + NodeTransitions: NodeTransitions{Enabled: true}, + LLMJudge: LLMJudge{Enabled: false, Model: evaluator.DefaultJudgeModel}, + }, + } +} + +// Load reads a policy file, layering it over Default so an omitted field keeps +// its default (partial configs are valid). Unknown fields are rejected, so a +// typo like "maxRepeat" is an error instead of being silently ignored. The file +// must declare version == SupportedVersion. +func Load(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + cfg := Default() + // Clear Version so we can tell whether the file actually declared it: an + // omitted version leaves it 0 and fails the check below. + cfg.Version = 0 + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&cfg); err != nil { + return nil, fmt.Errorf("config %s: %w", path, err) + } + + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("config %s: %w", path, err) + } + return &cfg, nil +} + +func (c *Config) validate() error { + if c.Version != SupportedVersion { + return fmt.Errorf("version %d is unsupported (this build accepts version %d)", c.Version, SupportedVersion) + } + if c.Evaluators.Loops.MaxRepeats < 0 { + return fmt.Errorf("loops.maxRepeats is negative (%d)", c.Evaluators.Loops.MaxRepeats) + } + cl := c.Evaluators.CostLatency + if cl.MaxStepCost < 0 || cl.MaxRunCost < 0 { + return fmt.Errorf("cost_latency costs must be non-negative") + } + if cl.MaxStepLatencyMs < 0 || cl.MaxRunLatencyMs < 0 { + return fmt.Errorf("cost_latency latencies must be non-negative") + } + for i, n := range c.Evaluators.NodeTransitions.TerminalNodes { + if n == "" { + return fmt.Errorf("node_transitions.terminalNodes[%d] is empty", i) + } + } + return nil +} + +// Build turns the policy into the evaluator set to run. The LLM judge is +// constructed via newJudge only when enabled; newJudge may fail (missing API +// key), so the caller supplies it and handles that error. +func (c *Config) Build(newJudge func(model string) (evaluator.Evaluator, error)) ([]evaluator.Evaluator, error) { + var evals []evaluator.Evaluator + e := c.Evaluators + + if e.ToolCalls.Enabled { + evals = append(evals, &evaluator.ToolCallEvaluator{}) + } + if e.Loops.Enabled { + evals = append(evals, &evaluator.LoopEvaluator{MaxRepeats: e.Loops.MaxRepeats}) + } + if e.CostLatency.Enabled { + evals = append(evals, &evaluator.CostLatencyEvaluator{ + MaxStepCost: e.CostLatency.MaxStepCost, + MaxStepLatencyMs: e.CostLatency.MaxStepLatencyMs, + MaxRunCost: e.CostLatency.MaxRunCost, + MaxRunLatencyMs: e.CostLatency.MaxRunLatencyMs, + }) + } + if e.NodeTransitions.Enabled { + evals = append(evals, &evaluator.NodeTransitionEvaluator{TerminalNodes: e.NodeTransitions.TerminalNodes}) + } + if e.LLMJudge.Enabled { + judge, err := newJudge(e.LLMJudge.Model) + if err != nil { + return nil, err + } + evals = append(evals, judge) + } + return evals, nil +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..83cca3d --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,129 @@ +package config + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Cro22/trazo/evaluator" + "github.com/Cro22/trazo/trajectory" +) + +func writeConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "trazo.config.json") + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +// stubEvaluator stands in for a real evaluator so Build can be tested without a +// network-backed judge. +type stubEvaluator struct{} + +func (stubEvaluator) EvaluateRun(context.Context, *trajectory.Run) (*evaluator.Evaluation, error) { + return &evaluator.Evaluation{}, nil +} + +func noJudge(string) (evaluator.Evaluator, error) { + return nil, errors.New("newJudge should not be called when the judge is disabled") +} + +func TestDefault_BuildsFourEvaluators(t *testing.T) { + cfg := Default() + evals, err := cfg.Build(noJudge) + if err != nil { + t.Fatalf("Build: %v", err) + } + if len(evals) != 4 { + t.Fatalf("default policy should build 4 evaluators (judge off), got %d", len(evals)) + } +} + +func TestLoad_PartialLayersOverDefaults(t *testing.T) { + // Only override one nested field; everything else must keep its default. + path := writeConfig(t, `{"version": 1, "evaluators": {"loops": {"maxRepeats": 9}}}`) + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Evaluators.Loops.MaxRepeats != 9 { + t.Errorf("maxRepeats override lost: got %d", cfg.Evaluators.Loops.MaxRepeats) + } + if !cfg.Evaluators.Loops.Enabled { + t.Error("omitted loops.enabled should keep the default (true)") + } + if cfg.Evaluators.CostLatency.MaxStepCost != evaluator.DefaultMaxStepCost { + t.Errorf("omitted cost_latency should keep defaults, got %v", cfg.Evaluators.CostLatency.MaxStepCost) + } +} + +func TestLoad_DisableEvaluator(t *testing.T) { + path := writeConfig(t, `{"version": 1, "evaluators": {"cost_latency": {"enabled": false}, "loops": {"enabled": false}}}`) + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + evals, err := cfg.Build(noJudge) + if err != nil { + t.Fatalf("Build: %v", err) + } + // tool_calls and node_transitions remain enabled by default => 2. + if len(evals) != 2 { + t.Fatalf("expected 2 evaluators after disabling two, got %d", len(evals)) + } +} + +func TestLoad_RejectsUnknownField(t *testing.T) { + path := writeConfig(t, `{"version": 1, "evaluators": {"loops": {"maxRepeat": 3}}}`) + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("expected unknown-field error, got: %v", err) + } +} + +func TestLoad_RejectsBadVersion(t *testing.T) { + for _, body := range []string{ + `{"evaluators": {}}`, // missing version + `{"version": 2, "evaluators": {}}`, // unsupported version + } { + path := writeConfig(t, body) + if _, err := Load(path); err == nil { + t.Errorf("expected version error for %s", body) + } + } +} + +func TestLoad_RejectsNegativeThresholds(t *testing.T) { + path := writeConfig(t, `{"version": 1, "evaluators": {"loops": {"maxRepeats": -1}}}`) + if _, err := Load(path); err == nil { + t.Fatal("expected negative-threshold error") + } +} + +func TestBuild_JudgeEnabledCallsFactory(t *testing.T) { + cfg := Default() + cfg.Evaluators.LLMJudge.Enabled = true + + called := false + evals, err := cfg.Build(func(model string) (evaluator.Evaluator, error) { + called = true + if model != evaluator.DefaultJudgeModel { + t.Errorf("expected default judge model, got %q", model) + } + return stubEvaluator{}, nil + }) + if err != nil { + t.Fatalf("Build: %v", err) + } + if !called { + t.Error("judge factory was not called though the judge is enabled") + } + if len(evals) != 5 { + t.Fatalf("expected 5 evaluators with judge on, got %d", len(evals)) + } +} diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000..8cb5822 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,74 @@ +# Evaluator policy file (`-config`) + +Passing `-config ` loads a JSON policy that pins which evaluators run and +with what thresholds. Commit it to a repo and every developer, CI job, and +teammate evaluates traces the same way, instead of remembering a string of flags. + +The format is stdlib-only JSON (no new dependency) and is versioned in its own +right. A full example is [trazo.config.example.json](trazo.config.example.json). + +```json +{ + "version": 1, + "evaluators": { + "tool_calls": { "enabled": true }, + "loops": { "enabled": true, "maxRepeats": 3 }, + "cost_latency": { + "enabled": true, + "maxStepCost": 0.05, + "maxStepLatencyMs": 30000, + "maxRunCost": 0.2, + "maxRunLatencyMs": 120000 + }, + "node_transitions": { + "enabled": true, + "terminalNodes": ["end", "__end__", "finish", "done"] + }, + "llm_judge": { "enabled": false, "model": "gemini-2.5-flash" } + } +} +``` + +## Rules + +- `version` is required and must be `1` (the format version this build accepts). +- Every field is optional beyond `version`: an omitted field keeps its built-in + default, so a partial config is valid. `{"version": 1, "evaluators": {}}` is the + default policy. +- Unknown fields are rejected. A typo like `maxRepeat` is an error, not a silent + no-op. +- Thresholds must be non-negative; `terminalNodes` entries must be non-empty. +- Omitting `node_transitions.terminalNodes` falls back to the evaluator's default + terminal set. + +## Precedence + +From lowest to highest: + +1. Built-in defaults. +2. The `-config` file. +3. Explicitly-set command-line flags. + +A flag only overrides the config when you actually pass it, so the config pins a +reproducible baseline while a one-off flag (`-max-step-cost 0.01`) still wins for +a single run. Example: + +```bash +# Reproducible team policy, committed to the repo: +trazo -config trazo.config.json ./traces + +# Same policy, but tighten one threshold for this run only: +trazo -config trazo.config.json -max-step-cost 0.01 ./traces +``` + +## Using it as a library + +The policy is a normal Go package, independent of the CLI: + +```go +cfg, err := config.Load("trazo.config.json") +evals, err := cfg.Build(func(model string) (evaluator.Evaluator, error) { + // construct your judge, or return an error to disable it +}) +resp := runner.NewRunner(evals).RunFiles(ctx, files) +``` diff --git a/docs/trazo.config.example.json b/docs/trazo.config.example.json new file mode 100644 index 0000000..0e948e2 --- /dev/null +++ b/docs/trazo.config.example.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "evaluators": { + "tool_calls": { + "enabled": true + }, + "loops": { + "enabled": true, + "maxRepeats": 3 + }, + "cost_latency": { + "enabled": true, + "maxStepCost": 0.05, + "maxStepLatencyMs": 30000, + "maxRunCost": 0.2, + "maxRunLatencyMs": 120000 + }, + "node_transitions": { + "enabled": true, + "terminalNodes": ["end", "__end__", "finish", "done"] + }, + "llm_judge": { + "enabled": false, + "model": "gemini-2.5-flash" + } + } +} From 66b7091ee63b8df2c631869fbbd67d6505363016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 21:19:42 -0400 Subject: [PATCH 04/11] test(cli): black-box tests for exit codes, formats, and precedence (0.2.0 M4) Build the binary once in TestMain and exercise it end to end: clean exit 0, a bad finding exit 1, a file error exit 2, valid JSON envelope, Markdown FAIL report, missing path, invalid -format, two-PATH error, version subcommand, empty directory, and config/flag precedence. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + cmd/trazo/cli_test.go | 217 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 cmd/trazo/cli_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cca563..ccc6a14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ product version. The trace **schema** version is tracked separately; see thresholds, for reproducibility across dev, CI, and teams. Precedence is defaults < config < explicitly-set flags. Documented in [docs/config.md](docs/config.md), with an example config. +- Black-box CLI tests (`cmd/trazo/cli_test.go`) that build the binary and assert + exit codes (0/1/2), each output format, invalid flags, missing paths, empty + directories, and config/flag precedence. - This changelog. ### Changed diff --git a/cmd/trazo/cli_test.go b/cmd/trazo/cli_test.go new file mode 100644 index 0000000..eb18e0c --- /dev/null +++ b/cmd/trazo/cli_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// trazoBin is the freshly built binary under test, shared across the black-box +// tests below. Building once in TestMain keeps the suite fast. +var trazoBin string + +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "trazo-cli-test") + if err != nil { + fmt.Fprintln(os.Stderr, "mktemp:", err) + os.Exit(1) + } + bin := filepath.Join(dir, "trazo") + if runtime.GOOS == "windows" { + bin += ".exe" + } + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + fmt.Fprintln(os.Stderr, "building trazo:", err) + os.RemoveAll(dir) + os.Exit(1) + } + trazoBin = bin + + code := m.Run() + os.RemoveAll(dir) + os.Exit(code) +} + +// Fixture paths, relative to this package directory (cmd/trazo). +const ( + cleanDir = "../../testdata/ci/clean" + cleanFile = "../../testdata/ci/clean/triage_clean.json" + failingFile = "../../testdata/ci/failing/tool_error.json" + brokenFile = "../../testdata/runs/broken.json" +) + +type cliResult struct { + stdout string + stderr string + code int +} + +func runCLI(t *testing.T, args ...string) cliResult { + t.Helper() + cmd := exec.Command(trazoBin, args...) + var out, errb bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &errb + err := cmd.Run() + + code := 0 + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + code = ee.ExitCode() + } else { + t.Fatalf("running %v: %v", args, err) + } + } + return cliResult{stdout: out.String(), stderr: errb.String(), code: code} +} + +func TestCLI_TextCleanExit0(t *testing.T) { + r := runCLI(t, "-dir", cleanDir) + if r.code != 0 { + t.Fatalf("expected exit 0, got %d (stderr: %s)", r.code, r.stderr) + } + if !strings.Contains(r.stdout, "clean") || !strings.Contains(r.stdout, "Summary:") { + t.Errorf("expected a clean summary, got:\n%s", r.stdout) + } +} + +func TestCLI_BadExit1(t *testing.T) { + r := runCLI(t, failingFile) + if r.code != 1 { + t.Fatalf("expected exit 1 on a bad finding, got %d (stdout: %s)", r.code, r.stdout) + } + if !strings.Contains(r.stdout, "[BAD]") { + t.Errorf("expected a BAD finding in output, got:\n%s", r.stdout) + } +} + +func TestCLI_FileErrorExit2(t *testing.T) { + r := runCLI(t, brokenFile) + if r.code != 2 { + t.Fatalf("expected exit 2 on a file error, got %d", r.code) + } + if !strings.Contains(r.stdout, "File errors:") { + t.Errorf("expected a file errors section, got:\n%s", r.stdout) + } +} + +func TestCLI_JSONBadExit1(t *testing.T) { + r := runCLI(t, "-json", failingFile) + if r.code != 1 { + t.Fatalf("expected exit 1, got %d", r.code) + } + var env struct { + OutputVersion string `json:"outputVersion"` + Summary struct { + Bad int `json:"bad"` + } `json:"summary"` + Results []json.RawMessage `json:"results"` + } + if err := json.Unmarshal([]byte(r.stdout), &env); err != nil { + t.Fatalf("output is not valid JSON: %v\n%s", err, r.stdout) + } + if env.OutputVersion == "" { + t.Error("JSON envelope missing outputVersion") + } + if env.Summary.Bad < 1 { + t.Errorf("expected at least one bad finding in summary, got %d", env.Summary.Bad) + } + if len(env.Results) == 0 { + t.Error("expected non-empty results") + } +} + +func TestCLI_MarkdownFileErrorExit2(t *testing.T) { + r := runCLI(t, "-format", "md", brokenFile) + if r.code != 2 { + t.Fatalf("expected exit 2, got %d", r.code) + } + if !strings.Contains(r.stdout, "**Verdict: FAIL**") || !strings.Contains(r.stdout, "File errors") { + t.Errorf("expected a FAIL markdown report with file errors, got:\n%s", r.stdout) + } +} + +func TestCLI_DirectoryDoesNotExist(t *testing.T) { + r := runCLI(t, "./does-not-exist-xyz") + if r.code == 0 { + t.Fatalf("expected a non-zero exit for a missing path, got 0") + } + if !strings.Contains(r.stderr, "trazo:") { + t.Errorf("expected a prefixed error on stderr, got:\n%s", r.stderr) + } +} + +func TestCLI_InvalidFormat(t *testing.T) { + r := runCLI(t, "-format", "xml", cleanFile) + if r.code == 0 { + t.Fatalf("expected a non-zero exit for an invalid format") + } + if !strings.Contains(r.stderr, "unknown -format") { + t.Errorf("expected an unknown-format error, got:\n%s", r.stderr) + } +} + +func TestCLI_TwoPathsError(t *testing.T) { + r := runCLI(t, cleanFile, failingFile) + if r.code != 2 { + t.Fatalf("expected exit 2 for two PATH args, got %d", r.code) + } + if !strings.Contains(r.stderr, "at most one PATH") { + t.Errorf("expected a one-PATH error, got:\n%s", r.stderr) + } +} + +func TestCLI_Version(t *testing.T) { + r := runCLI(t, "version") + if r.code != 0 { + t.Fatalf("expected exit 0, got %d", r.code) + } + if !strings.Contains(r.stdout, "trazo "+Version) || !strings.Contains(r.stdout, "trace schema ") { + t.Errorf("unexpected version output:\n%s", r.stdout) + } +} + +func TestCLI_EmptyDirectory(t *testing.T) { + empty := t.TempDir() + r := runCLI(t, "-dir", empty) + if r.code != 0 { + t.Fatalf("expected exit 0 for an empty directory, got %d", r.code) + } + if !strings.Contains(r.stderr, "no .json traces") { + t.Errorf("expected a no-traces notice on stderr, got:\n%s", r.stderr) + } +} + +// TestCLI_FlagOverridesConfig pins the precedence rule: a config sets a lax cost +// threshold (no finding), and an explicit -max-step-cost flag overrides it (a +// finding appears). Both runs stay exit 0 because a cost finding is neutral. +func TestCLI_FlagOverridesConfig(t *testing.T) { + cfgPath := filepath.Join(t.TempDir(), "policy.json") + cfg := `{"version":1,"evaluators":{"cost_latency":{"enabled":true,"maxStepCost":1.0,"maxRunCost":1.0,"maxStepLatencyMs":600000,"maxRunLatencyMs":600000}}}` + if err := os.WriteFile(cfgPath, []byte(cfg), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + withConfig := runCLI(t, "-config", cfgPath, cleanFile) + if withConfig.code != 0 { + t.Fatalf("config-only run should be clean (exit 0), got %d:\n%s", withConfig.code, withConfig.stdout) + } + if strings.Contains(withConfig.stdout, "cost_latency") { + t.Errorf("lax config should produce no cost finding, got:\n%s", withConfig.stdout) + } + + overridden := runCLI(t, "-config", cfgPath, "-max-step-cost", "0.00001", cleanFile) + if !strings.Contains(overridden.stdout, "cost_latency") { + t.Errorf("flag should override config and produce a cost finding, got:\n%s", overridden.stdout) + } +} From 08942a3935e4b84d35c8db41c7a95d9b2afdf44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 22:22:09 -0400 Subject: [PATCH 05/11] feat: typed file errors surfaced in output (0.2.0) Classify why a file could not be evaluated with runner.ErrorKind (read_file, invalid_json, invalid_trace, evaluator, canceled) instead of leaving callers to match on message strings. The kind is set at each failure point in the runner, emitted as errors[].kind in the JSON output (outputVersion bumped to 1.1, an additive change), and tagged in the text and validate output. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ docs/output.md | 16 +++++++++++++--- report/golden_test.go | 2 +- report/json.go | 5 +++-- report/testdata/json.golden | 3 ++- report/testdata/text.golden | 2 +- report/text.go | 12 +++++++++++- report/validate.go | 2 +- runner/runner.go | 24 +++++++++++++++++++----- runner/runner_test.go | 6 ++++++ 10 files changed, 61 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc6a14..c50193d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ product version. The trace **schema** version is tracked separately; see - Black-box CLI tests (`cmd/trazo/cli_test.go`) that build the binary and assert exit codes (0/1/2), each output format, invalid flags, missing paths, empty directories, and config/flag precedence. +- Typed file errors: `runner.FileError` now carries an `ErrorKind` + (`read_file`, `invalid_json`, `invalid_trace`, `evaluator`, `canceled`), + surfaced as `errors[].kind` in the JSON output (bumped to `outputVersion` 1.1, + additive) and tagged in the text output. - This changelog. ### Changed diff --git a/docs/output.md b/docs/output.md index 26d33a4..86f19a0 100644 --- a/docs/output.md +++ b/docs/output.md @@ -8,7 +8,7 @@ independent of the product and trace-schema versions. ```json { - "outputVersion": "1.0", + "outputVersion": "1.1", "trazoVersion": "0.2.0", "traceSchemaVersion": "0.1.0", "generatedAt": "2026-08-07T12:00:00Z", @@ -32,7 +32,7 @@ independent of the product and trace-schema versions. } ], "errors": [ - { "file": "broken.json", "error": "unexpected end of JSON input" } + { "file": "broken.json", "kind": "invalid_json", "error": "unexpected end of JSON input" } ] } ``` @@ -64,7 +64,17 @@ Each `results[i]` object: `evaluatorName` (string), `runId` (string), `findings` (array). Each finding: `stepIndex` (int; `-1` is run-level), `judgment` (`good` | `neutral` | `bad`), `comment` (string), and `score` (number, omitted when zero). -Each `errors[i]` object: `file` (path as given to trazo) and `error` (message). +Each `errors[i]` object: `file` (path as given to trazo), `kind` (category, see +below), and `error` (message). `kind` lets a consumer react by category instead +of matching message strings: + +| `kind` | Meaning | +|--------|---------| +| `read_file` | The file could not be read. | +| `invalid_json` | The bytes are not valid JSON. | +| `invalid_trace` | JSON parsed but failed structural validation (`Run.Validate`). | +| `evaluator` | An evaluator returned an error. | +| `canceled` | The context was canceled (Ctrl+C, timeout) before processing. | ## Stability diff --git a/report/golden_test.go b/report/golden_test.go index 3a68a66..3a96f76 100644 --- a/report/golden_test.go +++ b/report/golden_test.go @@ -52,7 +52,7 @@ func goldenFixture() *runner.Response { }, }, FileErrors: []runner.FileError{ - {File: "broken.json", Err: errors.New("unexpected end of JSON input")}, + {File: "broken.json", Kind: runner.ErrorKindInvalidJSON, Err: errors.New("unexpected end of JSON input")}, }, } } diff --git a/report/json.go b/report/json.go index 9cdf3a2..e0360c9 100644 --- a/report/json.go +++ b/report/json.go @@ -12,7 +12,7 @@ import ( // envelope shape below), independent of the product and trace-schema versions. // Bump the minor for additive fields, the major for a breaking change. See // docs/output.md. -const OutputVersion = "1.0" +const OutputVersion = "1.1" // Meta is the context the caller supplies for the JSON envelope: the versions to // stamp, the number of files considered, and the generation time. GeneratedAt is @@ -26,6 +26,7 @@ type Meta struct { type fileErrorJSON struct { File string `json:"file"` + Kind string `json:"kind"` Error string `json:"error"` } @@ -57,7 +58,7 @@ func JSON(resp *runner.Response, meta Meta) (string, error) { env.Results = []*evaluator.Evaluation{} } for _, fe := range resp.FileErrors { - env.Errors = append(env.Errors, fileErrorJSON{File: fe.File, Error: fe.Err.Error()}) + env.Errors = append(env.Errors, fileErrorJSON{File: fe.File, Kind: string(fe.Kind), Error: fe.Err.Error()}) } data, err := json.MarshalIndent(env, "", " ") diff --git a/report/testdata/json.golden b/report/testdata/json.golden index 6c23ed9..e32f601 100644 --- a/report/testdata/json.golden +++ b/report/testdata/json.golden @@ -1,5 +1,5 @@ { - "outputVersion": "1.0", + "outputVersion": "1.1", "trazoVersion": "0.2.0", "traceSchemaVersion": "0.1.0", "generatedAt": "2026-08-07T12:00:00Z", @@ -51,6 +51,7 @@ "errors": [ { "file": "broken.json", + "kind": "invalid_json", "error": "unexpected end of JSON input" } ] diff --git a/report/testdata/text.golden b/report/testdata/text.golden index 2ef862c..dc6a153 100644 --- a/report/testdata/text.golden +++ b/report/testdata/text.golden @@ -8,4 +8,4 @@ run-ok (data_extractor) clean Summary: 2 runs, 1 bad, 2 neutral, 0 good, 1 file error File errors: - broken.json: unexpected end of JSON input + broken.json [invalid_json]: unexpected end of JSON input diff --git a/report/text.go b/report/text.go index 0d622b3..37af7d2 100644 --- a/report/text.go +++ b/report/text.go @@ -38,7 +38,7 @@ func Text(resp *runner.Response) string { if len(resp.FileErrors) > 0 { b.WriteString("\nFile errors:\n") for _, fe := range resp.FileErrors { - fmt.Fprintf(&b, " %s: %s\n", fe.File, oneline(fe.Err.Error())) + b.WriteString(fileErrorLine(fe)) } } return b.String() @@ -140,6 +140,16 @@ func oneline(s string) string { return strings.ReplaceAll(s, "\n", "; ") } +// fileErrorLine renders one file error, tagging it with its kind when known so +// the reader can tell a parse error from an invalid trace at a glance. +func fileErrorLine(fe runner.FileError) string { + msg := oneline(fe.Err.Error()) + if fe.Kind != "" { + return fmt.Sprintf(" %s [%s]: %s\n", fe.File, fe.Kind, msg) + } + return fmt.Sprintf(" %s: %s\n", fe.File, msg) +} + func plural(n int, unit string) string { if n == 1 { return fmt.Sprintf("%d %s", n, unit) diff --git a/report/validate.go b/report/validate.go index ba44479..49709a7 100644 --- a/report/validate.go +++ b/report/validate.go @@ -22,7 +22,7 @@ func ValidateSummary(resp *runner.Response, total int) string { var b strings.Builder fmt.Fprintf(&b, "Validated %d file(s): %d valid, %d invalid.\n", total, valid, invalid) for _, fe := range resp.FileErrors { - fmt.Fprintf(&b, " %s: %s\n", fe.File, cell(fe.Err.Error())) + b.WriteString(fileErrorLine(fe)) } return b.String() } diff --git a/runner/runner.go b/runner/runner.go index d490d98..4053b57 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -16,8 +16,22 @@ type Runner struct { evals []evaluator.Evaluator } +// ErrorKind classifies why a file could not be evaluated, so callers (the CLI, +// the JSON output, CI) can react by category instead of matching on message +// strings. +type ErrorKind string + +const ( + ErrorKindReadFile ErrorKind = "read_file" // the file could not be read + ErrorKindInvalidJSON ErrorKind = "invalid_json" // the bytes are not valid JSON + ErrorKindInvalidTrace ErrorKind = "invalid_trace" // JSON parsed but failed Run.Validate + ErrorKindEvaluator ErrorKind = "evaluator" // an evaluator returned an error + ErrorKindCanceled ErrorKind = "canceled" // the context was canceled before processing +) + type FileError struct { File string + Kind ErrorKind Err error } @@ -132,28 +146,28 @@ func (r *Runner) processFile(ctx context.Context, path string) fileResult { var res fileResult if err := ctx.Err(); err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindCanceled, Err: err}) return res } fileBytes, err := os.ReadFile(path) if err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindReadFile, Err: err}) return res } run, err := trajectory.LoadRun(fileBytes) if err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindInvalidJSON, Err: err}) return res } if err := run.Validate(); err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindInvalidTrace, Err: err}) return res } for _, judge := range r.evals { eval, err := judge.EvaluateRun(ctx, run) if err != nil { - res.errs = append(res.errs, FileError{File: path, Err: err}) + res.errs = append(res.errs, FileError{File: path, Kind: ErrorKindEvaluator, Err: err}) continue } // Carry the agent name for human-facing output; evaluators only set RunID. diff --git a/runner/runner_test.go b/runner/runner_test.go index 938a92b..5900d4a 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -37,9 +37,15 @@ func TestRunner_Run(t *testing.T) { if resp.FileErrors[0].File != wantBroken { t.Errorf("expected first file error on %s, got %s", wantBroken, resp.FileErrors[0].File) } + if resp.FileErrors[0].Kind != ErrorKindInvalidJSON { + t.Errorf("broken.json: expected kind %q, got %q", ErrorKindInvalidJSON, resp.FileErrors[0].Kind) + } if resp.FileErrors[1].File != wantInvalid { t.Errorf("expected second file error on %s, got %s", wantInvalid, resp.FileErrors[1].File) } + if resp.FileErrors[1].Kind != ErrorKindInvalidTrace { + t.Errorf("invalid_run.json: expected kind %q, got %q", ErrorKindInvalidTrace, resp.FileErrors[1].Kind) + } } func TestCollectFiles_NonRecursiveSkipsSubdirs(t *testing.T) { From 46ed3a2dc9ce10fd9845495dae5536bac708b024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 22:24:29 -0400 Subject: [PATCH 06/11] feat(cli): -verbose operational metrics (0.2.0) Add a -verbose flag that prints one metrics line to stderr after a run (loaded/valid/invalid/evaluated/duration), so batch and CI runs are observable without parsing the results. A trace-evaluation tool should itself be observable. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 +++ cmd/trazo/cli_test.go | 11 +++++++++++ cmd/trazo/main.go | 20 ++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c50193d..f9bba57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ product version. The trace **schema** version is tracked separately; see - Black-box CLI tests (`cmd/trazo/cli_test.go`) that build the binary and assert exit codes (0/1/2), each output format, invalid flags, missing paths, empty directories, and config/flag precedence. +- `-verbose` flag: prints operational metrics to stderr after a run + (`loaded=N valid=V invalid=I evaluated=E duration=Xms`), so batch runs are + observable without parsing the results. - Typed file errors: `runner.FileError` now carries an `ErrorKind` (`read_file`, `invalid_json`, `invalid_trace`, `evaluator`, `canceled`), surfaced as `errors[].kind` in the JSON output (bumped to `outputVersion` 1.1, diff --git a/cmd/trazo/cli_test.go b/cmd/trazo/cli_test.go index eb18e0c..6754abe 100644 --- a/cmd/trazo/cli_test.go +++ b/cmd/trazo/cli_test.go @@ -192,6 +192,17 @@ func TestCLI_EmptyDirectory(t *testing.T) { } } +func TestCLI_Verbose(t *testing.T) { + r := runCLI(t, "-verbose", "-dir", "../../testdata/runs") + if !strings.Contains(r.stderr, "loaded=") || !strings.Contains(r.stderr, "duration=") { + t.Errorf("expected operational metrics on stderr, got:\n%s", r.stderr) + } + // testdata/runs has 2 valid and 2 invalid fixtures. + if !strings.Contains(r.stderr, "valid=2") || !strings.Contains(r.stderr, "invalid=2") { + t.Errorf("unexpected metric counts, got:\n%s", r.stderr) + } +} + // TestCLI_FlagOverridesConfig pins the precedence rule: a config sets a lax cost // threshold (no finding), and an explicit -max-step-cost flag overrides it (a // finding appears). Both runs stay exit 0 because a cost finding is neutral. diff --git a/cmd/trazo/main.go b/cmd/trazo/main.go index 994430f..6f14639 100644 --- a/cmd/trazo/main.go +++ b/cmd/trazo/main.go @@ -41,6 +41,7 @@ func main() { configPath := flag.String("config", "", "path to a JSON evaluator policy file (see docs/config.md)") dir := flag.String("dir", "./testdata/runs", "directory of traces to scan when no PATH is given") recursive := flag.Bool("recursive", false, "descend into subdirectories when PATH is a directory") + verbose := flag.Bool("verbose", false, "print operational metrics (loaded/valid/invalid/evaluated/duration) to stderr") validate := flag.Bool("validate", false, "only check that traces load and pass structural validation; skip evaluators") asJSON := flag.Bool("json", false, "print results as JSON (alias for -format json)") format := flag.String("format", "text", "output format: text, json, or md") @@ -143,7 +144,16 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() + start := time.Now() resp := runner.NewRunner(evaluators).RunFiles(ctx, files) + elapsed := time.Since(start) + + if *verbose { + invalid := distinctInvalidFiles(resp) + evaluated := report.Summarize(resp, len(files)).Runs + log.Printf("loaded=%d valid=%d invalid=%d evaluated=%d duration=%s", + len(files), len(files)-invalid, invalid, evaluated, elapsed.Round(time.Millisecond)) + } meta := report.Meta{ TrazoVersion: Version, @@ -197,6 +207,16 @@ func render(out string, validate bool, resp *runner.Response, meta report.Meta) } } +// distinctInvalidFiles counts the unique files that produced at least one error, +// so a file with several evaluator errors is still counted once. +func distinctInvalidFiles(resp *runner.Response) int { + seen := map[string]bool{} + for _, fe := range resp.FileErrors { + seen[fe.File] = true + } + return len(seen) +} + // splitCSV parses a comma-separated flag value into a trimmed, non-empty slice, // returning nil for an empty value so the evaluator falls back to its default. func splitCSV(s string) []string { From 54f661476fb201c85408011cd2f6aa27aeef318a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 22:26:06 -0400 Subject: [PATCH 07/11] ci: make schema conformance an explicit gate (0.2.0) Add a dedicated schema job that runs the Go schema-sync tests and validates the fixtures and emitter output against the published JSON Schema, so the schema is an active part of the contract rather than just documentation. Add a Go test that the shipped example config always loads, keeping the docs from drifting. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 25 +++++++++++++++++++++++++ CHANGELOG.md | 3 +++ config/config_test.go | 12 ++++++++++++ 3 files changed, 40 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b5e659..967aafb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,31 @@ jobs: working-directory: agents/langgraph-reference run: pytest -q + schema: + name: schema conformance + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + + - name: JSON Schema stays in sync with the Go types + run: go test ./trajectory/ -run "TestSchema" -v + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install the schema validator + working-directory: agents/langgraph-reference + run: pip install -r requirements.txt + + - name: Fixtures and emitter output conform to the published JSON Schema + working-directory: agents/langgraph-reference + run: pytest -q tests/test_schema.py + trazo-gate: name: trazo gate demo runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index f9bba57..d27e306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ product version. The trace **schema** version is tracked separately; see - Black-box CLI tests (`cmd/trazo/cli_test.go`) that build the binary and assert exit codes (0/1/2), each output format, invalid flags, missing paths, empty directories, and config/flag precedence. +- A dedicated `schema conformance` CI job that runs the Go schema-sync tests and + validates fixtures and emitter output against the published JSON Schema, plus a + test that the shipped example config always loads. - `-verbose` flag: prints operational metrics to stderr after a run (`loaded=N valid=V invalid=I evaluated=E duration=Xms`), so batch runs are observable without parsing the results. diff --git a/config/config_test.go b/config/config_test.go index 83cca3d..df4a1b5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -105,6 +105,18 @@ func TestLoad_RejectsNegativeThresholds(t *testing.T) { } } +// TestExampleConfigLoads keeps the shipped example policy honest: it must always +// load and build under the current loader, so the docs never drift from the code. +func TestExampleConfigLoads(t *testing.T) { + cfg, err := Load(filepath.Join("..", "docs", "trazo.config.example.json")) + if err != nil { + t.Fatalf("example config failed to load: %v", err) + } + if _, err := cfg.Build(noJudge); err != nil { + t.Fatalf("example config failed to build: %v", err) + } +} + func TestBuild_JudgeEnabledCallsFactory(t *testing.T) { cfg := Default() cfg.Evaluators.LLMJudge.Enabled = true From 627c4011db1e25ad3fffb44d4feceb2beab002b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Fri, 7 Aug 2026 22:39:44 -0400 Subject: [PATCH 08/11] test(evaluator): complex realistic fixture across all evaluators (0.2.0) Add testdata/complex_run.json: one trace that interleaves two tool calls paired by id (no orphans), a tool error, a high-cost step, and a node visited enough times to look like a loop. An integration test runs the full structural evaluator set over it and asserts how the work divides (tool error -> bad, loop -> bad, cost overrun -> neutral, terminal end -> no node finding). Co-Authored-By: Claude Opus 4.8 (1M context) --- evaluator/integration_test.go | 74 +++++++++++++++++++++++++++++++ testdata/complex_run.json | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 evaluator/integration_test.go create mode 100644 testdata/complex_run.json diff --git a/evaluator/integration_test.go b/evaluator/integration_test.go new file mode 100644 index 0000000..5ec003b --- /dev/null +++ b/evaluator/integration_test.go @@ -0,0 +1,74 @@ +package evaluator + +import ( + "context" + "os" + "testing" + + "github.com/Cro22/trazo/trajectory" +) + +// TestComplexRun_AllEvaluators exercises a single realistic trace against the +// full structural evaluator set: interleaved tool calls paired by id (no +// orphans), a tool error, a high-cost step, and a node visited enough times to +// look like a loop. It documents how the evaluators divide the work on one trace. +func TestComplexRun_AllEvaluators(t *testing.T) { + data, err := os.ReadFile("../testdata/complex_run.json") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + run, err := trajectory.LoadRun(data) + if err != nil { + t.Fatalf("load run: %v", err) + } + if err := run.Validate(); err != nil { + t.Fatalf("fixture must be a valid trace: %v", err) + } + + ctx := context.Background() + + // tool_calls: exactly one bad (the fetch 429), and no orphans, because the + // interleaved search/fetch calls each pair with their result by id. + tc, _ := (&ToolCallEvaluator{}).EvaluateRun(ctx, run) + if got := countBy(tc, JudgmentBad); got != 1 { + t.Errorf("tool_calls: expected 1 bad, got %d (%+v)", got, tc.Findings) + } + if got := countBy(tc, JudgmentNeutral); got != 0 { + t.Errorf("tool_calls: expected 0 neutral (no orphans), got %d (%+v)", got, tc.Findings) + } + + // loops: the node "retry" is visited three times, hitting the default limit. + lp, _ := (&LoopEvaluator{}).EvaluateRun(ctx, run) + if got := countBy(lp, JudgmentBad); got != 1 { + t.Errorf("loops: expected 1 bad, got %d (%+v)", got, lp.Findings) + } + + // cost_latency: the llm step costs 0.10, over the default per-step budget. A + // cost overrun is neutral, not bad: it needs review but is not necessarily the + // agent's fault. + cl, _ := (&CostLatencyEvaluator{ + MaxStepCost: DefaultMaxStepCost, + MaxStepLatencyMs: DefaultMaxStepLatencyMs, + MaxRunCost: DefaultMaxRunCost, + MaxRunLatencyMs: DefaultMaxRunLatencyMs, + }).EvaluateRun(ctx, run) + if countBy(cl, JudgmentNeutral) < 1 { + t.Errorf("cost_latency: expected at least 1 neutral, got %+v", cl.Findings) + } + + // node_transitions: the run ends at the terminal node "end", so nothing here. + nt, _ := (&NodeTransitionEvaluator{}).EvaluateRun(ctx, run) + if len(nt.Findings) != 0 { + t.Errorf("node_transitions: expected 0 findings (ends at terminal), got %+v", nt.Findings) + } +} + +func countBy(e *Evaluation, j Judgment) int { + n := 0 + for _, f := range e.Findings { + if f.Judgment == j { + n++ + } + } + return n +} diff --git a/testdata/complex_run.json b/testdata/complex_run.json new file mode 100644 index 0000000..f21b860 --- /dev/null +++ b/testdata/complex_run.json @@ -0,0 +1,82 @@ +{ + "id": "run-complex-01", + "agent": "researcher", + "version": "0.1.0", + "startTime": "2026-08-07T09:00:00Z", + "endTime": "2026-08-07T09:00:12Z", + "steps": [ + { + "node": "start", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:00Z", + "durationMs": 3 + }, + { + "llm": "gemini-2.5-flash", + "type": "llm_call", + "timestamp": "2026-08-07T09:00:01Z", + "input": {"system": "plan the research", "user": "compare two libraries"}, + "output": "I will search and fetch in parallel.", + "cost": 0.10, + "inputTokens": 900, + "outputTokens": 120, + "durationMs": 1200 + }, + { + "tool": "search", + "toolCallId": "call-search-1", + "type": "tool_call", + "timestamp": "2026-08-07T09:00:02Z", + "input": {"q": "library A benchmarks"}, + "durationMs": 400 + }, + { + "tool": "fetch", + "toolCallId": "call-fetch-1", + "type": "tool_call", + "timestamp": "2026-08-07T09:00:03Z", + "input": {"url": "https://example.com/a"}, + "durationMs": 500 + }, + { + "tool": "fetch", + "toolCallId": "call-fetch-1", + "type": "tool_result", + "timestamp": "2026-08-07T09:00:04Z", + "durationMs": 500, + "error": "429 Too Many Requests" + }, + { + "tool": "search", + "toolCallId": "call-search-1", + "type": "tool_result", + "timestamp": "2026-08-07T09:00:05Z", + "output": {"hits": 3}, + "durationMs": 400 + }, + { + "node": "retry", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:06Z", + "durationMs": 2 + }, + { + "node": "retry", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:07Z", + "durationMs": 2 + }, + { + "node": "retry", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:08Z", + "durationMs": 2 + }, + { + "node": "end", + "type": "node_transition", + "timestamp": "2026-08-07T09:00:10Z", + "durationMs": 3 + } + ] +} From bfdf82e4aad4431dabbdb6b7f41d53a456cdba1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Sun, 9 Aug 2026 09:52:25 -0400 Subject: [PATCH 09/11] test(schema): cover complex_run.json in Python conformance suite (0.2.0) The M8 integration fixture testdata/complex_run.json is consumed by the Go integration test; add it to VALID_FIXTURES so the published JSON Schema is proven to accept it too, closing the cross-language loop for that fixture. Co-Authored-By: Claude Opus 4.8 (1M context) --- agents/langgraph-reference/tests/test_schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/agents/langgraph-reference/tests/test_schema.py b/agents/langgraph-reference/tests/test_schema.py index 15f2259..a0be645 100644 --- a/agents/langgraph-reference/tests/test_schema.py +++ b/agents/langgraph-reference/tests/test_schema.py @@ -32,6 +32,7 @@ "testdata/sample_run_missing_result.json", "testdata/ci/clean/triage_clean.json", "testdata/ci/failing/tool_error.json", + "testdata/complex_run.json", ] From 0b1e8ddbbaa00d6d1c06bd191a7e355c210af04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Sun, 9 Aug 2026 09:54:23 -0400 Subject: [PATCH 10/11] docs: privacy/data-handling and LLM-judge guides (0.2.0) docs/security.md: what a trace holds (opaque input/output/error payloads), the one component that sends data off-machine (the LLM judge, final output only), how to disable network egress, redaction at emit time, and not committing private traces. docs/llm-judge.md: the judge's opt-in flags and config keys, what it grades (last llm_call output, once), determinism (temperature 0, capped tokens), the timeout/cancellation model, no internal retries, and the deliberate split between a malformed verdict (neutral finding) and a transport failure (errors[].kind=evaluator, structural evaluators still run). Notes network-free testing via the fakeJudge Completion. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/llm-judge.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++ docs/security.md | 86 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 docs/llm-judge.md create mode 100644 docs/security.md diff --git a/docs/llm-judge.md b/docs/llm-judge.md new file mode 100644 index 0000000..0e9c12e --- /dev/null +++ b/docs/llm-judge.md @@ -0,0 +1,98 @@ +# LLM-as-judge evaluator (`-llm-judge`) + +Most trazo evaluators are structural: they reason about the shape of a trace +(orphan tool calls, loops, cost, latency) without ever reading the semantics of +what the agent produced. The LLM judge is the one exception. It asks a model to +grade the agent's final answer for correctness and usefulness, turning a +subjective "was this a good run?" into a `good` / `neutral` / `bad` judgment +alongside the structural findings. + +Because it makes a network call and costs money, it is **opt-in** and is not part +of the default evaluator set. + +## Enabling it + +```sh +export GEMINI_API_KEY=... # never commit this +go run ./cmd/trazo -llm-judge testdata/complex_run.json +``` + +Flags: + +| Flag | Default | Meaning | +| --- | --- | --- | +| `-llm-judge` | off | enable the judge (requires `GEMINI_API_KEY`) | +| `-judge-model` | `gemini-2.5-flash` | model passed to the Gemini API | + +The same two settings exist in the config file under `evaluators.llm_judge` +(`enabled`, `model`), so a policy file can pin the judge on for a whole team. As +everywhere else, an explicit `-llm-judge` flag overrides the config value. + +## What it judges, and how + +The judge looks at the run's **last `llm_call` output only**, once per run. If a +trace has no `llm_call` step there is nothing to grade and the evaluator returns +no findings. It does not re-grade intermediate reasoning or tool output; the +final answer is the thing a user sees, so it is the thing that gets graded. + +The prompt asks the model to reply with a single JSON object: + +```json +{"judgment": "good|neutral|bad", "score": 0.0-1.0, "comment": "short reason"} +``` + +- `good` maps to `JudgmentGood`, `bad` to `JudgmentBad`, `neutral` to + `JudgmentNeutral` (trazo's own taxonomy). +- Surrounding prose or code fences are tolerated: the parser extracts the first + `{ ... }` span from the reply. + +## Determinism and cost discipline + +- **Temperature 0.** The request pins `temperature: 0`, so for a given model and + input the verdict is as stable as the provider allows. It is still an LLM, so + treat verdicts as a strong signal, not a hard oracle. +- **Capped output.** `maxOutputTokens: 256`. The verdict is tiny; there is no + reason to pay for more. +- **One call per run.** The judge never loops or retries internally. One trace is + one completion. Cost scales linearly with the number of runs you judge, not + with trace size. + +## Timeouts, cancellation, and retries + +- **Timeout.** Each judge call is bounded by a timeout (default **30s**, + overridable on the evaluator). The bound is layered on top of the caller's + context, so a run cancelled with Ctrl+C or by a CI timeout also aborts the + in-flight network call promptly. +- **No retries.** A failed call is not retried. This is deliberate: retries hide + provider instability and multiply cost. If you need retry semantics, wrap the + provider at the HTTP layer. + +## What happens when the judge fails + +The two failure modes are treated differently on purpose. + +1. **A malformed but returned verdict** (the model replied, but the JSON is + missing, unparseable, or carries an unknown judgment word) becomes a single + `JudgmentNeutral` finding whose comment quotes the offending reply. The run is + still evaluated; you just get "the judge could not make up its mind" instead + of a grade. + +2. **A transport-level failure** (no API key, network error, non-200 status, + timeout, empty candidate list) is surfaced as a run **error**, not a finding. + In the CLI's JSON output it appears in `errors[]` with `kind: "evaluator"`; + in text output it is reported as a failed evaluation. Crucially, the failure + is isolated to the judge: the structural evaluators for that same file still + run and still produce their findings. A judge outage degrades the report, it + does not abort it. + +So: an ambiguous answer is neutral; an unreachable provider is an error. Neither +one silently passes a bad run. + +## Testing without a network + +The judge depends only on a small `Completion` interface +(`Complete(ctx, prompt) (string, error)`), so tests inject a fake instead of +calling Gemini. See `evaluator/judge_test.go`: `fakeJudge` returns a canned reply +(or a canned error) and lets the tests exercise every branch above, including the +Gemini HTTP client against an `httptest` server. Running `go test ./evaluator/` +never touches the network and needs no API key. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..a7cf038 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,86 @@ +# Privacy and data handling + +Trazo evaluates agent traces. Those traces are a faithful recording of what an +agent did, which means they can contain whatever the agent read, wrote, or was +told. This document describes exactly what a trace holds, where that data can +leave your machine, and how to keep private data private. + +## What a trace file contains + +A trace is a JSON `Run` with a list of `Step`s. The fields that can carry +sensitive content are: + +| Field | Present on | Can contain | +| --- | --- | --- | +| `input` | any step | the LLM prompt, tool arguments, user-supplied text | +| `output` | any step | the LLM completion, tool results, fetched documents | +| `error` | any step | error strings, which sometimes echo inputs or paths | +| `agent`, `llm`, `tool`, `node` | identifiers | model names, internal tool and node names | + +`input` and `output` are opaque payloads (`json.RawMessage`): trazo does not +inspect or constrain their contents, so anything the agent handled can end up +there verbatim. Prompts, retrieved passages, API responses, file contents, PII +in a user request: if the agent saw it, the trace can hold it. + +The remaining fields (`cost`, `inputTokens`, `outputTokens`, `durationMs`, +timestamps, IDs, step `type`) are operational metadata and are not sensitive on +their own. + +## Where data goes + +By default, trazo is **fully local**. The core loads trace files from disk, runs +the structural evaluators (tool calls, loops, cost, latency, nodes) entirely +in-process, and writes findings to stdout. No trace data leaves the machine. The +core is standard-library only and opens no network connections in this mode. + +There is exactly **one** component that sends trace data off the machine: the +[LLM judge](llm-judge.md). When you pass `-llm-judge`, the agent's final +`llm_call` **output** is placed in a prompt and sent to the Gemini API for +grading. That output text leaves your machine and is subject to the provider's +data-handling terms. + +Nothing else is transmitted: not tool inputs, not intermediate steps, not the +whole trace. Only the final answer, and only when the judge is explicitly +enabled. + +## How to disable network egress + +- **Do not pass `-llm-judge`** (and leave `evaluators.llm_judge.enabled` false or + absent in any config file). This is the default. With the judge off, trazo + makes no outbound calls at all. +- If you want a hard guarantee, run trazo on a host with no network access, or + omit `GEMINI_API_KEY` from the environment. Without the key the judge cannot be + constructed and the run fails fast rather than sending anything. + +## Redaction + +Trazo does not redact for you; it evaluates whatever it is given. Redaction +belongs at trace-creation time, in whatever emits the trace (for the reference +agent, that is the Python `trazo_emitter`). Strip or mask secrets, credentials, +and PII from `input`, `output`, and `error` payloads before writing the file. +Because payloads are opaque to the core, a redacted trace evaluates exactly like +an unredacted one; the structural evaluators care about shape, not content. + +If you enable the judge, remember that redaction of the final output directly +changes what is sent to Gemini. A masked answer is a masked prompt. + +## Do not commit private traces + +Real agent runs make excellent test fixtures, which is precisely why they leak. +Treat trace files from real workloads as you would logs or a database dump: + +- Keep them out of the repository. Add your traces directory to `.gitignore`. +- The trace fixtures that **are** committed under `testdata/` and + `agents/langgraph-reference/docs/` are hand-authored or produced against public + repositories with synthetic content. Keep it that way: do not replace them with + captures from private runs. +- Before sharing a trace for a bug report, open it and check `input` / `output` / + `error` on every step. A trace is human-readable JSON; there is no excuse for + not looking. + +## API keys + +`GEMINI_API_KEY` is read from the environment and is the only secret trazo +consumes. It is never written to a trace, never logged, and never printed. Never +hardcode it, never commit it, and never paste it into a config file (the config +schema has no field for it, by design). From ff95d6d22062f3a52d837683e19a1a95bc257c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Sun, 9 Aug 2026 10:12:57 -0400 Subject: [PATCH 11/11] build: packaging and release plumbing for 0.2.0 (M10) - LICENSE: MIT. - Makefile: build, check (vet+test), gate (local mirror of the CI evaluator gate), install, agent-test, and a cross-compiled `dist` target that produces per-platform archives (.tar.gz / .zip) plus a sha256 checksums.txt. - .github/workflows/release.yml: on a `v*` tag, guard that the tag matches the compiled-in const Version, vet+test, `make dist`, then publish the binaries to a GitHub release via `gh release create`. No third-party actions; stdlib-only ethos extended to CI. - README: status/release/go-version/license badges, an Install section (`go install`, from-source, prebuilt binaries with checksum verification), and a Core documentation section linking config/output/versioning/llm-judge/security. - .gitignore: ignore /bin and a bare ./trazo so ad-hoc builds do not dirty the vcs stamp; dist/ is already ignored. - CHANGELOG: record the integration fixture, the two new docs, and this packaging. Verified: cross-compiles clean for linux/darwin/windows (amd64/arm64) to static binaries; `trazo version` and the gate exit codes (0 clean, 1 bad) both correct. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release.yml | 51 +++++++++++++++++++++++ .gitignore | 4 ++ CHANGELOG.md | 16 +++++++ LICENSE | 21 ++++++++++ Makefile | 78 +++++++++++++++++++++++++++++++++++ README.md | 41 ++++++++++++++++++ 6 files changed, 211 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 LICENSE create mode 100644 Makefile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4032b27 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: Release + +# Cut a release by pushing a semver tag, e.g. `git tag v0.2.0 && git push --tags`. +# The tag must match the compiled-in const Version in cmd/trazo/version.go, or the +# guard below fails the build before anything is published. +on: + push: + tags: ["v*"] + +permissions: + contents: write # required to create the GitHub release and upload artifacts + +jobs: + release: + name: build and publish binaries + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # tags and full history so build info is stamped correctly + + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + + - name: Tag matches the compiled-in version + run: | + tag="${GITHUB_REF_NAME#v}" + code="$(sed -n 's/.*const Version = "\([^"]*\)".*/\1/p' cmd/trazo/version.go)" + echo "tag=$tag code=$code" + if [ "$tag" != "$code" ]; then + echo "::error::tag v$tag does not match const Version $code in cmd/trazo/version.go" + exit 1 + fi + + - name: Build, vet, test before releasing + run: | + go vet ./... + go test ./... + + - name: Cross-compile release archives + run: make dist VERSION="${GITHUB_REF_NAME}" + + - name: Create the GitHub release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "${GITHUB_REF_NAME}" \ + --title "${GITHUB_REF_NAME}" \ + --generate-notes \ + dist/* diff --git a/.gitignore b/.gitignore index c4123ba..a98ef3f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ # Test binary, built with `go test -c` *.test +# trazo build artifacts (Makefile `build`/`install` and ad-hoc `go build -o trazo`) +/bin/ +/trazo + # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/CHANGELOG.md b/CHANGELOG.md index d27e306..fd0b0b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,22 @@ product version. The trace **schema** version is tracked separately; see (`read_file`, `invalid_json`, `invalid_trace`, `evaluator`, `canceled`), surfaced as `errors[].kind` in the JSON output (bumped to `outputVersion` 1.1, additive) and tagged in the text output. +- A realistic integration fixture (`testdata/complex_run.json`) and test that + runs the full structural evaluator set over one interleaved trace, asserting + how the work divides across judgments; it is also covered by the Python schema + conformance suite. +- [docs/llm-judge.md](docs/llm-judge.md): the opt-in LLM-as-judge evaluator, its + determinism (temperature 0, capped tokens), timeout and cancellation model, and + the split between a malformed verdict (neutral finding) and a transport failure + (a run error; structural evaluators still run). +- [docs/security.md](docs/security.md): what a trace holds, the single component + that sends data off-machine (the judge, final output only), how to disable + network egress, redaction at emit time, and not committing private traces. +- Packaging and release: `LICENSE` (MIT), a `Makefile` (build, check, gate, + install, cross-compiled `dist` archives with checksums), a tag-triggered GitHub + Actions release workflow that guards the tag against the compiled-in version and + publishes binaries for Linux, macOS, and Windows (amd64/arm64), and a README + with badges, `go install` instructions, and links to the core docs. - This changelog. ### Changed diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ecdab74 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Jesús Núñez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..70e4ea5 --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +# trazo build and release tasks. +# +# The Go core is standard-library only, so these targets need nothing but the Go +# toolchain (plus tar/zip/sha256sum for `dist`, which are present on CI runners +# and most Unix hosts). On Windows, run under Git Bash or WSL. + +BIN := trazo +CMD := ./cmd/trazo +BINDIR := bin +DISTDIR := dist + +# Version stamped into archive names. The binary itself reports the compiled-in +# const Version plus git metadata from the Go toolchain (see cmd/trazo/version.go); +# this is only for naming the release artifacts. +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) + +# Release target matrix (GOOS/GOARCH). +PLATFORMS := linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 + +.DEFAULT_GOAL := build +.PHONY: build check test vet fmt install gate agent-test dist clean + +build: ## Build the CLI into ./bin + go build -trimpath -o $(BINDIR)/$(BIN) $(CMD) + +check: vet test ## Full Go gate: vet then test + go build ./... + +test: ## Run the Go test suite + go test ./... + +vet: ## Run go vet + go vet ./... + +fmt: ## Format all Go sources + go fmt ./... + +install: ## Install the CLI into GOBIN (go install) + go install $(CMD) + +# Mirror the CI trazo-gate job locally: clean traces pass, bad traces must fail. +gate: build ## Demonstrate the evaluator gate on the CI fixtures + $(BINDIR)/$(BIN) -dir testdata/ci/clean + @echo "clean traces passed (exit 0)" + @if $(BINDIR)/$(BIN) -dir testdata/ci/failing; then \ + echo "gate did not fail on a bad finding" >&2; exit 1; \ + fi + @echo "bad traces correctly failed the gate (exit non-zero)" + +# Run the Python reference-agent tests. Requires the venv from +# agents/langgraph-reference (see the README quickstart). +agent-test: ## Run the reference-agent pytest suite + cd agents/langgraph-reference && pytest -q + +# Cross-compile release archives plus a checksums file into ./dist. Each archive +# holds one static binary; unix targets are .tar.gz, windows is .zip. +dist: ## Build release archives for every target platform + rm -rf $(DISTDIR) + mkdir -p $(DISTDIR) + @for p in $(PLATFORMS); do \ + os=$${p%/*}; arch=$${p#*/}; \ + ext=; [ $$os = windows ] && ext=.exe; \ + echo "building $$os/$$arch"; \ + CGO_ENABLED=0 GOOS=$$os GOARCH=$$arch \ + go build -trimpath -o $(DISTDIR)/$(BIN)$$ext $(CMD) || exit 1; \ + base=$(BIN)_$(VERSION)_$$os_$$arch; \ + if [ $$os = windows ]; then \ + (cd $(DISTDIR) && zip -q $$base.zip $(BIN)$$ext && rm $(BIN)$$ext); \ + else \ + (cd $(DISTDIR) && tar czf $$base.tar.gz $(BIN)$$ext && rm $(BIN)$$ext); \ + fi; \ + done + @(cd $(DISTDIR) && sha256sum * > checksums.txt) + @echo "artifacts in $(DISTDIR):" + @ls -1 $(DISTDIR) + +clean: ## Remove build and release artifacts + rm -rf $(BINDIR) $(DISTDIR) $(BIN) $(BIN).exe diff --git a/README.md b/README.md index 1eb3697..fe1b76f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,35 @@ # trazo +[![CI](https://github.com/Cro22/trazo/actions/workflows/ci.yml/badge.svg)](https://github.com/Cro22/trazo/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/Cro22/trazo?sort=semver)](https://github.com/Cro22/trazo/releases) +[![Go version](https://img.shields.io/github/go-mod/go-version/Cro22/trazo)](go.mod) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) + Trajectory evaluation for LLM agents. trazo ingests agent run traces as JSON and runs evaluators over them, producing findings with a severity taxonomy. The core is written in Go; a Python LangGraph reference agent proves the framework end to end: real agent produces real traces, and the unmodified Go core evaluates them. +## Install + +The CLI is a single self-contained binary with no runtime dependencies. + +```bash +# With the Go toolchain (installs into $GOBIN): +go install github.com/Cro22/trazo/cmd/trazo@latest + +# Or from source: +git clone https://github.com/Cro22/trazo && cd trazo +make build # -> ./bin/trazo (or: go build -o trazo ./cmd/trazo) +``` + +Prebuilt binaries for Linux, macOS, and Windows (amd64 and arm64) are attached to +each [GitHub release](https://github.com/Cro22/trazo/releases). Verify a download +against the release's `checksums.txt`, then put the binary on your `PATH`. + +`trazo version` reports the product version, the supported trace schema version, +and the build's git commit and time. + ## The severity taxonomy Every finding carries one judgment (`evaluator/Evaluator.go`): @@ -62,6 +87,18 @@ directory with a bad finding (`testdata/ci/failing`) makes trazo exit non-zero. The Markdown report is written to the job summary. This is how trazo fails a build on a `JudgmentBad`. +## Core documentation + +- [docs/config.md](docs/config.md) — the `-config` evaluator policy file and + the defaults < config < flags precedence. +- [docs/output.md](docs/output.md) — the versioned JSON output envelope. +- [docs/versioning.md](docs/versioning.md) — product versus trace-schema + versioning and the release process. +- [docs/llm-judge.md](docs/llm-judge.md) — the opt-in LLM-as-judge evaluator: + determinism, timeouts, cost, and failure handling. +- [docs/security.md](docs/security.md) — what a trace holds, the one component + that sends data off-machine, and how to keep private traces private. + ## Reference agent (LangGraph) `agents/langgraph-reference/` is a GitHub-issue triage agent built with @@ -148,3 +185,7 @@ Model tier is a cheap flash model (`gemini-2.5-flash` by default) with temperature 0 and a low output-token cap. A full triage of one repo is about 2,400 tokens and costs roughly **$0.002** per run. Override the model with `--model` and bound the loop with `--iteration-cap`. + +## License + +MIT. See [LICENSE](LICENSE).