From 0dbd1420373f40db188e0bf058c4f7ad97390dce Mon Sep 17 00:00:00 2001 From: David Date: Wed, 15 Jul 2026 12:28:53 -0700 Subject: [PATCH 1/3] feat(eval): Terminal-Bench 2.0 integration via a Harbor installed-agent adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit integrations/terminal_bench/hi_agent.py implements Harbor's BaseInstalledAgent: uploads a Linux hi binary into each task container, runs one --plain --no-save --allow-unverified turn with the task instruction (exit code never fails the trial — Terminal-Bench scoring is outcome-only via the task's verifier), and feeds hi's --report usage/telemetry back into Harbor's per-trial accounting. scripts/terminal_bench.sh wraps the run (sample/full/task-glob modes, report artifacts downloaded per trial); docs/terminal-bench.md covers the containerized Linux build and cost caveats. Co-Authored-By: Claude Fable 5 --- docs/terminal-bench.md | 75 ++++++++++++++ integrations/__init__.py | 0 integrations/terminal_bench/__init__.py | 0 integrations/terminal_bench/hi_agent.py | 128 ++++++++++++++++++++++++ scripts/terminal_bench.sh | 51 ++++++++++ 5 files changed, 254 insertions(+) create mode 100644 docs/terminal-bench.md create mode 100644 integrations/__init__.py create mode 100644 integrations/terminal_bench/__init__.py create mode 100644 integrations/terminal_bench/hi_agent.py create mode 100755 scripts/terminal_bench.sh diff --git a/docs/terminal-bench.md b/docs/terminal-bench.md new file mode 100644 index 00000000..7d007791 --- /dev/null +++ b/docs/terminal-bench.md @@ -0,0 +1,75 @@ +# Running hi on Terminal-Bench 2.0 + +[Terminal-Bench](https://github.com/harbor-framework/terminal-bench) is the +standard benchmark for terminal coding agents; [Harbor](https://github.com/harbor-framework/harbor) +is its official harness. Each task runs in its own Docker environment; Harbor +installs the agent into the container, hands it the task instruction, and +scores the resulting container state with the task's own tests — outcome-only, +so intermediate behavior earns no credit. This is the setting studied in +"Failure as a Process" (arXiv:2607.09510), whose 21 model–scaffold baselines +span 19–45% pass rate. + +## One-time setup + +1. **Harbor** (needs Python ≥3.12 and Docker running): + + ```bash + uv tool install harbor + ``` + +2. **A Linux build of `hi`** matching the task-image architecture (aarch64 on + Apple Silicon Docker, x86_64 on most CI). The workspace's GPU crates don't + build in the container, but the binary crate builds alone; the easiest + reproducible path is a containerized build: + + ```bash + docker run --rm -v "$PWD":/src:ro -v "$PWD/target-linux":/out \ + rust:1-bookworm cargo build --release -p hi \ + --manifest-path /src/Cargo.toml --target-dir /out + export HI_AGENT_BINARY="$PWD/target-linux/release/hi" + ``` + +## Running + +```bash +export PIPENETWORK_API_KEY=... # or the provider your model needs +scripts/terminal_bench.sh sample # ~10 tasks, plumbing + cost check +scripts/terminal_bench.sh full # full dataset — mind the spend +scripts/terminal_bench.sh task 'git*' # tasks matching a glob +``` + +Model selection: `HI_TB_MODEL=/` (default +`pipenetwork/ipop/coder-balanced`). Known providers: `pipenetwork`, +`anthropic`, `openrouter`, `openai` — the adapter maps them to `hi --provider` +and forwards the matching API key from your environment into the container. + +Results land under `jobs/terminal-bench//` — per-trial directories +with the agent transcript (`hi.txt`), hi's own `--report` telemetry +(`hi-report.json`, including `repeated_verify_failures` and the trajectory +timeline), and Harbor's verifier verdicts. Harbor prints the aggregate pass +rate at the end of the job. + +## How the adapter works + +`integrations/terminal_bench/hi_agent.py` implements Harbor's +`BaseInstalledAgent`: + +- `install()` uploads `$HI_AGENT_BINARY` into the container at `/opt/hi/hi` + and symlinks it onto `PATH`. +- `run()` executes one `hi --plain --no-save --allow-unverified` turn with the + task instruction, `--report` pointed at Harbor's log directory, and + `XDG_CONFIG_HOME` isolated so a config file baked into a task image can + never shadow the model routing. The command never fails the trial on hi's + exit code — scoring is the verifier's job. +- `populate_context_post_run()` reads the downloaded report and fills Harbor's + token accounting plus a `hi_telemetry` metadata block. + +## Caveats + +- **Architecture**: the uploaded binary must match the task images. Harbor on + Apple Silicon uses aarch64 images by default; build accordingly. +- **Timeouts**: Terminal-Bench tasks carry their own generous agent timeouts; + hi's dynamic step caps apply within them. Use `--timeout-multiplier` on + `harbor run` for slow models. +- **Cost**: the full dataset is hundreds of agent turns. Always run `sample` + first and extrapolate spend before `full`. diff --git a/integrations/__init__.py b/integrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/integrations/terminal_bench/__init__.py b/integrations/terminal_bench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/integrations/terminal_bench/hi_agent.py b/integrations/terminal_bench/hi_agent.py new file mode 100644 index 00000000..9417e192 --- /dev/null +++ b/integrations/terminal_bench/hi_agent.py @@ -0,0 +1,128 @@ +"""Harbor (Terminal-Bench 2.0) installed-agent adapter for `hi`. + +Harbor spins up each task's Docker environment, calls ``install()`` to place +the agent inside it, runs the task instruction through ``run()``, then scores +the resulting container state with the task's own tests (outcome-only, like +the Terminal-Bench paper protocol). This adapter uploads a Linux build of +``hi`` (see docs/terminal-bench.md for producing one), runs it one-shot with +``--report``, and surfaces hi's token usage and trajectory telemetry back to +Harbor. + +Usage:: + + export HI_AGENT_BINARY=/path/to/linux/hi + export PIPENETWORK_API_KEY=... + harbor run -d terminal-bench@2.0 \ + --agent integrations.terminal_bench.hi_agent:HiAgent \ + -m pipenetwork/ipop/coder-balanced -n 4 +""" + +import json +import os +import shlex +from pathlib import Path +from typing import override + +from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +_REPORT = "/logs/agent/hi-report.json" +_OUTPUT = "/logs/agent/hi.txt" + +# Harbor model prefix (-m "/") → (hi --provider, key sources). +_PROVIDERS = { + "pipenetwork": ("pipenetwork", ["PIPENETWORK_API_KEY", "HI_API_KEY"]), + "anthropic": ("anthropic", ["ANTHROPIC_API_KEY", "HI_API_KEY"]), + "openrouter": ("openai", ["OPENROUTER_API_KEY", "HI_API_KEY"]), + "openai": ("openai", ["OPENAI_API_KEY", "HI_API_KEY"]), +} + + +class HiAgent(BaseInstalledAgent): + """Container-installed `hi` driven one-shot per task instruction.""" + + @staticmethod + @override + def name() -> str: + return "hi" + + @override + def get_version_command(self) -> str | None: + return "hi --version" + + @override + async def install(self, environment: BaseEnvironment) -> None: + binary = os.environ.get("HI_AGENT_BINARY", "") + if not binary or not Path(binary).is_file(): + raise ValueError( + "HI_AGENT_BINARY must point at a Linux build of `hi` matching " + "the task image architecture — see docs/terminal-bench.md" + ) + await self.exec_as_root(environment, command="mkdir -p /opt/hi") + await environment.upload_file(binary, "/opt/hi/hi") + await self.exec_as_root( + environment, + command=( + "chmod 755 /opt/hi/hi && ln -sf /opt/hi/hi /usr/local/bin/hi " + "&& hi --version" + ), + ) + + @with_prompt_template + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + if not self.model_name or "/" not in self.model_name: + raise ValueError( + "pass -m /, e.g. pipenetwork/ipop/coder-balanced" + ) + provider, model = self.model_name.split("/", 1) + if provider not in _PROVIDERS: + raise ValueError( + f"unknown provider {provider!r}; known: {sorted(_PROVIDERS)}" + ) + hi_provider, key_sources = _PROVIDERS[provider] + + env: dict[str, str] = {} + for key in key_sources: + value = os.environ.get(key) + if value: + env["HI_API_KEY"] = value + break + # Model routing comes entirely from flags/env — never let a config + # file baked into the task image shadow it. + env["XDG_CONFIG_HOME"] = "/tmp/hi-xdg" + + # `|| true`: Terminal-Bench scoring is outcome-only. An unverified or + # incomplete hi turn exits nonzero, but the task tests — not the exit + # code — decide the trial, so the verifier must always get to run. + command = ( + "{ hi --plain --no-save --allow-unverified " + f"--provider {hi_provider} -m {shlex.quote(model)} " + f"--report {_REPORT} " + f"{shlex.quote(instruction)} || true; }} " + f"2>&1 None: + report_path = self.logs_dir / "hi-report.json" + if not report_path.exists(): + return + try: + report = json.loads(report_path.read_text()) + except (OSError, json.JSONDecodeError): + return + usage = report.get("usage") or {} + context.n_input_tokens = usage.get("input_tokens") or report.get("input_tokens") + context.n_output_tokens = usage.get("output_tokens") + context.metadata = { + "hi_outcome": report.get("outcome"), + "hi_telemetry": report.get("telemetry"), + "hi_verification": report.get("verification"), + } diff --git a/scripts/terminal_bench.sh b/scripts/terminal_bench.sh new file mode 100755 index 00000000..e6a6bd57 --- /dev/null +++ b/scripts/terminal_bench.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Run hi on Terminal-Bench 2.0 via Harbor. +# +# Usage: +# scripts/terminal_bench.sh sample # 10-task health-check sample +# scripts/terminal_bench.sh full # the full dataset (mind cost!) +# scripts/terminal_bench.sh task # tasks matching a name glob +# +# Requires: docker running, `uv tool install harbor`, a Linux build of hi at +# $HI_AGENT_BINARY (see docs/terminal-bench.md), and PIPENETWORK_API_KEY (or +# override HI_TB_MODEL, e.g. anthropic/claude-sonnet-5 + ANTHROPIC_API_KEY). +set -euo pipefail +cd "$(dirname "$0")/.." + +MODE="${1:-sample}" +MODEL="${HI_TB_MODEL:-pipenetwork/ipop/coder-balanced}" +DATASET="${HI_TB_DATASET:-terminal-bench@2.0}" +JOBS_DIR="${HI_TB_JOBS_DIR:-jobs/terminal-bench}" +CONCURRENT="${HI_TB_CONCURRENT:-4}" + +: "${HI_AGENT_BINARY:?set HI_AGENT_BINARY to a Linux build of hi (docs/terminal-bench.md)}" +[ -f "$HI_AGENT_BINARY" ] || { echo "HI_AGENT_BINARY not found: $HI_AGENT_BINARY"; exit 2; } +docker ps >/dev/null || { echo "docker daemon not running"; exit 2; } + +ARGS=( + run + --dataset "$DATASET" + --agent integrations.terminal_bench.hi_agent:HiAgent + --model "$MODEL" + --n-concurrent "$CONCURRENT" + --jobs-dir "$JOBS_DIR" + --agent-include-logs 'hi-report.json' + --agent-include-logs 'hi.txt' +) + +case "$MODE" in + sample) + # A fixed, arbitrary-but-stable spread of tasks for plumbing checks. + for t in "chem*" "git*" "hello*" "csv*" "build*" "fix*" "path*" "log*" "compress*" "server*"; do + ARGS+=(--include-task-name "$t") + done + ;; + full) ;; + task) + ARGS+=(--include-task-name "${2:?usage: terminal_bench.sh task }") + ;; + *) + echo "unknown mode: $MODE (sample|full|task )"; exit 2 ;; +esac + +exec harbor "${ARGS[@]}" From c6fd37dda3b1733ab88355e083424bdd308b7bf4 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 15 Jul 2026 12:42:04 -0700 Subject: [PATCH 2/3] fix(tb): amd64-only task images, PYTHONPATH for the adapter import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Terminal-Bench 2.0 task images are published linux/amd64 only — on Apple Silicon they run under Rosetta and an aarch64 hi binary fails with exit 127 (missing loader). Document the --platform linux/amd64 containerized build. Also export the repo root on PYTHONPATH in scripts/terminal_bench.sh: harbor lives in its own uv tool environment and cannot otherwise import integrations.terminal_bench.hi_agent. Pin the sample mode to ten real task names (the dataset has no hello*). Co-Authored-By: Claude Fable 5 --- docs/terminal-bench.md | 17 +++++++++++------ .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 156 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 171 bytes .../__pycache__/hi_agent.cpython-312.pyc | Bin 0 -> 6098 bytes scripts/terminal_bench.sh | 11 +++++++++-- 5 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 integrations/__pycache__/__init__.cpython-312.pyc create mode 100644 integrations/terminal_bench/__pycache__/__init__.cpython-312.pyc create mode 100644 integrations/terminal_bench/__pycache__/hi_agent.cpython-312.pyc diff --git a/docs/terminal-bench.md b/docs/terminal-bench.md index 7d007791..16dd78c3 100644 --- a/docs/terminal-bench.md +++ b/docs/terminal-bench.md @@ -17,18 +17,23 @@ span 19–45% pass rate. uv tool install harbor ``` -2. **A Linux build of `hi`** matching the task-image architecture (aarch64 on - Apple Silicon Docker, x86_64 on most CI). The workspace's GPU crates don't - build in the container, but the binary crate builds alone; the easiest - reproducible path is a containerized build: +2. **A Linux build of `hi`** matching the task-image architecture. The + published Terminal-Bench 2.0 task images are **linux/amd64 only** — even on + Apple Silicon (Docker runs them under Rosetta), the binary must be x86_64. + The workspace's GPU crates don't build in the container, but the binary + crate builds alone; the reproducible path is a containerized build: ```bash - docker run --rm -v "$PWD":/src:ro -v "$PWD/target-linux":/out \ + docker run --rm --platform linux/amd64 -v "$PWD":/src:ro \ + -v "$PWD/target-linux-amd64":/out \ rust:1-bookworm cargo build --release -p hi \ --manifest-path /src/Cargo.toml --target-dir /out - export HI_AGENT_BINARY="$PWD/target-linux/release/hi" + export HI_AGENT_BINARY="$PWD/target-linux-amd64/release/hi" ``` + (An emulated build on Apple Silicon takes several minutes; it only needs + to happen once per hi revision.) + ## Running ```bash diff --git a/integrations/__pycache__/__init__.cpython-312.pyc b/integrations/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..791403529ef5714513b9a29ac9648c024e53ab42 GIT binary patch literal 156 zcmX@j%ge<81bX+wvq1D?5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!vepkRPAw|dPf0Ax zOwrHC)Xgs_$;_=vEXmBzD+cjPlJqn4N>bB{phEHSnR%Hd@$q^EmA5!-a`RJ4b5iY! WSb=6S0&y{j@sXL4k+Fyw$N~VXMJ6}^ literal 0 HcmV?d00001 diff --git a/integrations/terminal_bench/__pycache__/__init__.cpython-312.pyc b/integrations/terminal_bench/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e418d29b61872240dd3524a3302b7b4990070222 GIT binary patch literal 171 zcmX@j%ge<81bX+wvq1D?5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!^3o42PAw|dPf0Ax zOwrHC)Xgs_$;_=vEXmBzD+cjPlJqn4N>bB{ph6|7MY)-Ii8=8}sd>p6`tk9Zd6^~g l@p=W7w>WHa^HWN5QtgUZfkra|aWRPTk(rT^v4|PS0sxHyEHMB8 literal 0 HcmV?d00001 diff --git a/integrations/terminal_bench/__pycache__/hi_agent.cpython-312.pyc b/integrations/terminal_bench/__pycache__/hi_agent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb9d1c9b06c218beffac1b21a52de2bd1308c45c GIT binary patch literal 6098 zcmai2Z)_V!cAw?)?~)?*&yGb))L61)(`S;69mlccxF^{!X7Pujy!8h6f{_oaPi-IMd@1L;7XOLOM8Hy6x@(jl|% z%eCag=`hs&>4+Few~Cw?6hjNHd6(EChV5EJY;Dxq>{|O>ce>4mE+Vnx4J39-fe%<~ zwdwX1SA3uFzb=Hv1u~}+d~{YKd07#1iBU<(WbujQAH}h(Xu6QgNn%1+kQ5yYqEOH! zg6CntwXA$C$#M2@tpK#RSiq71gLOe$I-=nVYGw(jCFQzIR3#4_j^hmQ)9|%xHtVDD zYuB)@;zCZyNSvOPur(8O%c6v37(t4_nf6ew=8{M;qsyuSjYKUjWPuydP)^4wor1ND zN+iwX24yj)%gO@IsERJg3akhAq)YgQtiwWWwy;?B1~8N~UBgG!qMlLnQbJX7E5~t8 zUb0rUM?wXm0L%(R)zyreizo4HR@NFKDi(68Ai_7kA}hsZJXe%+B39?QYgBO8@KH^Y zu&8FV6bRH3{2Ud0GB29q1-@c2Ll>?pQrwn;tkbWAmQ{6g{lF}dAW}gkI^DdWh*&F< zc@U+BvodT-SC>E@MGb^%_Uc4flrp+XR$#9=DKF_zn-eliKxrQ(oasxHerSl}FeGW2 zvd7~S{K&hr0Bpl4IMn#$4brLuA}@$|&>*yPykkEUiW^CQy}{N=H$ z&nJ^flO${D8=M$Uh_H(0&3_-Xj?QVOgspV4qDu=zpeL>+9eUmpR#xUseHj{;M@$WE z49??%T#yt=zoC+)lw42?sf;R0BrzxC1SKPhIHBNYxZhEGn2a-qdm8owrC+@+5dyjw zcNy)Yf+kHkHegO*bW*+H;dJr^o!6y2D6THSxCo^k+tqYv3YjC$gWgHYdMTR9sS8@l zTrP;qKDOkdrYfcONBffcvTg*M)^Bt?Mxq8s&%~0h5fBt=4Fx%4w2e&8j?YX@Pn>V` zdQ}jGARFyda4s`bFU^k4H2T7{ZKS~|?j`{c2?;@FaC98l7)YIjp&`Bf2Cn02KR%`va2ANssfKy-=jdAI0m z>fxlI=$E`^Z$Ju}6n>FwqG*A!PLEp*HuZ#8-0_g%v$dpjjvk5$1_>L7Kp&`GVv)$# z!D1R#MooM#?j#K_k@O-_9uWuKBt_R9H)QK*g^9Z+4OdovACaeMGyKtamr^fj646qk za9tLu7N`YX&X+9Zrr-Kps#)I~;9%)HP+0n&EUXxAMaWB~kiDlQ{YtHEl z{Ez880i!ezWVg_RNRPP~BiQ5(GT?q1iHhxY`0reRDK8@iWT-bf7n>>V)@ZukOoEcvS z?|&17``4~@_Afjh*h86FjO^5BxPQ`! z?160q6>q>Q_O#-BQPXij1whlPKF9rrOVtdYt#}3tno6h%7(Q9!=jEJaa4!qFqBKT`N{o=SEM<5> z9qWb31hS%0M&NWT3-Q@#Qi?cMVK+PrRN8xx{U`eN7W((L zj`irB=z8o+y3#2f4mm%+71tFg$Fj~H*?kS@V0-r;vcSs z+b#N9=&8D!Y3r-|Pz3KFZ-o0F`G_jr5FS=Hp&w zH0u5schUM8X0+S=@lX`%pL8;#G506?ytLlU&^$(SBS`mbhiLS4P4}0f=ItaJOur8Q zofw#^W$IvFPH$65!Xh=irWU7qogqP;%HB+1^tNf6hUzpol}`W*)-l6ZVI`~XRZrQi zI{`g{Ibuz7xXbRh!B^=G@YtnvW`7&~!tK;wV|M$?ZVI609Ntx*j8^?+-v#ui$6(HY zPNiX{Me1|+q{qTXz?ox}D{~t2vw<>K_KV)LFqT0I<3^$nzWwxD43xcZBazeF9L9@q zf!UkN9)}~)2K7#-AHG6=U?kHtvPIv&JMtIYY7owjZhTKFyUM|`|HH6z8TtRPX@Z%@ z1hL4Z!K6yxF9@236M1|VA}?^A5;>PT3kV_Q&K<{6av}MLqP^cy{V25+l_iA+`VJRd z3JhbjX$TXF`=ww?&ldni63gO(5qagpMgIKMcf*M?F)EkMS0{&5+muYyJ*qy{pmtNwa%?j4B!QfIKE|)PFiZqN@@riSTPvf&G zQM#T|in$zy9=IRGU8D~dWw3x<@fL$Kcg#~yPma)bk5z ztxF>5hkkOHW&;*wHYY6`o*x%gT{3+9%-HnQ%q&zO7@U4-)@X51&x}vwAq(k_)A1DU z2#qWZzq2F5W3I#qLOjxF3oc13yrvcj+&;)jO3y-z;kiz+)!c_h72Ju?X=$M*3wME1 z&_PW27zZ+K5KhcQ%X%0hnV6@QsjX=09((VH)#&M;M%Mhd2Wr8NyF**S0Xn9=e`9>(?B>ef?f;akp1E9k?nLLJzM@*J;by`Ytex>dp5)mPyK56{o#AX>i3?n z96MJ>%pV`Pf1vmF$_8^k()sh7KfU?pa5ZwI5X;!_WEA?W_p>xKFwK^6^} zC~EHf7Non59*?aeQvy?<;nQLRVPTA&8i5kCf&xfy1YwH}+;k`r0uBwd+--F@ILNWe zmf6SQ+%BpwGn$J*fSZ89kELt`YCzcrVb|mNHK@uA?|0Vo1da_H2inFSp1`+(o=@oa zIy-$rzt8Cx{dZZIui+6^P(RxB4|>CgKF(9jyAC75GUzH=K{wDcdlf+l7?>=DAH8cD z9`iN82+2yJ2sc>zK%f~daK~%<^w}>UMxgOZV1z7W%b0hSIBRfV`wki0AK)%*zb6Y@p-*zOME3o%BX<8=u<3r>gjLb>DMqKL99edupA#!@YUB+S>nFYkY&O zd1{9S-|Kp}Yx7+7(9maxUioPL?-&1e@gHK72s`j0w4ZRPd z_?=^K9ox8CjizedeeYa)>(a)}YWK-n@4!2yw@RDIYVVncK6f{_=KB}_fp5PFqP7Ef zJGZ+MTV07tB>6RB0v%t#I$0>c(`fAQ%l`B2=r`?M=acMj5)9<=h~ei=U&Qm^fq3u; z#he875YPX(DCDS@f-fJ>!z&kUrq6D22977IiiE1N!9tu!qEI6}G@~~)@;#bSC?V8B z$$6TkX+|&6<`vRBDAGYcX$C4qse%OYmbB9IVFwNQ_ z{PlE#c)elTAKQ3w^O<+klqP1v_^2M|jax9LX}9&hmNfmimTWw4B6)(Nnngp8!8mAln|@xixFv0quuf)(vKpoty(?Jf@#q;Dej1{IQk*06Ktdxh?av_t ztzwvmt~SQKGl-bbmniTh3VnqRe1#7F2m0Yx=o$Du;0AB`YFzA=53W4>`YMOdd~~+b zKVIpWsI~W0@X33xR(eM(ZRcwp2P?4?_pVfWMk?*2wa8PI=&{W!m98_D@X*76C+xb# Tebf3f!*Jj+{~OVYDct`9iq Date: Wed, 15 Jul 2026 13:27:20 -0700 Subject: [PATCH 3/3] fix(tb): read schema-v2 nested usage in populate_context_post_run Report v2 nests token usage as usage.session/usage.turn; the adapter read flat top-level fields and booked zero tokens. Sample-run numbers: 10 TB-2.0 tasks, 5/10 pass, ~1.6M session input tokens per hard task. Co-Authored-By: Claude Fable 5 --- integrations/terminal_bench/hi_agent.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integrations/terminal_bench/hi_agent.py b/integrations/terminal_bench/hi_agent.py index 9417e192..2b696420 100644 --- a/integrations/terminal_bench/hi_agent.py +++ b/integrations/terminal_bench/hi_agent.py @@ -118,9 +118,13 @@ def populate_context_post_run(self, context: AgentContext) -> None: report = json.loads(report_path.read_text()) except (OSError, json.JSONDecodeError): return + # Schema v2 nests usage as {"session": {...}, "turn": {...}}; older + # reports carried flat top-level token fields. usage = report.get("usage") or {} - context.n_input_tokens = usage.get("input_tokens") or report.get("input_tokens") - context.n_output_tokens = usage.get("output_tokens") + session = usage.get("session") or usage + context.n_input_tokens = session.get("input_tokens") or report.get("input_tokens") + context.n_cache_tokens = session.get("cache_read_tokens") + context.n_output_tokens = session.get("output_tokens") context.metadata = { "hi_outcome": report.get("outcome"), "hi_telemetry": report.get("telemetry"),