(MOT-4327) feat(harness): live context accounting with a console chip

crates/console-ui/Cargo.toml

@@ -13,9 +13,11 @@ repository = "https://github.com/iii-hq/workers"
 publish = false
 
 [dependencies]
-# Same exact pin the workers carry — path-linking this crate must never be
-# what drags a worker onto a different SDK build.
-iii-sdk = "=0.21.6"
+# Range, not an exact pin: workers on this repo's SDK line carry their own
+# exact pins (0.21.6 for state, 0.21.8 for the harness) and cargo must be able
+# to unify them with this crate. Path-linking this crate must never be what
+# drags a worker onto a different SDK build, nor what blocks one.
+iii-sdk = ">=0.21.6, <0.22"
 tokio = { version = "1", features = ["rt", "sync", "time", "fs"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

eval/Cargo.toml

@@ -16,8 +16,8 @@ path = "src/lib.rs"
 
 [dependencies]
 harness = { path = "../harness" }
-iii-sdk = "=0.21.6"
-iii-helpers = "=0.21.6"
+iii-sdk = "=0.21.8"
+iii-helpers = "=0.21.8"
 iii-console-ui = { path = "../crates/console-ui" }
 tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] }
 serde = { version = "1", features = ["derive"] }

eval/src/runtime.rs

@@ -693,6 +693,7 @@ fn send_options(job: &EvalJobRecordV1, variant: &crate::contract::EvalVariantV1)
         output: Some(job.request.output.clone()),
         functions: Some(job.request.functions.clone()),
         metadata: job.request.metadata.clone(),
+        max_validation_retries: None,
     }
 }
 

eval/tests/golden/schemas/eval.assert.exact.json

@@ -3,6 +3,104 @@
   "request_schema": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "definitions": {
+      "ContextSnapshotV1": {
+        "description": "One generation's context accounting. `free = usable - total`, floored at zero: once provider usage lands, `total` is what was billed, which can exceed the `usable` budget the window was fit into — that budget was derived before the generation from an estimate.",
+        "properties": {
+          "categories": {
+            "$ref": "#/definitions/SnapshotCategoriesV1"
+          },
+          "compacted": {
+            "type": "boolean"
+          },
+          "effective_max_output_tokens": {
+            "description": "Output allocation `usable` was derived against.",
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "estimator": {
+            "description": "Which estimator produced the numbers (`heuristic` until the context-manager resolves a real tokenizer). Absent when the context-manager predates the breakdown response.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "free": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "model": {
+            "type": "string"
+          },
+          "provider": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "session_id": {
+            "type": "string"
+          },
+          "step": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "summarized_head_tokens": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "timestamp": {
+            "format": "int64",
+            "type": "integer"
+          },
+          "total": {
+            "description": "Final request estimate: categories plus post-assembly growth.",
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "turn_id": {
+            "type": "string"
+          },
+          "usable": {
+            "description": "The input budget the window was fit into.",
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "usage": {
+            "anyOf": [
+              {
+                "$ref": "#/definitions/Usage"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "Actual provider usage for this generation, stamped after the terminal frame; absent when the provider returned none (or the generation never completed)."
+          }
+        },
+        "required": [
+          "categories",
+          "compacted",
+          "effective_max_output_tokens",
+          "free",
+          "model",
+          "session_id",
+          "step",
+          "timestamp",
+          "total",
+          "turn_id",
+          "usable"
+        ],
+        "type": "object"
+      },
       "SessionMetricsResponseV1": {
         "additionalProperties": false,
         "properties": {
@@ -229,6 +327,17 @@
               "null"
             ]
           },
+          "context": {
+            "anyOf": [
+              {
+                "$ref": "#/definitions/ContextSnapshotV1"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "The session's latest per-generation context snapshot (categories, budget, usage) — absent for sessions that have not generated since snapshots landed."
+          },
           "cost_usd": {
             "format": "double",
             "type": [
@@ -299,6 +408,129 @@
         ],
         "type": "object"
       },
+      "SnapshotCategoriesV1": {
+        "description": "Where the request's tokens sit. Categories are assembly-time estimates; `hook_guidance` is the measured growth after assembly (pre-generate hook appends and orphan-repair patches), 0 when the request left assembly unchanged.",
+        "properties": {
+          "hook_guidance": {
+            "default": 0,
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "messages": {
+            "$ref": "#/definitions/SnapshotMessagesV1"
+          },
+          "overhead": {
+            "description": "Provider framing plus response_format / provider_options fields.",
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "system_prompt": {
+            "description": "Final assembled system prompt: mode paragraph, identity, per-step aids, and any compaction summary section.",
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "tools": {
+            "description": "Function schemas exposed to the model.",
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          }
+        },
+        "required": [
+          "messages",
+          "overhead",
+          "system_prompt",
+          "tools"
+        ],
+        "type": "object"
+      },
+      "SnapshotMessagesV1": {
+        "description": "Estimated tokens of the assembled window's messages, by role.",
+        "properties": {
+          "assistant": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "custom": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "function_result": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          },
+          "user": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": "integer"
+          }
+        },
+        "required": [
+          "assistant",
+          "custom",
+          "function_result",
+          "user"
+        ],
+        "type": "object"
+      },
+      "Usage": {
+        "properties": {
+          "cache_read": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "cache_write": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "cost_usd": {
+            "format": "double",
+            "type": [
+              "number",
+              "null"
+            ]
+          },
+          "input": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "output": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "reasoning": {
+            "format": "uint64",
+            "minimum": 0.0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          }
+        },
+        "type": "object"
+      },
       "VariantRoleV1": {
         "enum": [
           "control",