From a5c098faa6670290576490e6532d7e6bf9d97d77 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 3 Aug 2026 16:42:08 +0100 Subject: [PATCH 01/15] (MOT-4327) feat(harness): persist a per-generation context snapshot and serve it through metrics Each generate step folds the assembly it already performed into a ContextSnapshotV1: category token estimates (system prompt, function schemas, messages by role, overhead, post-assembly hook growth), the usable budget, the final request total, compaction state, and the estimator that produced the numbers. No extra counting round trips. After the terminal frame the snapshot is stamped with the generation's actual provider usage and stored under harness_context/. harness::turn-completed now carries the snapshot as context, and harness::metrics by_session entries return each session's latest snapshot beside the existing usage sums. context::assemble's new breakdown response feeds the categories; an older context-manager degrades to totals-only snapshots. --- harness/src/clients/context.rs | 28 +++ harness/src/context_snapshot.rs | 158 ++++++++++++ harness/src/events.rs | 7 + harness/src/functions/metrics.rs | 19 +- harness/src/functions/send.rs | 1 + harness/src/lib.rs | 1 + harness/src/subagent.rs | 2 + harness/src/turn_loop.rs | 80 ++++++ harness/src/types/turn.rs | 5 + .../tests/golden/schemas/harness.metrics.json | 232 ++++++++++++++++++ 10 files changed, 531 insertions(+), 2 deletions(-) create mode 100644 harness/src/context_snapshot.rs diff --git a/harness/src/clients/context.rs b/harness/src/clients/context.rs index 78863ac8f..f4a8e00cd 100644 --- a/harness/src/clients/context.rs +++ b/harness/src/clients/context.rs @@ -23,6 +23,34 @@ pub struct AssembleOutput { pub effective_max_output_tokens: u64, #[serde(default)] pub applied: Applied, + /// Per-category estimates of `token_count`; `None` when the installed + /// context-manager predates the breakdown response. + #[serde(default)] + pub breakdown: Option, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct AssembleBreakdown { + #[serde(default)] + pub system_prompt_tokens: u64, + #[serde(default)] + pub tools_tokens: u64, + #[serde(default)] + pub by_role: ByRoleTokens, + #[serde(default)] + pub estimator: Option, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ByRoleTokens { + #[serde(default)] + pub user: u64, + #[serde(default)] + pub assistant: u64, + #[serde(default)] + pub function_result: u64, + #[serde(default)] + pub custom: u64, } #[derive(Debug, Clone, Default, Deserialize)] diff --git a/harness/src/context_snapshot.rs b/harness/src/context_snapshot.rs new file mode 100644 index 000000000..584c40005 --- /dev/null +++ b/harness/src/context_snapshot.rs @@ -0,0 +1,158 @@ +//! Per-generation context snapshot (`harness_context/`): what +//! the last generation's model window held, by category, plus the provider +//! usage that came back for it. Built from the assembly the loop already +//! performs (no extra counting round trips), stamped with usage after the +//! terminal frame, stored once per generate step. Read back by +//! `harness::metrics` and pushed on `harness::turn-completed`. + +use iii_sdk::IIIClient; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::error::HarnessError; +use crate::types::event::Usage; + +pub const CONTEXT_SCOPE: &str = "harness_context"; + +/// Estimated tokens of the assembled window's messages, by role. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SnapshotMessagesV1 { + pub user: u64, + pub assistant: u64, + pub function_result: u64, + pub custom: u64, +} + +/// Where the request's tokens sit. Categories are assembly-time estimates; +/// `hook_guidance` is the measured growth after assembly (pre-generate hook +/// appends and orphan-repair patches), 0 when the request left assembly +/// unchanged. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SnapshotCategoriesV1 { + /// Final assembled system prompt: mode paragraph, identity, per-step + /// aids, and any compaction summary section. + pub system_prompt: u64, + /// Function schemas exposed to the model. + pub tools: u64, + pub messages: SnapshotMessagesV1, + /// Provider framing plus response_format / provider_options fields. + pub overhead: u64, + #[serde(default)] + pub hook_guidance: u64, +} + +/// One generation's context accounting. `total <= usable` always holds for +/// a generation that ran; `free = usable - total`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ContextSnapshotV1 { + pub session_id: String, + pub turn_id: String, + pub step: u64, + pub model: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Which estimator produced the numbers (`heuristic` until the + /// context-manager resolves a real tokenizer). Absent when the + /// context-manager predates the breakdown response. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimator: Option, + /// The input budget the window was fit into. + pub usable: u64, + /// Output allocation `usable` was derived against. + pub effective_max_output_tokens: u64, + /// Final request estimate: categories plus post-assembly growth. + pub total: u64, + pub free: u64, + pub categories: SnapshotCategoriesV1, + pub compacted: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summarized_head_tokens: Option, + /// Actual provider usage for this generation, stamped after the + /// terminal frame; absent when the provider returned none (or the + /// generation never completed). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + pub timestamp: i64, +} + +/// Store the session's latest snapshot (whole-value write; the loop holds +/// the only writer per session). +pub async fn put( + iii: &IIIClient, + snapshot: &ContextSnapshotV1, + timeout_ms: u64, +) -> Result<(), HarnessError> { + let value = serde_json::to_value(snapshot) + .map_err(|e| HarnessError::State(format!("context snapshot serialize: {e}")))?; + crate::state::state_set(iii, CONTEXT_SCOPE, &snapshot.session_id, value, timeout_ms).await +} + +/// The session's latest snapshot (`None` when absent or unparseable — a +/// snapshot from a newer harness must degrade to "no data", never an error). +pub async fn get( + iii: &IIIClient, + session_id: &str, + timeout_ms: u64, +) -> Result, HarnessError> { + let v = crate::state::state_get(iii, CONTEXT_SCOPE, session_id, timeout_ms).await?; + if v.is_null() { + return Ok(None); + } + Ok(serde_json::from_value(v).ok()) +} + +pub async fn delete( + iii: &IIIClient, + session_id: &str, + timeout_ms: u64, +) -> Result<(), HarnessError> { + crate::state::state_delete(iii, CONTEXT_SCOPE, session_id, timeout_ms).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn snapshot_round_trips_and_tolerates_unknown_fields() { + let snapshot = ContextSnapshotV1 { + session_id: "s_1".into(), + turn_id: "t_1".into(), + step: 3, + model: "m".into(), + provider: Some("p".into()), + estimator: Some("heuristic".into()), + usable: 100_000, + effective_max_output_tokens: 8_192, + total: 62_000, + free: 38_000, + categories: SnapshotCategoriesV1 { + system_prompt: 2_400, + tools: 9_800, + messages: SnapshotMessagesV1 { + user: 10_000, + assistant: 20_000, + function_result: 19_000, + custom: 0, + }, + overhead: 300, + hook_guidance: 500, + }, + compacted: true, + summarized_head_tokens: Some(3_100), + usage: Some(Usage { + input: Some(61_400), + output: Some(900), + cache_read: Some(55_000), + cache_write: None, + reasoning: None, + cost_usd: Some(0.42), + }), + timestamp: 1_722_700_000_000, + }; + let mut value = serde_json::to_value(&snapshot).unwrap(); + value["from_the_future"] = serde_json::json!(true); + let parsed: ContextSnapshotV1 = serde_json::from_value(value).unwrap(); + assert_eq!(parsed, snapshot); + } +} diff --git a/harness/src/events.rs b/harness/src/events.rs index 84e7cde69..fa29d6f2a 100644 --- a/harness/src/events.rs +++ b/harness/src/events.rs @@ -420,6 +420,7 @@ impl TurnEvents { .await; } + #[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)] pub async fn emit_completed( &self, @@ -433,6 +434,7 @@ impl TurnEvents { display_parent: Option<&str>, reactive: ReactiveMeta<'_>, terminal: bool, + context: Option<&crate::context_snapshot::ContextSnapshotV1>, ) { tracing::info!( session_id, @@ -471,6 +473,11 @@ impl TurnEvents { if let Some(dp) = display_parent { payload["parent_session_id"] = Value::String(dp.to_string()); } + // The latest generation's context accounting (categories, budget, + // usage) so live consumers never re-walk the transcript for it. + if let Some(snapshot) = context { + payload["context"] = serde_json::to_value(snapshot).unwrap_or(Value::Null); + } reactive.stamp(&mut payload); self.fan_out( &self.completed, diff --git a/harness/src/functions/metrics.rs b/harness/src/functions/metrics.rs index fc4b63f82..de9c4542e 100644 --- a/harness/src/functions/metrics.rs +++ b/harness/src/functions/metrics.rs @@ -9,6 +9,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::json; +use crate::context_snapshot::ContextSnapshotV1; use crate::deps::Deps; use crate::error::HarnessError; use crate::types::content::ContentBlock; @@ -77,6 +78,11 @@ pub struct SessionUsageV1 { pub reasoning_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cost_usd: Option, + /// The session's latest per-generation context snapshot (categories, + /// budget, usage) — absent for sessions that have not generated since + /// snapshots landed. + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -153,7 +159,11 @@ pub async fn handle( total.observe(message); } } - by_session.push(current.finish_session(node)); + let context = + crate::context_snapshot::get(&deps.iii, &node.session_id, cfg.session_timeout_ms) + .await + .unwrap_or(None); + by_session.push(current.finish_session(node, context)); } // Partial snapshots are polled as progress signals. Trace aggregation is // comparatively expensive and does not help the watchdog decide whether @@ -323,7 +333,11 @@ impl UsageAccumulator { } } - fn finish_session(self, node: &SessionTreeNodeV1) -> SessionUsageV1 { + fn finish_session( + self, + node: &SessionTreeNodeV1, + context: Option, + ) -> SessionUsageV1 { SessionUsageV1 { session_id: node.session_id.clone(), parent_session_id: node.parent_session_id.clone(), @@ -337,6 +351,7 @@ impl UsageAccumulator { cache_write_tokens: self.cache_write.finish(), reasoning_tokens: self.reasoning.finish(), cost_usd: self.cost_usd.finish(), + context, } } diff --git a/harness/src/functions/send.rs b/harness/src/functions/send.rs index ddfc3ae24..ea2a68a9f 100644 --- a/harness/src/functions/send.rs +++ b/harness/src/functions/send.rs @@ -687,6 +687,7 @@ pub(crate) async fn seed_new( reactive_depth: None, reactive_owner_session_id: None, functions_generation, + context_snapshot: None, result: None, result_error: None, validation_retries: 0, diff --git a/harness/src/lib.rs b/harness/src/lib.rs index 6342c0f57..e44713346 100644 --- a/harness/src/lib.rs +++ b/harness/src/lib.rs @@ -11,6 +11,7 @@ pub mod budget; pub mod clients; pub mod config; pub mod configuration; +pub mod context_snapshot; pub mod contract; pub mod deferred; pub mod deps; diff --git a/harness/src/subagent.rs b/harness/src/subagent.rs index eedebfb84..2f18e063a 100644 --- a/harness/src/subagent.rs +++ b/harness/src/subagent.rs @@ -371,6 +371,7 @@ async fn seed_child( reactive_depth: req.reactive_depth, reactive_owner_session_id: reactive_owner_session_id.map(str::to_string), functions_generation: None, + context_snapshot: None, result: None, result_error: None, validation_retries: 0, @@ -490,6 +491,7 @@ mod tests { reactive_depth: None, reactive_owner_session_id: None, functions_generation: None, + context_snapshot: None, result: None, result_error: None, validation_retries: 0, diff --git a/harness/src/turn_loop.rs b/harness/src/turn_loop.rs index 22523279f..96e38c243 100644 --- a/harness/src/turn_loop.rs +++ b/harness/src/turn_loop.rs @@ -507,6 +507,14 @@ pub async fn run_step( .max_output_tokens .unwrap_or(assembled.effective_max_output_tokens) .min(assembled.effective_max_output_tokens); + let snapshot = build_context_snapshot( + &record, + payload.step, + &assembled, + final_request_tokens, + request_overhead_tokens, + ); + record.context_snapshot = Some(snapshot); break ( gen_system_prompt, gen_annotations, @@ -698,6 +706,23 @@ pub async fn run_step( // advance()/finalize call that ends this step). record.watermark_entry_id = watermark; + // Stamp the generation's actual usage into the snapshot and store the + // session's latest copy. Best-effort: accounting must never fail a turn + // that generated successfully. + if let Some(snapshot) = record.context_snapshot.as_mut() { + snapshot.usage = outcome.message.usage.clone(); + if let Err(error) = + crate::context_snapshot::put(&deps.iii, snapshot, cfg.session_timeout_ms).await + { + tracing::warn!( + session_id = %record.session_id, + turn_id = %record.turn_id, + %error, + "context snapshot store failed" + ); + } + } + // Persist the final assistant message into the streamed entry. let _ = session .update_message( @@ -1433,6 +1458,7 @@ async fn finalize_completed( depth: record.reactive_depth, }, turn_is_terminal(deps, &record.session_id), + record.context_snapshot.as_ref(), ) .await; // Sub-agent turns resolve the parent's pending call with their result. @@ -1525,6 +1551,7 @@ async fn finalize_failed( depth: record.reactive_depth, }, turn_is_terminal(deps, &record.session_id), + record.context_snapshot.as_ref(), ) .await; if let Some(parent) = record.parent.clone() { @@ -1792,6 +1819,7 @@ async fn finalize_cancelled( depth: record.reactive_depth, }, turn_is_terminal(deps, &record.session_id), + record.context_snapshot.as_ref(), ) .await; if let Some(parent) = record.parent.clone() { @@ -2136,6 +2164,9 @@ async fn assemble_context( usable: out.usable, token_count: out.token_count, effective_max_output_tokens: out.effective_max_output_tokens, + compacted: out.applied.compacted, + summarized_head_tokens: out.applied.summarized_head_tokens, + breakdown: out.breakdown, }) } @@ -2143,6 +2174,52 @@ fn is_context_overflow_error(error: &str) -> bool { error.contains("context/overflow:") } +/// Fold the assembly the loop already performed into the session's context +/// snapshot. `final_request_tokens >= assembled.token_count` when hooks or +/// orphan repair grew the request; the difference is the hook_guidance +/// category. No counting round trips happen here. +fn build_context_snapshot( + record: &TurnRecord, + step: u64, + assembled: &Assembled, + final_request_tokens: u64, + request_overhead_tokens: u64, +) -> crate::context_snapshot::ContextSnapshotV1 { + use crate::context_snapshot::{ContextSnapshotV1, SnapshotCategoriesV1, SnapshotMessagesV1}; + let breakdown = assembled.breakdown.as_ref(); + let messages = breakdown + .map(|b| SnapshotMessagesV1 { + user: b.by_role.user, + assistant: b.by_role.assistant, + function_result: b.by_role.function_result, + custom: b.by_role.custom, + }) + .unwrap_or_default(); + ContextSnapshotV1 { + session_id: record.session_id.clone(), + turn_id: record.turn_id.clone(), + step, + model: record.options.model.clone(), + provider: record.options.provider.clone(), + estimator: breakdown.and_then(|b| b.estimator.clone()), + usable: assembled.usable, + effective_max_output_tokens: assembled.effective_max_output_tokens, + total: final_request_tokens, + free: assembled.usable.saturating_sub(final_request_tokens), + categories: SnapshotCategoriesV1 { + system_prompt: breakdown.map(|b| b.system_prompt_tokens).unwrap_or(0), + tools: breakdown.map(|b| b.tools_tokens).unwrap_or(0), + messages, + overhead: request_overhead_tokens, + hook_guidance: final_request_tokens.saturating_sub(assembled.token_count), + }, + compacted: assembled.compacted, + summarized_head_tokens: assembled.summarized_head_tokens, + usage: None, + timestamp: AgentMessage::now_ms(), + } +} + /// Append model-facing context aid lines to the system prompt: the session id /// (always — it makes the prompt's "" recipes actionable, e.g. /// `turn-completed` filters and reactive spawns that deliver into this chat), @@ -2214,6 +2291,9 @@ struct Assembled { token_count: u64, /// Model/output ceiling resolved by context-manager for this request. effective_max_output_tokens: u64, + compacted: bool, + summarized_head_tokens: Option, + breakdown: Option, } struct ContextAssemblyInputs<'a> { diff --git a/harness/src/types/turn.rs b/harness/src/types/turn.rs index 4a36afc21..317b68966 100644 --- a/harness/src/types/turn.rs +++ b/harness/src/types/turn.rs @@ -262,6 +262,10 @@ pub struct TurnRecord { /// contracts get re-fetched. #[serde(default, skip_serializing_if = "Option::is_none")] pub functions_generation: Option, + /// Latest generation's context accounting (also stored under + /// `harness_context/` once the generation completes). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context_snapshot: Option, #[serde(skip_serializing_if = "Option::is_none")] pub result: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -352,6 +356,7 @@ mod tests { reactive_depth: None, reactive_owner_session_id: None, functions_generation: None, + context_snapshot: None, result: None, result_error: None, validation_retries: 0, diff --git a/harness/tests/golden/schemas/harness.metrics.json b/harness/tests/golden/schemas/harness.metrics.json index b01c6cd8c..397119829 100644 --- a/harness/tests/golden/schemas/harness.metrics.json +++ b/harness/tests/golden/schemas/harness.metrics.json @@ -17,6 +17,104 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "definitions": { + "ContextSnapshotV1": { + "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "properties": { + "categories": { + "$ref": "#/definitions/SnapshotCategoriesV1" + }, + "compacted": { + "type": "boolean" + }, + "effective_max_output_tokens": { + "description": "Output allocation `usable` was derived against.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "estimator": { + "description": "Which estimator produced the numbers (`heuristic` until the context-manager resolves a real tokenizer). Absent when the context-manager predates the breakdown response.", + "type": [ + "string", + "null" + ] + }, + "free": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "model": { + "type": "string" + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "step": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "summarized_head_tokens": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "timestamp": { + "format": "int64", + "type": "integer" + }, + "total": { + "description": "Final request estimate: categories plus post-assembly growth.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "turn_id": { + "type": "string" + }, + "usable": { + "description": "The input budget the window was fit into.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/definitions/Usage" + }, + { + "type": "null" + } + ], + "description": "Actual provider usage for this generation, stamped after the terminal frame; absent when the provider returned none (or the generation never completed)." + } + }, + "required": [ + "categories", + "compacted", + "effective_max_output_tokens", + "free", + "model", + "session_id", + "step", + "timestamp", + "total", + "turn_id", + "usable" + ], + "type": "object" + }, "SessionTraceMetricsV1": { "additionalProperties": false, "properties": { @@ -205,6 +303,17 @@ "null" ] }, + "context": { + "anyOf": [ + { + "$ref": "#/definitions/ContextSnapshotV1" + }, + { + "type": "null" + } + ], + "description": "The session's latest per-generation context snapshot (categories, budget, usage) — absent for sessions that have not generated since snapshots landed." + }, "cost_usd": { "format": "double", "type": [ @@ -274,6 +383,129 @@ "turns" ], "type": "object" + }, + "SnapshotCategoriesV1": { + "description": "Where the request's tokens sit. Categories are assembly-time estimates; `hook_guidance` is the measured growth after assembly (pre-generate hook appends and orphan-repair patches), 0 when the request left assembly unchanged.", + "properties": { + "hook_guidance": { + "default": 0, + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "messages": { + "$ref": "#/definitions/SnapshotMessagesV1" + }, + "overhead": { + "description": "Provider framing plus response_format / provider_options fields.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "system_prompt": { + "description": "Final assembled system prompt: mode paragraph, identity, per-step aids, and any compaction summary section.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "tools": { + "description": "Function schemas exposed to the model.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "messages", + "overhead", + "system_prompt", + "tools" + ], + "type": "object" + }, + "SnapshotMessagesV1": { + "description": "Estimated tokens of the assembled window's messages, by role.", + "properties": { + "assistant": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "custom": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "function_result": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "user": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "assistant", + "custom", + "function_result", + "user" + ], + "type": "object" + }, + "Usage": { + "properties": { + "cache_read": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "output": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "reasoning": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" } }, "properties": { From fc81335fb9911c9d387c0f6648bf604d005383ec Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 3 Aug 2026 16:57:50 +0100 Subject: [PATCH 02/15] (MOT-4327) feat(harness): injectable console UI with a live context chip and metrics renderer harness ships console assets: a session chip (id context) rendering the real per-turn context meter with a breakdown popover (stacked category bar, legend, estimator provenance, actual usage and cost), fed by harness::metrics on mount and harness::turn-completed live; and a function-trigger renderer for harness::metrics results in chat. The chip registers through the console's session chip slot and skips silently on consoles that predate it. Registration implements the console asset wire contract directly because the shared console-ui crate pins iii-sdk =0.21.6 while harness pins =0.21.8, which cargo cannot co-resolve; relaxing the crate pin is a follow-up outside this worker. --- harness/Cargo.toml | 2 +- harness/build.rs | 173 +++++++++ harness/src/lib.rs | 1 + harness/src/main.rs | 8 +- harness/src/ui.rs | 353 ++++++++++++++++++ harness/ui/build.mjs | 121 ++++++ harness/ui/package.json | 18 + harness/ui/page.tsx | 38 ++ harness/ui/src/context-chip/index.tsx | 338 +++++++++++++++++ .../ui/src/function-trigger-message/index.tsx | 151 ++++++++ harness/ui/src/lib/format.ts | 17 + harness/ui/src/lib/metrics.ts | 115 ++++++ harness/ui/styles.css | 275 ++++++++++++++ harness/ui/tsconfig.json | 14 + pnpm-lock.yaml | 16 + pnpm-workspace.yaml | 1 + 16 files changed, 1639 insertions(+), 2 deletions(-) create mode 100644 harness/src/ui.rs create mode 100644 harness/ui/build.mjs create mode 100644 harness/ui/package.json create mode 100644 harness/ui/page.tsx create mode 100644 harness/ui/src/context-chip/index.tsx create mode 100644 harness/ui/src/function-trigger-message/index.tsx create mode 100644 harness/ui/src/lib/format.ts create mode 100644 harness/ui/src/lib/metrics.ts create mode 100644 harness/ui/styles.css create mode 100644 harness/ui/tsconfig.json diff --git a/harness/Cargo.toml b/harness/Cargo.toml index 8e08f82d6..34840b432 100644 --- a/harness/Cargo.toml +++ b/harness/Cargo.toml @@ -25,7 +25,7 @@ iii-sdk = "=0.21.8" # 0.21.5 adds the live span-start push (`LiveSpanStartProcessor`), so the # console renders `harness::turn step` while it runs instead of on close. iii-helpers = "=0.21.8" -tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time", "fs"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" diff --git a/harness/build.rs b/harness/build.rs index 81caa36d6..029f6d310 100644 --- a/harness/build.rs +++ b/harness/build.rs @@ -1,6 +1,179 @@ +//! Build script for the `harness` worker. +//! +//! 1. Forwards the build-time target triple to the binary as `env!("TARGET")` +//! (used by `manifest.rs` for the registry `supported_targets` field). +//! 2. Ensures the injected console UI assets exist: `src/ui.rs` embeds +//! `ui/dist/page.js` and `ui/dist/styles.css` via `include_str!`, so if +//! either is missing or stale we run `pnpm install && pnpm build` inside +//! `ui/` first (the console worker's `web/` precedent). Set +//! `SKIP_UI_BUILD=1` to use the existing `ui/dist/` outputs as-is. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::SystemTime; + fn main() { println!( "cargo:rustc-env=TARGET={}", std::env::var("TARGET").unwrap() ); + + // `dist/` itself is not listed: include_str! reads it directly, and + // listing it would rebuild-loop on our own output. + println!("cargo:rerun-if-changed=ui/page.tsx"); + println!("cargo:rerun-if-changed=ui/styles.css"); + println!("cargo:rerun-if-changed=ui/src"); + println!("cargo:rerun-if-changed=ui/build.mjs"); + println!("cargo:rerun-if-changed=ui/package.json"); + // The lockfile lives at the workers-repo root (pnpm workspace: the ui + // project links @iii-dev/console-ui from packages/console-ui). + println!("cargo:rerun-if-changed=../pnpm-lock.yaml"); + println!("cargo:rerun-if-changed=ui/tsconfig.json"); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let ui_dir = manifest_dir.join("ui"); + let dist_assets = [ + ui_dir.join("dist").join("page.js"), + ui_dir.join("dist").join("styles.css"), + ]; + + if dist_assets + .iter() + .all(|a| a.exists() && dist_is_fresh(a, &ui_dir)) + { + return; + } + + if std::env::var_os("SKIP_UI_BUILD").is_some() { + for asset in &dist_assets { + if !asset.exists() { + panic!( + "SKIP_UI_BUILD set but {} is missing — build the UI manually \ + (cd ui && pnpm install && pnpm build) or unset the env var", + asset.display() + ); + } + } + return; + } + + let pnpm = locate_pnpm(); + + let status = Command::new(&pnpm) + .args(["install"]) + .current_dir(&ui_dir) + .status() + .unwrap_or_else(|e| { + panic!( + "failed to spawn `pnpm install` in {}: {e}", + ui_dir.display() + ) + }); + if !status.success() { + panic!("`pnpm install` exited with {status} — see logs above"); + } + + let status = Command::new(&pnpm) + .args(["build"]) + .current_dir(&ui_dir) + .status() + .unwrap_or_else(|e| panic!("failed to spawn `pnpm build` in {}: {e}", ui_dir.display())); + if !status.success() { + panic!("`pnpm build` exited with {status} — see logs above"); + } + + for asset in &dist_assets { + if !asset.exists() { + panic!( + "`pnpm build` finished but {} is still missing — check the esbuild \ + output above", + asset.display() + ); + } + } +} + +/// `true` when the built asset is at least as new as every source that +/// contributes to it. Conservative: any I/O failure forces a rebuild. +fn dist_is_fresh(dist_asset: &Path, ui_dir: &Path) -> bool { + let Ok(dist_mtime) = dist_asset.metadata().and_then(|m| m.modified()) else { + return false; + }; + + let watched_files = [ + ui_dir.join("page.tsx"), + ui_dir.join("styles.css"), + ui_dir.join("build.mjs"), + ui_dir.join("package.json"), + ui_dir.join("../../pnpm-lock.yaml"), + ui_dir.join("tsconfig.json"), + ]; + for f in watched_files.iter() { + if !f.exists() { + continue; + } + let Ok(m) = f.metadata().and_then(|m| m.modified()) else { + return false; + }; + if m > dist_mtime { + return false; + } + } + + for dir in [ui_dir.join("src")] { + if dir.exists() && !subtree_older_than(&dir, dist_mtime) { + return false; + } + } + + true +} + +fn subtree_older_than(root: &Path, ceiling: SystemTime) -> bool { + let Ok(read) = std::fs::read_dir(root) else { + return false; + }; + for entry in read.flatten() { + let path = entry.path(); + let Ok(meta) = entry.metadata() else { + return false; + }; + if meta.is_dir() { + if !subtree_older_than(&path, ceiling) { + return false; + } + } else { + let Ok(m) = meta.modified() else { + return false; + }; + if m > ceiling { + return false; + } + } + } + true +} + +fn locate_pnpm() -> PathBuf { + if let Ok(explicit) = std::env::var("PNPM") { + return PathBuf::from(explicit); + } + let candidates = if cfg!(windows) { + ["pnpm.cmd", "pnpm.exe", "pnpm"].as_slice() + } else { + ["pnpm"].as_slice() + }; + let path = std::env::var_os("PATH").unwrap_or_default(); + for dir in std::env::split_paths(&path) { + for name in candidates { + let candidate = dir.join(name); + if candidate.is_file() { + return candidate; + } + } + } + panic!( + "pnpm not found on PATH — install Node + pnpm, or set SKIP_UI_BUILD=1 \ + after building the UI manually with `cd ui && pnpm install && pnpm build`" + ); } diff --git a/harness/src/lib.rs b/harness/src/lib.rs index e44713346..2d787ad74 100644 --- a/harness/src/lib.rs +++ b/harness/src/lib.rs @@ -36,3 +36,4 @@ pub(crate) mod trace_tags; pub mod trigger; pub mod turn_loop; pub mod types; +pub mod ui; diff --git a/harness/src/main.rs b/harness/src/main.rs index ff3d9ca7b..015352d6b 100644 --- a/harness/src/main.rs +++ b/harness/src/main.rs @@ -32,7 +32,7 @@ use harness::configuration::{self, ConfigCell, TriggerHandles}; use harness::deps::Deps; use harness::events::TurnEvents; use harness::hooks::HookRegistry; -use harness::{config, discovery, functions, manifest, queue, subscriptions}; +use harness::{config, discovery, functions, manifest, queue, subscriptions, ui}; #[derive(Parser, Debug)] #[command( @@ -134,6 +134,12 @@ async fn main() -> Result<()> { functions::register_all(&iii, &deps); + // Injectable console UI: content function + console:script/style triggers. + // Ordering doesn't matter for the console side (the engine parks the + // registration until a console owns the type), but the content function + // must exist before the trigger names it. + ui::register(&iii); + // The queue consumer may restore durable jobs as soon as this definition // succeeds. Discovery and `harness::turn` are already ready; do not bind // lifecycle triggers or announce ready unless the queue is usable. diff --git a/harness/src/ui.rs b/harness/src/ui.rs new file mode 100644 index 000000000..cc4dc420a --- /dev/null +++ b/harness/src/ui.rs @@ -0,0 +1,353 @@ +//! Injectable console UI for the harness worker +//! (iii/tech-specs/2026-07-17-injectable-ui; authoring SOP: +//! workers/docs/sops/injectable-console-ui.md). +//! +//! Ships two assets into any running console: +//! +//! - `harness/page.js` (`console:script`) — the `context` session chip +//! (live context-window usage with a category-breakdown popover; the +//! `chat.registerSessionChip` slot is feature-detected, older consoles +//! skip it) plus the `harness::metrics` function-trigger renderer its +//! `setup(host)` registers. +//! - `harness/styles.css` (`console:style`) — the stylesheet, every rule +//! scoped under `[data-iii-ui="harness"]`; the console mounts it as a +//! `` and link-swaps it on change, styles-before-scripts on boot. +//! +//! Other workers get the registration machinery from the shared +//! `iii-console-ui` crate (`workers/crates/console-ui`), but that crate +//! pins `iii-sdk = "=0.21.6"` while the harness needs `=0.21.8` (the +//! reconnect reattach handshake) — two semver-compatible exact pins cargo +//! cannot co-resolve. This module therefore implements the same wire +//! contract against the harness's own SDK: the content function +//! `harness::ui-content` (`{path}` in, `{content, content_type}` out, +//! flagged internal), one Message-path trigger per asset (never +//! `engine::register_trigger`), and the `III_HARNESS_UI_WATCH` dev poller +//! (swap the served bytes, register a FRESH trigger for the same path, +//! THEN unregister the previous handle). +//! +//! The assets are compiled from `ui/` by esbuild (react + @iii-dev/console-ui +//! external — they resolve through the console's import map at runtime) and +//! embedded at compile time so the worker stays one self-contained binary. +//! For the dev loop, set `III_HARNESS_UI_WATCH` to the build output +//! directory (or `1` for `ui/dist`): the worker polls both files and +//! re-registers a changed asset's trigger — every open console tab +//! hot-swaps it. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use iii_sdk::errors::Error; +use iii_sdk::protocol::RegisterTriggerInput; +use iii_sdk::{IIIClient, RegisterFunction}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; + +pub const PAGE_PATH: &str = "harness/page.js"; +pub const STYLES_PATH: &str = "harness/styles.css"; + +const CONTENT_FN: &str = "harness::ui-content"; +const WATCH_ENV: &str = "III_HARNESS_UI_WATCH"; +const WATCH_DEFAULT_DIR: &str = "ui/dist"; +const WATCH_POLL: Duration = Duration::from_millis(1000); + +/// Built by `build.rs` (esbuild over `ui/`). +const PAGE_JS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/page.js")); +const STYLES_CSS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/styles.css")); + +/// Input of the content function: the console asks for one asset by path. +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct UiContentInput { + pub path: String, +} + +/// Output of the content function. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct UiContentResult { + pub content: String, + pub content_type: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AssetKind { + Script, + Style, +} + +impl AssetKind { + fn trigger_type(self) -> &'static str { + match self { + AssetKind::Script => "console:script", + AssetKind::Style => "console:style", + } + } + + fn content_type(self) -> &'static str { + match self { + AssetKind::Script => "text/javascript; charset=utf-8", + AssetKind::Style => "text/css; charset=utf-8", + } + } +} + +struct AssetSpec { + path: &'static str, + kind: AssetKind, + /// File inside the watch dir the dev poller reads (last path segment). + file: &'static str, + content: &'static str, +} + +const ASSETS: [AssetSpec; 2] = [ + AssetSpec { + path: PAGE_PATH, + kind: AssetKind::Script, + file: "page.js", + content: PAGE_JS, + }, + AssetSpec { + path: STYLES_PATH, + kind: AssetKind::Style, + file: "styles.css", + content: STYLES_CSS, + }, +]; + +/// Register the harness's console UI: the content function plus one +/// Message-path trigger per asset, and the dev watcher when +/// `III_HARNESS_UI_WATCH` is set. Call after `functions::register_all`. +/// Trigger registration failures are warn-logged, not fatal — injected UI +/// is an accessory, and the SDK replays surviving registrations on +/// reconnect. +pub fn register(iii: &Arc) { + let served = Arc::new(Served::new(&ASSETS)); + + { + let served = served.clone(); + iii.register_function( + CONTENT_FN, + RegisterFunction::new_async(move |input: UiContentInput| { + let served = served.clone(); + async move { served.content_for(&input.path).await } + }) + .description( + "Serve the harness worker's injected console UI assets (content function \ + for its console:script / console:style triggers).", + ) + .metadata(serde_json::json!({ "internal": true })), + ); + } + + let mut watched = Vec::new(); + for spec in &ASSETS { + match register_asset_trigger(iii, spec.kind, spec.path) { + Ok(handle) => { + tracing::info!(path = spec.path, "registered console ui asset"); + watched.push(WatchedAsset { + path: spec.path, + kind: spec.kind, + file: spec.file, + handle, + prev: spec.content.to_string(), + }); + } + Err(e) => tracing::warn!( + error = %e, + path = spec.path, + "failed to register console ui trigger" + ), + } + } + + if let Some(dist) = watch_target() { + if watched.is_empty() { + tracing::warn!("ui watch requested but no ui trigger registered — watcher not started"); + } else { + spawn_watcher(iii.clone(), served, watched, dist); + } + } +} + +/// Serve the assets from in-memory cells so the dev watcher can swap the +/// bytes without re-registering the function. +struct Served { + content: RwLock>, +} + +struct ServedAsset { + content: String, + content_type: &'static str, +} + +impl Served { + fn new(assets: &[AssetSpec]) -> Self { + Self { + content: RwLock::new( + assets + .iter() + .map(|a| { + ( + a.path, + ServedAsset { + content: a.content.to_string(), + content_type: a.kind.content_type(), + }, + ) + }) + .collect(), + ), + } + } + + async fn content_for(&self, path: &str) -> Result { + let guard = self.content.read().await; + let asset = guard.get(path).ok_or_else(|| { + Error::Handler(format!( + "UNKNOWN_UI_ASSET: '{path}' is not one of this worker's ui assets \ + (expected one of: {PAGE_PATH}, {STYLES_PATH})" + )) + })?; + Ok(UiContentResult { + content: asset.content.clone(), + content_type: asset.content_type.to_string(), + }) + } + + async fn swap(&self, path: &str, next: String) { + if let Some(asset) = self.content.write().await.get_mut(path) { + asset.content = next; + } + } +} + +fn register_asset_trigger( + iii: &Arc, + kind: AssetKind, + path: &str, +) -> Result { + iii.register_trigger(RegisterTriggerInput { + trigger_type: kind.trigger_type().to_string(), + function_id: CONTENT_FN.to_string(), + config: serde_json::json!({ "path": path }), + metadata: None, + }) +} + +fn watch_target() -> Option { + parse_watch_target(&std::env::var(WATCH_ENV).ok()?) +} + +fn parse_watch_target(raw: &str) -> Option { + if raw.is_empty() || raw == "0" || raw.eq_ignore_ascii_case("false") { + return None; + } + if raw == "1" || raw.eq_ignore_ascii_case("true") { + return Some(PathBuf::from(WATCH_DEFAULT_DIR)); + } + Some(PathBuf::from(raw)) +} + +struct WatchedAsset { + path: &'static str, + kind: AssetKind, + file: &'static str, + handle: iii_sdk::trigger::Trigger, + prev: String, +} + +/// Dev-loop hot reload: poll the built files; on change, swap the served +/// bytes, register a fresh trigger for the same path (the console supersedes +/// + re-fetches + pushes to every tab), THEN unregister the previous handle. +fn spawn_watcher( + iii: Arc, + served: Arc, + mut watched: Vec, + dist: PathBuf, +) { + tokio::spawn(async move { + tracing::info!(dir = %dist.display(), "ui watch enabled — hot reload on rebuild"); + loop { + tokio::time::sleep(WATCH_POLL).await; + for asset in watched.iter_mut() { + let file = dist.join(asset.file); + let Ok(next) = tokio::fs::read_to_string(&file).await else { + continue; + }; + if next == asset.prev { + continue; + } + served.swap(asset.path, next.clone()).await; + asset.prev = next; + match register_asset_trigger(&iii, asset.kind, asset.path) { + Ok(next_handle) => { + let old = std::mem::replace(&mut asset.handle, next_handle); + old.unregister(); + tracing::info!(path = asset.path, "ui asset re-registered (hot reload)"); + } + Err(e) => tracing::warn!( + error = %e, + path = asset.path, + "ui hot-reload re-register failed" + ), + } + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn embedded_page_is_nonempty_esm() { + assert!(PAGE_JS.contains("export"), "built page.js looks wrong"); + } + + #[test] + fn embedded_styles_are_scoped() { + // esbuild prints the attribute selector unquoted ([data-iii-ui=harness]). + assert!( + STYLES_CSS.contains(r#"[data-iii-ui="harness"]"#) + || STYLES_CSS.contains("[data-iii-ui=harness]"), + "built styles.css must be scoped under the worker's data-iii-ui attribute" + ); + } + + #[tokio::test] + async fn serves_registered_assets_with_content_types() { + let served = Served::new(&ASSETS); + let page = served.content_for(PAGE_PATH).await.unwrap(); + assert!(page.content_type.starts_with("text/javascript")); + let styles = served.content_for(STYLES_PATH).await.unwrap(); + assert!(styles.content_type.starts_with("text/css")); + } + + #[tokio::test] + async fn unknown_path_errors_and_names_the_known_paths() { + let err = Served::new(&ASSETS) + .content_for("harness/nope.js") + .await + .unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("UNKNOWN_UI_ASSET")); + assert!(msg.contains(PAGE_PATH)); + assert!(msg.contains(STYLES_PATH)); + } + + #[test] + fn parse_watch_target_conventions() { + assert_eq!(parse_watch_target(""), None); + assert_eq!(parse_watch_target("0"), None); + assert_eq!(parse_watch_target("false"), None); + assert_eq!(parse_watch_target("FALSE"), None); + assert_eq!(parse_watch_target("1"), Some(PathBuf::from("ui/dist"))); + assert_eq!(parse_watch_target("true"), Some(PathBuf::from("ui/dist"))); + assert_eq!( + parse_watch_target("build/out"), + Some(PathBuf::from("build/out")) + ); + } +} diff --git a/harness/ui/build.mjs b/harness/ui/build.mjs new file mode 100644 index 000000000..16d3131d5 --- /dev/null +++ b/harness/ui/build.mjs @@ -0,0 +1,121 @@ +/** + * Build the worker's two console assets: + * + * page.tsx → dist/page.js (injected over `console:script`) + * styles.css → dist/styles.css (injected over `console:style`) + * + * The five shared specifiers stay EXTERNAL — they resolve at runtime + * through the console's import map (a bundled React copy would surface as + * a cryptic "Invalid hook call"). Everything else the page needs gets + * bundled in. `--watch` pairs with the worker's III_HARNESS_UI_WATCH + * poller for the hot-reload dev loop. + */ + +import { readFileSync } from 'node:fs' +import esbuild from 'esbuild' + +const watch = process.argv.includes('--watch') + +const options = { + entryPoints: ['page.tsx', 'styles.css'], + bundle: true, + format: 'esm', + jsx: 'automatic', + outdir: 'dist', + // The bundle is `include_str!`'d into the worker binary and the injectable-UI + // protocol rejects an asset over 8 MiB outright, so ship it minified. Left + // off under --watch, where readable stack traces matter more than bytes. + minify: !watch, + external: [ + 'react', + 'react-dom', + 'react-dom/client', + 'react/jsx-runtime', + '@iii-dev/console-ui', + ], + logLevel: 'info', +} + +// Minification strips the quotes from an attribute selector, so the scope +// prefix has to be matched in both spellings. +const SCOPE = '[data-iii-ui="harness"]' +const SCOPE_RE = /^\[data-iii-ui=("harness"|'harness'|harness)\]/ +const KEYFRAME_PREFIXES = ['harness-ui-'] + +/** + * Fail the build if any rule could escape this worker's subtree. + * + * The console mounts every worker's UI into one document, so a single + * unscoped selector restyles the whole app. The Rust side already asserts + * that the scope string appears *somewhere* in the sheet, which a file + * containing one stray `body { }` would also pass. This checks every + * selector, and runs in CI for free because CI runs `pnpm build`. + */ +function assertScoped(css) { + // Strip comments, then strings, so a brace or selector inside either can't + // desynchronise the scan. + const src = css + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'/g, '""') + + const problems = [] + let depth = 0 + let buf = '' + // Nested at-rules (@media, @container, @supports) open a block whose + // children are still top-level selectors, so track them rather than + // skipping their contents. + const atStack = [] + + for (let i = 0; i < src.length; i++) { + const ch = src[i] + if (ch === '{') { + const head = buf.trim() + buf = '' + if (head.startsWith('@')) { + const name = head.split(/[\s(]/)[0] + atStack.push(name) + if (name === '@keyframes') { + const kf = head.slice('@keyframes'.length).trim() + if (!KEYFRAME_PREFIXES.some((p) => kf.startsWith(p))) { + problems.push(`@keyframes ${kf} is not prefixed ${KEYFRAME_PREFIXES.join(' or ')}`) + } + } else if (name === '@font-face') { + problems.push('@font-face is not allowed in an injected sheet') + } + depth++ + continue + } + // Inside @keyframes the "selectors" are 0%/from/to, not real ones. + if (!atStack.includes('@keyframes') && head) { + for (const sel of head.split(',')) { + const s = sel.trim() + if (s && !SCOPE_RE.test(s)) { + problems.push(`selector not scoped: ${s}`) + } + } + } + depth++ + } else if (ch === '}') { + depth-- + buf = '' + if (atStack.length && depth < atStack.length) atStack.pop() + } else { + buf += ch + } + } + + if (problems.length) { + console.error(`\n${problems.length} unscoped rule(s) in dist/styles.css:\n`) + for (const p of [...new Set(problems)]) console.error(` ${p}`) + console.error(`\nEvery rule must start with ${SCOPE}.\n`) + process.exit(1) + } +} + +if (watch) { + const ctx = await esbuild.context(options) + await ctx.watch() +} else { + await esbuild.build(options) + assertScoped(readFileSync('dist/styles.css', 'utf8')) +} diff --git a/harness/ui/package.json b/harness/ui/package.json new file mode 100644 index 000000000..44530f14e --- /dev/null +++ b/harness/ui/package.json @@ -0,0 +1,18 @@ +{ + "name": "@iii-workers/harness-ui", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "build": "tsc --noEmit && node build.mjs", + "watch": "node build.mjs --watch" + }, + "dependencies": { + "@iii-dev/console-ui": "workspace:*" + }, + "devDependencies": { + "@types/react": "^19.2.14", + "esbuild": "^0.25.0", + "typescript": "^5.9.2" + } +} diff --git a/harness/ui/page.tsx b/harness/ui/page.tsx new file mode 100644 index 000000000..1d681c728 --- /dev/null +++ b/harness/ui/page.tsx @@ -0,0 +1,38 @@ +/** + * Entry for the harness worker's injected console UI — compiled by esbuild + * (react + @iii-dev/console-ui external) into dist/page.js and served over + * the `console:script` trigger (see src/ui.rs). The stylesheet is its own + * asset: ../styles.css ships over `console:style` as harness/styles.css. + * + * `setup(host)` composes the worker's two console contributions: + * + * - src/context-chip/ — the `context` session chip (live context + * window usage). The `chat.registerSessionChip` slot is newer than the + * published Host type, so it is feature-detected: an older console simply + * gets no chip. + * - src/function-trigger-message/ — how `harness::metrics` calls render. + * + * Registrations go through `host` so the loader disposes them on hot + * reload / worker disconnect. + */ + +import type { ComponentType } from 'react' +import type { Host } from '@iii-dev/console-ui' +import { createContextChip, type SessionChipProps } from './src/context-chip' +import { createMetricsRenderer } from './src/function-trigger-message' + +type SessionChipHost = Host & { + chat?: { + registerSessionChip?: (chip: { + id: string + render: ComponentType + }) => () => void + } +} + +export default function setup(host: Host) { + const chat = (host as SessionChipHost).chat + chat?.registerSessionChip?.({ id: 'context', render: createContextChip(host) }) + + host.functionTriggers.register(createMetricsRenderer()) +} diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx new file mode 100644 index 000000000..be23d263c --- /dev/null +++ b/harness/ui/src/context-chip/index.tsx @@ -0,0 +1,338 @@ +/** + * The `context` session chip — a live view of the session's context window + * matching the console's ContextUsage aesthetic (`ctx` label, bordered bar, + * percent, `12.3k/200k` counts). Hydrates from `harness::metrics` on mount + * and per session change; stays live over the worker's own + * `harness::turn-completed` trigger (Message-path binding, GC'd with the + * tab). Click toggles an anchored popover breaking the window down by + * category with a stacked segment bar, legend, and last-turn actuals. + */ + +import { useEffect, useRef, useState } from 'react' +import type { Host } from '@iii-dev/console-ui' +import { formatCost, formatTokens } from '../lib/format' +import { + type ContextSnapshot, + type SnapshotMessages, + isSnapshot, + parseMetrics, +} from '../lib/metrics' + +/** Per-tab handler id (host.iii.on namespaces it `::`). */ +const EVENTS_FN = 'iii::harness-ui::events' + +const WARN_THRESHOLD = 0.75 +const DANGER_THRESHOLD = 0.9 + +export interface SessionChipProps { + sessionId: string + modelId?: string + contextWindow?: number +} + +interface TurnCompletedEvent { + session_id?: string + turn_id?: string + terminal?: boolean + context?: unknown +} + +type Tone = 'ok' | 'warn' | 'alert' + +const TONE_COLOR: Record = { + ok: 'var(--color-accent)', + warn: 'var(--color-warn)', + alert: 'var(--color-alert)', +} + +function toneFor(ratio: number): Tone { + if (ratio >= DANGER_THRESHOLD) return 'alert' + if (ratio >= WARN_THRESHOLD) return 'warn' + return 'ok' +} + +const ink = (opacity: number) => + `color-mix(in srgb, var(--color-ink) ${opacity}%, transparent)` +const accent = (opacity: number) => + `color-mix(in srgb, var(--color-accent) ${opacity}%, transparent)` + +const COLOR_SYSTEM = ink(80) +const COLOR_TOOLS = ink(55) +const COLOR_USER = accent(95) +const COLOR_ASSISTANT = accent(70) +const COLOR_RESULTS = accent(45) +const COLOR_HOOKS = ink(35) +const COLOR_OVERHEAD = ink(20) +const COLOR_FREE = 'var(--color-rule-2)' + +const EMPTY_MESSAGES: SnapshotMessages = { + user: 0, + assistant: 0, + function_result: 0, + custom: 0, +} + +function segments(snapshot: ContextSnapshot) { + const cats = snapshot.categories + const messages = cats.messages ?? EMPTY_MESSAGES + return [ + { key: 'system', tokens: cats.system_prompt ?? 0, color: COLOR_SYSTEM }, + { key: 'tools', tokens: cats.tools ?? 0, color: COLOR_TOOLS }, + { key: 'user', tokens: messages.user ?? 0, color: COLOR_USER }, + { key: 'assistant', tokens: messages.assistant ?? 0, color: COLOR_ASSISTANT }, + { + key: 'results', + tokens: (messages.function_result ?? 0) + (messages.custom ?? 0), + color: COLOR_RESULTS, + }, + { key: 'hooks', tokens: cats.hook_guidance ?? 0, color: COLOR_HOOKS }, + { key: 'overhead', tokens: cats.overhead ?? 0, color: COLOR_OVERHEAD }, + ] +} + +function LegendRow({ + color, + label, + tokens, + usable, + badge, +}: { + color: string | null + label: string + tokens: number + usable: number + badge?: string +}) { + const pct = usable > 0 ? Math.round((tokens / usable) * 100) : 0 + return ( +
+ + {label} + {badge ? {badge} : null} + {formatTokens(tokens)} + {pct}% +
+ ) +} + +function ContextPopover({ + snapshot, + modelId, +}: { + snapshot: ContextSnapshot + modelId?: string +}) { + const usable = snapshot.usable + const pct = + usable > 0 ? Math.round(Math.min(1, snapshot.total / usable) * 100) : 0 + const messages = snapshot.categories.messages ?? EMPTY_MESSAGES + const conversation = + (messages.user ?? 0) + + (messages.assistant ?? 0) + + (messages.function_result ?? 0) + + (messages.custom ?? 0) + const hookGuidance = snapshot.categories.hook_guidance ?? 0 + const free = snapshot.free ?? Math.max(0, usable - snapshot.total) + const usage = snapshot.usage + const hasActuals = + usage != null && (usage.input != null || usage.cache_read != null) + return ( +
+
+ + {snapshot.model || modelId || 'model'} + + + {pct}% of {formatTokens(usable)} + +
+
+ {segments(snapshot) + .filter((segment) => segment.tokens > 0) + .map((segment) => ( + 0 ? (segment.tokens / usable) * 100 : 0}%`, + background: segment.color, + }} + /> + ))} +
+
+ + + + {hookGuidance > 0 ? ( + + ) : null} + {snapshot.compacted ? ( + + ) : null} + + +
+
+ est. {snapshot.estimator ?? 'unknown'} + {hasActuals ? ( + + last turn actual{' '} + {formatTokens((usage?.input ?? 0) + (usage?.cache_read ?? 0))} · + output {formatTokens(usage?.output ?? 0)} + + ) : null} + {usage?.cost_usd != null ? ( + cost {formatCost(usage.cost_usd)} + ) : null} +
+
+ ) +} + +export function createContextChip(host: Host) { + return function ContextChip({ sessionId, modelId }: SessionChipProps) { + const [snapshot, setSnapshot] = useState(null) + const [open, setOpen] = useState(false) + const rootRef = useRef(null) + + useEffect(() => { + let cancelled = false + setSnapshot(null) + setOpen(false) + host.iii + .trigger('harness::metrics', { root_session_id: sessionId }) + .then((result) => { + if (cancelled) return + const metrics = parseMetrics(result) + const own = metrics?.by_session.find( + (row) => row.session_id === sessionId, + ) + if (own?.context && isSnapshot(own.context)) setSnapshot(own.context) + }) + .catch(() => {}) + return () => { + cancelled = true + } + }, [host, sessionId]) + + useEffect(() => { + const offHandler = host.iii.on(EVENTS_FN, (event) => { + if (!event || event.session_id !== sessionId) return + if (isSnapshot(event.context)) setSnapshot(event.context) + }) + const offTrigger = host.iii.registerTrigger({ + type: 'harness::turn-completed', + function_id: `${EVENTS_FN}::${host.iii.browserId}`, + config: { session_id: sessionId }, + }) + return () => { + offTrigger() + offHandler() + } + }, [host, sessionId]) + + useEffect(() => { + if (!open) return + const onPointerDown = (event: MouseEvent) => { + const root = rootRef.current + if (root && !root.contains(event.target as Node)) setOpen(false) + } + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') setOpen(false) + } + document.addEventListener('mousedown', onPointerDown) + document.addEventListener('keydown', onKeyDown) + return () => { + document.removeEventListener('mousedown', onPointerDown) + document.removeEventListener('keydown', onKeyDown) + } + }, [open]) + + if (!snapshot || snapshot.usable <= 0) { + return ( +
+ ctx + — +
+ ) + } + + const ratio = Math.min(1, snapshot.total / snapshot.usable) + const pct = Math.round(ratio * 100) + const tone = toneFor(ratio) + + return ( +
+ + {open ? : null} +
+ ) + } +} diff --git a/harness/ui/src/function-trigger-message/index.tsx b/harness/ui/src/function-trigger-message/index.tsx new file mode 100644 index 000000000..be475310e --- /dev/null +++ b/harness/ui/src/function-trigger-message/index.tsx @@ -0,0 +1,151 @@ +/** + * Injected function-trigger message renderer for `harness::metrics` — + * registered through `host.functionTriggers`, so it dispatches BEFORE the + * console's built-in families and replaces the raw JSON card in chat and + * traces with a totals row plus a compact per-session table (depth-indented + * ids, turn/token/cost columns, and a mini context-usage bar when the + * session carries a snapshot). Anything unparseable returns null and falls + * through to the default rendering. + */ + +import type { + FunctionTriggerMessage, + FunctionTriggerRenderer, +} from '@iii-dev/console-ui' +import { formatCost, formatTokens } from '../lib/format' +import { + type MetricsResponse, + type SessionUsage, + parseMetrics, +} from '../lib/metrics' + +const METRICS_ID = 'harness::metrics' + +function TotalsChip({ label, value }: { label: string; value: string }) { + return ( + + {label} + {value} + + ) +} + +function TotalsRow({ metrics }: { metrics: MetricsResponse }) { + const totals = metrics.totals + return ( +
+ + + {totals.input_tokens != null ? ( + + ) : null} + {totals.output_tokens != null ? ( + + ) : null} + {totals.cache_read_tokens != null ? ( + + ) : null} + {totals.cost_usd != null ? ( + + ) : null} +
+ ) +} + +function MiniUsageBar({ total, usable }: { total: number; usable: number }) { + if (usable <= 0) return null + const ratio = Math.min(1, total / usable) + const pct = Math.round(ratio * 100) + const color = + ratio >= 0.9 + ? 'var(--color-alert)' + : ratio >= 0.75 + ? 'var(--color-warn)' + : 'var(--color-accent)' + return ( + + + + ) +} + +function SessionRow({ row }: { row: SessionUsage }) { + return ( + + + + {row.session_id} + + + {row.turns ?? 0} + + {row.input_tokens != null ? formatTokens(row.input_tokens) : '—'} + + + {row.output_tokens != null ? formatTokens(row.output_tokens) : '—'} + + + {row.cost_usd != null ? formatCost(row.cost_usd) : '—'} + + + {row.context ? ( + + ) : null} + + + ) +} + +function MetricsCard({ metrics }: { metrics: MetricsResponse }) { + return ( +
+
+ metrics + {metrics.complete === false ? ( + partial + ) : null} + harness ui +
+ + {metrics.by_session.length > 0 ? ( +
+ + + + + + + + + + + + + {metrics.by_session.map((row) => ( + + ))} + +
sessionturnsinoutcostctx
+
+ ) : null} +
+ ) +} + +export function createMetricsRenderer(): FunctionTriggerRenderer { + return { + id: 'harness/page.js#metrics', + isMatch: (functionId) => functionId === METRICS_ID, + tryRender: (message: FunctionTriggerMessage) => { + const metrics = parseMetrics(message.output) + if (!metrics) return null + return + }, + } +} diff --git a/harness/ui/src/lib/format.ts b/harness/ui/src/lib/format.ts new file mode 100644 index 000000000..8947d478b --- /dev/null +++ b/harness/ui/src/lib/format.ts @@ -0,0 +1,17 @@ +/** Token-count formatting shared by the chip, popover, and metrics card. */ + +export function formatTokens(n: number): string { + if (!Number.isFinite(n) || n < 0) return '0' + if (n >= 1_000_000) return `${trimmed(n / 1_000_000)}m` + if (n >= 1000) return `${trimmed(n / 1000)}k` + return String(Math.round(n)) +} + +export function formatCost(usd: number): string { + return `$${usd.toFixed(4)}` +} + +function trimmed(value: number): string { + const fixed = value.toFixed(1) + return fixed.endsWith('.0') ? fixed.slice(0, -2) : fixed +} diff --git a/harness/ui/src/lib/metrics.ts b/harness/ui/src/lib/metrics.ts new file mode 100644 index 000000000..a6dc91595 --- /dev/null +++ b/harness/ui/src/lib/metrics.ts @@ -0,0 +1,115 @@ +/** + * Wire shapes of `harness::metrics` (SessionMetricsResponseV1) and the + * per-generation context snapshot (ContextSnapshotV1) it carries, plus the + * tolerant parsers the chip and the function-trigger renderer share. A + * snapshot from a newer harness must degrade to "no data", never a crash. + */ + +export interface SnapshotMessages { + user: number + assistant: number + function_result: number + custom: number +} + +export interface SnapshotCategories { + system_prompt: number + tools: number + messages?: SnapshotMessages + overhead: number + hook_guidance?: number +} + +export interface SnapshotUsage { + input?: number + output?: number + cache_read?: number + cache_write?: number + reasoning?: number + cost_usd?: number +} + +export interface ContextSnapshot { + session_id: string + turn_id: string + step: number + model: string + provider?: string + estimator?: string + usable: number + effective_max_output_tokens: number + total: number + free: number + categories: SnapshotCategories + compacted: boolean + summarized_head_tokens?: number + usage?: SnapshotUsage + timestamp: number +} + +export interface SessionUsage { + session_id: string + parent_session_id?: string + depth: number + turns: number + function_calls: number + function_call_errors: number + input_tokens?: number + output_tokens?: number + cache_read_tokens?: number + cache_write_tokens?: number + reasoning_tokens?: number + cost_usd?: number + context?: ContextSnapshot +} + +export interface MetricsTotals { + sessions: number + turns: number + function_calls: number + function_call_errors: number + input_tokens?: number + output_tokens?: number + cache_read_tokens?: number + cache_write_tokens?: number + reasoning_tokens?: number + cost_usd?: number +} + +export interface MetricsResponse { + root_session_id?: string + complete?: boolean + totals: MetricsTotals + by_session: SessionUsage[] +} + +/** `{ content: [...], details }` harness result envelope → details. */ +export function unwrapEnvelope(value: unknown): unknown { + if (!value || typeof value !== 'object' || Array.isArray(value)) return value + const obj = value as Record + if (Array.isArray(obj.content) && 'details' in obj) return obj.details + return value +} + +export function parseMetrics(value: unknown): MetricsResponse | null { + const raw = unwrapEnvelope(value) + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null + const obj = raw as Record + if (!Array.isArray(obj.by_session)) return null + if (!obj.totals || typeof obj.totals !== 'object' || Array.isArray(obj.totals)) { + return null + } + return obj as unknown as MetricsResponse +} + +export function isSnapshot(value: unknown): value is ContextSnapshot { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false + const snap = value as Record + return ( + typeof snap.total === 'number' && + typeof snap.usable === 'number' && + !!snap.categories && + typeof snap.categories === 'object' && + !Array.isArray(snap.categories) + ) +} diff --git a/harness/ui/styles.css b/harness/ui/styles.css new file mode 100644 index 000000000..25b23419b --- /dev/null +++ b/harness/ui/styles.css @@ -0,0 +1,275 @@ +/* + * The harness worker's console stylesheet, shipped as its own `console:style` + * asset (harness/styles.css) — the console mounts it as a in + * document.head and link-swaps it on hot reload, styles-before-scripts on + * boot. Every rule is scoped under `[data-iii-ui="harness"]`, the wrapper the + * console mounts around every injected render — session chip and + * function-trigger message alike. Styling uses the console's design tokens, + * so light/dark theming is free. + */ + +/* --- session chip (context) ------------------------------------------ */ +[data-iii-ui="harness"] .harness-ui-chip { + position: relative; + display: flex; + align-items: center; + gap: 6px; + font-family: var(--font-mono, ui-monospace, monospace); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--color-ink-faint); +} +[data-iii-ui="harness"] .harness-ui-chip-empty { + color: var(--color-ink-ghost); +} +[data-iii-ui="harness"] .harness-ui-chip-btn { + display: flex; + align-items: center; + gap: 6px; + background: transparent; + border: 0; + padding: 0; + cursor: pointer; + font-family: inherit; + font-size: inherit; + text-transform: inherit; + letter-spacing: inherit; + color: inherit; +} +[data-iii-ui="harness"] .harness-ui-chip-bar { + position: relative; + display: block; + width: 56px; + height: 6px; + background: var(--color-rule-2); + border: 1px solid var(--color-rule); + overflow: hidden; +} +[data-iii-ui="harness"] .harness-ui-chip-fill { + display: block; + height: 100%; + transition: width 0.2s, background-color 0.2s; +} +[data-iii-ui="harness"] .harness-ui-chip-pct { + font-variant-numeric: tabular-nums; +} +[data-iii-ui="harness"] .harness-ui-chip-counts { + color: var(--color-ink-ghost); + text-transform: none; + letter-spacing: normal; + font-variant-numeric: tabular-nums; +} + +/* --- context popover -------------------------------------------------- */ +[data-iii-ui="harness"] .harness-ui-pop { + position: absolute; + top: calc(100% + 6px); + right: 0; + z-index: 1000; + width: 300px; + padding: 12px; + display: flex; + flex-direction: column; + gap: 10px; + background: var(--color-panel); + border: 1px solid var(--color-rule); + color: var(--color-ink); + font-size: 12px; + text-transform: none; + letter-spacing: normal; + text-align: left; + cursor: default; +} +[data-iii-ui="harness"] .harness-ui-pop-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px; +} +[data-iii-ui="harness"] .harness-ui-pop-model { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; +} +[data-iii-ui="harness"] .harness-ui-pop-usage { + flex: none; + color: var(--color-ink-faint); + font-variant-numeric: tabular-nums; +} +[data-iii-ui="harness"] .harness-ui-stack { + display: flex; + width: 100%; + height: 8px; + background: var(--color-rule-2); + border: 1px solid var(--color-rule); + overflow: hidden; + box-sizing: border-box; +} +[data-iii-ui="harness"] .harness-ui-seg { + display: block; + height: 100%; + flex: none; +} +[data-iii-ui="harness"] .harness-ui-legend { + display: flex; + flex-direction: column; + gap: 4px; +} +[data-iii-ui="harness"] .harness-ui-legend-row { + display: flex; + align-items: center; + gap: 8px; + font-size: 11.5px; +} +[data-iii-ui="harness"] .harness-ui-swatch { + flex: none; + width: 8px; + height: 8px; +} +[data-iii-ui="harness"] .harness-ui-legend-label { + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--color-ink-faint); +} +[data-iii-ui="harness"] .harness-ui-legend-val { + font-variant-numeric: tabular-nums; +} +[data-iii-ui="harness"] .harness-ui-legend-pct { + flex: none; + width: 36px; + text-align: right; + color: var(--color-ink-ghost); + font-variant-numeric: tabular-nums; +} +[data-iii-ui="harness"] .harness-ui-badge { + flex: none; + border: 1px solid var(--color-warn); + color: var(--color-warn); + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.08em; + padding: 0 4px; +} +[data-iii-ui="harness"] .harness-ui-pop-foot { + display: flex; + flex-direction: column; + gap: 2px; + padding-top: 8px; + border-top: 1px solid var(--color-rule-2); + font-size: 11px; + color: var(--color-ink-ghost); +} + +/* --- function-trigger message (chat / traces card) ------------------- */ +[data-iii-ui="harness"] .harness-ui-msg { + border-top: 1px solid var(--color-rule-2); + background: var(--color-bg); + font-family: var(--font-mono, ui-monospace, monospace); +} +[data-iii-ui="harness"] .harness-ui-msg-head { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + flex-wrap: wrap; +} +[data-iii-ui="harness"] .harness-ui-pill { + border: 1px solid var(--color-accent); + color: var(--color-accent); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.08em; + padding: 1px 8px; +} +[data-iii-ui="harness"] .harness-ui-partial { + border: 1px solid var(--color-warn); + color: var(--color-warn); + font-size: 10.5px; + text-transform: uppercase; + letter-spacing: 0.08em; + padding: 1px 6px; +} +[data-iii-ui="harness"] .harness-ui-msg-tag { + margin-left: auto; + font-size: 10.5px; + text-transform: uppercase; + letter-spacing: 0.1em; + color: var(--color-ink-ghost); +} +[data-iii-ui="harness"] .harness-ui-totals { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; + padding: 0 12px 8px; +} +[data-iii-ui="harness"] .harness-ui-chip-stat { + border: 1px solid var(--color-rule); + color: var(--color-ink); + font-size: 11.5px; + padding: 1px 8px; + font-variant-numeric: tabular-nums; +} +[data-iii-ui="harness"] .harness-ui-chip-stat .k { + color: var(--color-ink-ghost); +} +[data-iii-ui="harness"] .harness-ui-table-wrap { + padding: 0 12px 10px; + overflow-x: auto; +} +[data-iii-ui="harness"] .harness-ui-table { + width: 100%; + border-collapse: collapse; + font-size: 11.5px; +} +[data-iii-ui="harness"] .harness-ui-table th { + text-align: left; + font-weight: 400; + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--color-ink-ghost); + border-bottom: 1px solid var(--color-rule-2); + padding: 3px 8px 3px 0; +} +[data-iii-ui="harness"] .harness-ui-table td { + color: var(--color-ink); + border-bottom: 1px solid var(--color-rule-2); + padding: 3px 8px 3px 0; + font-variant-numeric: tabular-nums; + white-space: nowrap; +} +[data-iii-ui="harness"] .harness-ui-table tr:last-child td { + border-bottom: 0; +} +[data-iii-ui="harness"] .harness-ui-table th.harness-ui-num, +[data-iii-ui="harness"] .harness-ui-table td.harness-ui-num { + text-align: right; +} +[data-iii-ui="harness"] .harness-ui-sid span { + display: inline-block; + max-width: 160px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + vertical-align: bottom; +} +[data-iii-ui="harness"] .harness-ui-mini-bar { + position: relative; + display: inline-block; + width: 48px; + height: 4px; + background: var(--color-rule-2); + border: 1px solid var(--color-rule); + overflow: hidden; + vertical-align: middle; +} +[data-iii-ui="harness"] .harness-ui-mini-fill { + display: block; + height: 100%; +} diff --git a/harness/ui/tsconfig.json b/harness/ui/tsconfig.json new file mode 100644 index 000000000..e5ac60540 --- /dev/null +++ b/harness/ui/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": true, + "noEmit": true, + "types": [] + }, + "include": ["page.tsx", "src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89d682eda..9ee8938f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -277,6 +277,22 @@ importers: specifier: ^5.9.2 version: 5.9.3 + harness/ui: + dependencies: + '@iii-dev/console-ui': + specifier: workspace:* + version: link:../../packages/console-ui + devDependencies: + '@types/react': + specifier: ^19.2.14 + version: 19.2.17 + esbuild: + specifier: ^0.25.0 + version: 0.25.12 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + iii-directory/ui: dependencies: '@iii-dev/console-ui': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index cf84811de..70cd916d2 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -14,6 +14,7 @@ packages: - database/ui - editor/ui - eval/ui + - harness/ui - memory/ui - state/ui - iii-directory/ui From 2c7f908d73438eb6679442f8897ec0afd740a829 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 3 Aug 2026 17:30:30 +0100 Subject: [PATCH 03/15] (MOT-4327) fix(harness): context chip shows the catalog window pre-turn and ticks mid-turn Before the first generation the chip now renders 0% of the model's catalog context window instead of a bare dash. And because snapshots are written after every generate step while the turn-completed push only fires at turn end, the chip additionally polls the session's harness_context state row every five seconds so long multi-step turns update live instead of staying frozen until the turn finishes. --- harness/ui/src/context-chip/index.tsx | 59 +++++++++++++++++++++------ 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index be23d263c..73d5d42f8 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -15,7 +15,6 @@ import { type ContextSnapshot, type SnapshotMessages, isSnapshot, - parseMetrics, } from '../lib/metrics' /** Per-tab handler id (host.iii.on namespaces it `::`). */ @@ -24,6 +23,10 @@ const EVENTS_FN = 'iii::harness-ui::events' const WARN_THRESHOLD = 0.75 const DANGER_THRESHOLD = 0.9 +/** Snapshots update after every generate step, but the push only fires at + turn end — poll the state row so long multi-step turns tick live. */ +const POLL_MS = 5_000 + export interface SessionChipProps { sessionId: string modelId?: string @@ -225,7 +228,11 @@ function ContextPopover({ } export function createContextChip(host: Host) { - return function ContextChip({ sessionId, modelId }: SessionChipProps) { + return function ContextChip({ + sessionId, + modelId, + contextWindow, + }: SessionChipProps) { const [snapshot, setSnapshot] = useState(null) const [open, setOpen] = useState(false) const rootRef = useRef(null) @@ -234,19 +241,21 @@ export function createContextChip(host: Host) { let cancelled = false setSnapshot(null) setOpen(false) - host.iii - .trigger('harness::metrics', { root_session_id: sessionId }) - .then((result) => { - if (cancelled) return - const metrics = parseMetrics(result) - const own = metrics?.by_session.find( - (row) => row.session_id === sessionId, - ) - if (own?.context && isSnapshot(own.context)) setSnapshot(own.context) - }) - .catch(() => {}) + const read = () => { + host.iii + .trigger('state::get', { scope: 'harness_context', key: sessionId }) + .then((value) => { + if (cancelled) return + if (isSnapshot(value) && value.session_id === sessionId) + setSnapshot(value) + }) + .catch(() => {}) + } + read() + const interval = window.setInterval(read, POLL_MS) return () => { cancelled = true + window.clearInterval(interval) } }, [host, sessionId]) @@ -284,6 +293,30 @@ export function createContextChip(host: Host) { }, [open]) if (!snapshot || snapshot.usable <= 0) { + if (contextWindow && contextWindow > 0) { + return ( +
+ ctx + + + + 0% + + 0/{formatTokens(contextWindow)} + +
+ ) + } return (
ctx From 5d19d1a3a84cf1ca91a09ff2071e8b800d064fff Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 3 Aug 2026 17:39:43 +0100 Subject: [PATCH 04/15] (MOT-4327) fix(harness): stream mid-turn context updates over the state trigger Drop the interval poll. The chip now binds the state worker's `state` trigger with an engine-side scope and key filter on the session's harness_context row, so every per-step snapshot write streams to the chip in real time. One state::get on mount hydrates; everything after is push. --- harness/ui/src/context-chip/index.tsx | 59 +++++++++++++++++++-------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index 73d5d42f8..e2510d87e 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -17,16 +17,13 @@ import { isSnapshot, } from '../lib/metrics' -/** Per-tab handler id (host.iii.on namespaces it `::`). */ +/** Per-tab handler ids (host.iii.on namespaces them `::`). */ const EVENTS_FN = 'iii::harness-ui::events' +const STATE_FN = 'iii::harness-ui::ctx-state' const WARN_THRESHOLD = 0.75 const DANGER_THRESHOLD = 0.9 -/** Snapshots update after every generate step, but the push only fires at - turn end — poll the state row so long multi-step turns tick live. */ -const POLL_MS = 5_000 - export interface SessionChipProps { sessionId: string modelId?: string @@ -40,6 +37,16 @@ interface TurnCompletedEvent { context?: unknown } +/** The message a `state` function trigger delivers per write (the state + worker's own streaming trigger type — no polling anywhere). */ +interface StateEvent { + type?: string + event_type?: string + scope?: string + key?: string + new_value?: unknown +} + type Tone = 'ok' | 'warn' | 'alert' const TONE_COLOR: Record = { @@ -241,21 +248,37 @@ export function createContextChip(host: Host) { let cancelled = false setSnapshot(null) setOpen(false) - const read = () => { - host.iii - .trigger('state::get', { scope: 'harness_context', key: sessionId }) - .then((value) => { - if (cancelled) return - if (isSnapshot(value) && value.session_id === sessionId) - setSnapshot(value) - }) - .catch(() => {}) - } - read() - const interval = window.setInterval(read, POLL_MS) + host.iii + .trigger('state::get', { scope: 'harness_context', key: sessionId }) + .then((value) => { + if (cancelled) return + if (isSnapshot(value) && value.session_id === sessionId) + setSnapshot(value) + }) + .catch(() => {}) return () => { cancelled = true - window.clearInterval(interval) + } + }, [host, sessionId]) + + // Snapshots are written after every generate step; the state worker's + // `state` trigger streams each write (engine-side scope/key filter), so + // long multi-step turns tick live without any polling. + useEffect(() => { + const offHandler = host.iii.on(STATE_FN, (event) => { + if (!event || event.key !== sessionId) return + if (event.event_type === 'state:deleted') return + if (isSnapshot(event.new_value) && event.new_value.session_id === sessionId) + setSnapshot(event.new_value) + }) + const offTrigger = host.iii.registerTrigger({ + type: 'state', + function_id: `${STATE_FN}::${host.iii.browserId}`, + config: { scope: 'harness_context', key: sessionId }, + }) + return () => { + offTrigger() + offHandler() } }, [host, sessionId]) From 81ded541fb5ab2196832185b4e7e196b916b0c82 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 3 Aug 2026 18:02:38 +0100 Subject: [PATCH 05/15] (MOT-4329) feat(harness): provider-exact context snapshot via router token counting When a provider token counter is available (router::count_tokens), the snapshot's categories become real numbers: the generation's billed usage is the exact total, the system prompt and tool schemas are counted through the provider's tokenizer with a probe-delta and cached per content hash (one wire count per session, not per step), and the message window is the remainder with the estimated by-role proportions rescaled onto it. The estimator field reports what produced the numbers; rigs without a counter keep the heuristic snapshot untouched. Counting never runs the model, never enters the session context, and costs nothing. --- harness/src/clients/router.rs | 42 +++++++++ harness/src/context_snapshot.rs | 162 ++++++++++++++++++++++++++++++++ harness/src/turn_loop.rs | 16 +++- 3 files changed, 217 insertions(+), 3 deletions(-) diff --git a/harness/src/clients/router.rs b/harness/src/clients/router.rs index 3344a9c9c..36fab9a55 100644 --- a/harness/src/clients/router.rs +++ b/harness/src/clients/router.rs @@ -450,6 +450,48 @@ impl RouterClient { .map(String::from) } + /// Exact token count for a request shape via the resolved provider's + /// tokenizer (`None` when the router is absent, the provider has no + /// counter, or the call fails — the caller falls back to its estimate). + /// Returns `(tokens, estimator)`. Counting never runs the model and + /// never enters the session context. + pub async fn count_tokens( + &self, + model: &str, + provider: Option<&str>, + system_prompt: Option<&str>, + tools: Option<&[AgentFunction]>, + messages: &[Value], + ) -> Option<(u64, String)> { + let mut payload = json!({ "model": model, "messages": messages }); + if let Some(p) = provider { + payload["provider"] = json!(p); + } + if let Some(sp) = system_prompt { + payload["system_prompt"] = json!(sp); + } + if let Some(t) = tools { + payload["tools"] = serde_json::to_value(t).ok()?; + } + let resp = self + .iii + .trigger(TriggerRequest { + function_id: "router::count_tokens".into(), + payload, + action: None, + timeout_ms: Some(self.timeout_ms), + }) + .await + .ok()?; + let tokens = resp.get("tokens").and_then(Value::as_u64)?; + let estimator = resp + .get("estimator") + .and_then(Value::as_str) + .unwrap_or("provider") + .to_string(); + Some((tokens, estimator)) + } + /// Look up one model's capabilities (`None` when unregistered or router /// absent — the caller degrades). pub async fn models_get(&self, provider: Option<&str>, id: &str) -> Option { diff --git a/harness/src/context_snapshot.rs b/harness/src/context_snapshot.rs index 584c40005..98e3d4b36 100644 --- a/harness/src/context_snapshot.rs +++ b/harness/src/context_snapshot.rs @@ -5,12 +5,21 @@ //! terminal frame, stored once per generate step. Read back by //! `harness::metrics` and pushed on `harness::turn-completed`. +use std::collections::hash_map::DefaultHasher; +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::sync::Mutex; +use std::sync::OnceLock; + use iii_sdk::IIIClient; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use serde_json::json; +use crate::clients::RouterClient; use crate::error::HarnessError; use crate::types::event::Usage; +use crate::types::model::AgentFunction; pub const CONTEXT_SCOPE: &str = "harness_context"; @@ -109,6 +118,159 @@ pub async fn delete( crate::state::state_delete(iii, CONTEXT_SCOPE, session_id, timeout_ms).await } +/// Process-lifetime cache of provider token counts keyed by +/// (model, kind, content hash) — the system prompt and tool schemas are +/// stable across a session's steps, so each is counted over the wire once. +fn count_cache() -> &'static Mutex> { + static CACHE: OnceLock>> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn cache_key(model: &str, kind: &str, content: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + model.hash(&mut hasher); + kind.hash(&mut hasher); + content.hash(&mut hasher); + hasher.finish() +} + +/// One-message probe the delta counts subtract away: provider metering +/// endpoints refuse an empty messages array, so category counts are +/// measured as count(probe + part) - count(probe). +fn probe_messages() -> Vec { + vec![json!({ + "role": "user", + "content": [{ "type": "text", "text": "x" }], + "timestamp": 0 + })] +} + +async fn counted_delta( + router: &RouterClient, + model: &str, + provider: Option<&str>, + kind: &str, + content_key: &str, + system_prompt: Option<&str>, + tools: Option<&[AgentFunction]>, +) -> Option<(u64, String)> { + let key = cache_key(model, kind, content_key); + if let Some(tokens) = count_cache().lock().ok()?.get(&key).copied() { + return Some((tokens, "provider".into())); + } + let probe = probe_messages(); + let (base, _) = router + .count_tokens(model, provider, None, None, &probe) + .await?; + let (with_part, estimator) = router + .count_tokens(model, provider, system_prompt, tools, &probe) + .await?; + let tokens = with_part.saturating_sub(base); + if let Ok(mut cache) = count_cache().lock() { + cache.insert(key, tokens); + } + Some((tokens, estimator)) +} + +/// Replace the snapshot's estimated categories with provider-exact numbers +/// where they exist: the generation's billed usage is the exact total, the +/// system prompt and tool schemas are counted once via the provider's +/// tokenizer (cached by content), and the message window is the remainder. +/// A rig without a provider counter keeps the heuristic snapshot untouched. +pub async fn exactify( + snapshot: &mut ContextSnapshotV1, + router: &RouterClient, + system_prompt: Option<&str>, + tools: &[AgentFunction], +) { + let Some(usage) = snapshot.usage.as_ref() else { + return; + }; + let billed = + usage.input.unwrap_or(0) + usage.cache_read.unwrap_or(0) + usage.cache_write.unwrap_or(0); + if billed == 0 { + return; + } + let provider = snapshot.provider.clone(); + let model = snapshot.model.clone(); + + let system_exact = match system_prompt { + Some(sp) if !sp.is_empty() => { + counted_delta( + &router.clone(), + &model, + provider.as_deref(), + "system_prompt", + sp, + Some(sp), + None, + ) + .await + } + _ => Some((0, String::new())), + }; + let tools_exact = if tools.is_empty() { + Some((0, String::new())) + } else { + let tools_key = serde_json::to_string(tools).unwrap_or_default(); + counted_delta( + &router.clone(), + &model, + provider.as_deref(), + "tools", + &tools_key, + None, + Some(tools), + ) + .await + }; + let (Some((system_tokens, sys_est)), Some((tools_tokens, tools_est))) = + (system_exact, tools_exact) + else { + return; + }; + let estimator = [sys_est, tools_est] + .into_iter() + .find(|e| !e.is_empty()) + .unwrap_or_else(|| "provider".into()); + + let remainder = billed + .saturating_sub(system_tokens) + .saturating_sub(tools_tokens); + let heuristic_messages = { + let m = &snapshot.categories.messages; + m.user + m.assistant + m.function_result + m.custom + }; + // Keep the by-role proportions from the estimate but rescale them onto + // the exact remainder (providers report only the request total). + let scaled = if heuristic_messages > 0 { + let scale = remainder as f64 / heuristic_messages as f64; + let m = &snapshot.categories.messages; + SnapshotMessagesV1 { + user: (m.user as f64 * scale) as u64, + assistant: (m.assistant as f64 * scale) as u64, + function_result: (m.function_result as f64 * scale) as u64, + custom: (m.custom as f64 * scale) as u64, + } + } else { + SnapshotMessagesV1 { + user: remainder, + assistant: 0, + function_result: 0, + custom: 0, + } + }; + + snapshot.categories.system_prompt = system_tokens; + snapshot.categories.tools = tools_tokens; + snapshot.categories.messages = scaled; + snapshot.categories.overhead = 0; + snapshot.categories.hook_guidance = 0; + snapshot.total = billed; + snapshot.free = snapshot.usable.saturating_sub(billed); + snapshot.estimator = Some(estimator); +} + #[cfg(test)] mod tests { use super::*; diff --git a/harness/src/turn_loop.rs b/harness/src/turn_loop.rs index 96e38c243..4e1d7c2cb 100644 --- a/harness/src/turn_loop.rs +++ b/harness/src/turn_loop.rs @@ -612,6 +612,8 @@ pub async fn run_step( entry_id: assistant_id.clone(), turn_id: record.turn_id.clone(), }; + let snapshot_system_prompt = gen_system_prompt.clone(); + let snapshot_tools = tools.clone(); let params = ChatParams { request_id: format!("{}:{}", record.turn_id, payload.step), model: record.options.model.clone(), @@ -706,11 +708,19 @@ pub async fn run_step( // advance()/finalize call that ends this step). record.watermark_entry_id = watermark; - // Stamp the generation's actual usage into the snapshot and store the - // session's latest copy. Best-effort: accounting must never fail a turn - // that generated successfully. + // Stamp the generation's actual usage into the snapshot, replace the + // estimated categories with provider-exact counts where a counter + // exists, and store the session's latest copy. Best-effort: accounting + // must never fail a turn that generated successfully. if let Some(snapshot) = record.context_snapshot.as_mut() { snapshot.usage = outcome.message.usage.clone(); + crate::context_snapshot::exactify( + snapshot, + &router, + snapshot_system_prompt.as_deref(), + &snapshot_tools, + ) + .await; if let Err(error) = crate::context_snapshot::put(&deps.iii, snapshot, cfg.session_timeout_ms).await { From 00907ce589ac8dc14693e05d834d132ad8784576 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 3 Aug 2026 18:20:46 +0100 Subject: [PATCH 06/15] (MOT-4329) fix(harness): context panel says exact when provider-counted The popover foot renders exact plus the counting source when the snapshot numbers came from a provider tokenizer, keeping est. only for the chars heuristic fallback. --- harness/ui/src/context-chip/index.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index e2510d87e..1ef046c4a 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -218,7 +218,15 @@ function ContextPopover({
- est. {snapshot.estimator ?? 'unknown'} + + {!snapshot.estimator || snapshot.estimator === 'heuristic' + ? `est. ${snapshot.estimator ?? 'unknown'}` + : `exact · ${ + snapshot.estimator === 'provider' + ? 'provider tokenizer' + : snapshot.estimator + }`} + {hasActuals ? ( last turn actual{' '} From 0cb2c9468c249f483cebca781efeb28344d82fdb Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Tue, 4 Aug 2026 09:16:42 +0100 Subject: [PATCH 07/15] (MOT-4327) refactor(harness): adopt the shared console-ui crate and slim the accounting paths Review cleanups, wire-identical (goldens unchanged). The console-ui crate's iii-sdk pin relaxes to a 0.21 range so harness can link it, deleting the 314-line wire-contract fork; harness keeps a small stub naming its two assets. exactify loses the probe-per-part duplication (base counted once and cached, the two category deltas run concurrently) and the empty-string sentinel; a rig without a counter now keeps its heuristic snapshot instead of stamping exact totals. The snapshot build reads the assemble breakdown through Default and a From impl, Assembled carries Applied whole, and metrics reads the snapshot off the turn record it already fetched instead of a second state round trip per session. Chip UI: the turn-completed subscription goes away (mount get plus the state trigger already cover every write, earlier and finer), snapshot category types match what the wire always sends, one categories() list feeds both the stacked bar and the legend, and the tone thresholds live once in lib/tone shared with the metrics renderer. --- crates/console-ui/Cargo.toml | 8 +- harness/Cargo.lock | 13 + harness/Cargo.toml | 3 + harness/src/context_snapshot.rs | 232 ++++++++----- harness/src/events.rs | 1 - harness/src/functions/metrics.rs | 8 +- harness/src/lib.rs | 1 - harness/src/main.rs | 6 +- harness/src/turn_loop.rs | 51 ++- harness/src/ui.rs | 314 ++---------------- harness/ui/src/context-chip/index.tsx | 252 +++++++------- .../ui/src/function-trigger-message/index.tsx | 8 +- harness/ui/src/lib/metrics.ts | 4 +- harness/ui/src/lib/tone.ts | 24 ++ 14 files changed, 384 insertions(+), 541 deletions(-) create mode 100644 harness/ui/src/lib/tone.ts diff --git a/crates/console-ui/Cargo.toml b/crates/console-ui/Cargo.toml index d9c92bf9f..e20ece1f7 100644 --- a/crates/console-ui/Cargo.toml +++ b/crates/console-ui/Cargo.toml @@ -13,9 +13,11 @@ repository = "https://github.com/iii-hq/workers" publish = false [dependencies] -# Same exact pin the workers carry — path-linking this crate must never be -# what drags a worker onto a different SDK build. -iii-sdk = "=0.21.6" +# Range, not an exact pin: workers on this repo's SDK line carry their own +# exact pins (0.21.6 for state, 0.21.8 for the harness) and cargo must be able +# to unify them with this crate. Path-linking this crate must never be what +# drags a worker onto a different SDK build, nor what blocks one. +iii-sdk = ">=0.21.6, <0.22" tokio = { version = "1", features = ["rt", "sync", "time", "fs"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/harness/Cargo.lock b/harness/Cargo.lock index e472f7aa2..1317fe0e5 100644 --- a/harness/Cargo.lock +++ b/harness/Cargo.lock @@ -508,6 +508,7 @@ dependencies = [ "async-trait", "clap", "globset", + "iii-console-ui", "iii-helpers", "iii-sdk", "jsonschema", @@ -802,6 +803,18 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "iii-console-ui" +version = "0.1.0" +dependencies = [ + "iii-sdk", + "schemars", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "iii-helpers" version = "0.21.8" diff --git a/harness/Cargo.toml b/harness/Cargo.toml index 34840b432..7d6e35834 100644 --- a/harness/Cargo.toml +++ b/harness/Cargo.toml @@ -25,6 +25,9 @@ iii-sdk = "=0.21.8" # 0.21.5 adds the live span-start push (`LiveSpanStartProcessor`), so the # console renders `harness::turn step` while it runs instead of on close. iii-helpers = "=0.21.8" +# Worker-side injectable console UI (content function + console:script/style +# triggers + hot-reload watcher) — direct link, never published. +iii-console-ui = { path = "../crates/console-ui" } tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time", "fs"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/harness/src/context_snapshot.rs b/harness/src/context_snapshot.rs index 98e3d4b36..f6939f1c4 100644 --- a/harness/src/context_snapshot.rs +++ b/harness/src/context_snapshot.rs @@ -32,6 +32,41 @@ pub struct SnapshotMessagesV1 { pub custom: u64, } +impl SnapshotMessagesV1 { + /// What the whole messages category contributes to the request total. + pub fn total(&self) -> u64 { + self.user + self.assistant + self.function_result + self.custom + } + + /// The same by-role proportions carried onto a different total + /// (`numerator / denominator`). `denominator == 0` has no proportions to + /// carry, so it yields all-zero. + fn scaled(&self, numerator: u64, denominator: u64) -> Self { + if denominator == 0 { + return Self::default(); + } + let scale = numerator as f64 / denominator as f64; + let apply = |tokens: u64| (tokens as f64 * scale) as u64; + Self { + user: apply(self.user), + assistant: apply(self.assistant), + function_result: apply(self.function_result), + custom: apply(self.custom), + } + } +} + +impl From for SnapshotMessagesV1 { + fn from(by_role: crate::clients::context::ByRoleTokens) -> Self { + Self { + user: by_role.user, + assistant: by_role.assistant, + function_result: by_role.function_result, + custom: by_role.custom, + } + } +} + /// Where the request's tokens sit. Categories are assembly-time estimates; /// `hook_guidance` is the measured growth after assembly (pre-generate hook /// appends and orphan-repair patches), 0 when the request left assembly @@ -119,8 +154,9 @@ pub async fn delete( } /// Process-lifetime cache of provider token counts keyed by -/// (model, kind, content hash) — the system prompt and tool schemas are -/// stable across a session's steps, so each is counted over the wire once. +/// (model, kind, content hash) — the probe base, the system prompt and the +/// tool schemas are stable across a session's steps, so each is counted over +/// the wire once. fn count_cache() -> &'static Mutex> { static CACHE: OnceLock>> = OnceLock::new(); CACHE.get_or_init(|| Mutex::new(HashMap::new())) @@ -134,6 +170,18 @@ fn cache_key(model: &str, kind: &str, content: &str) -> u64 { hasher.finish() } +/// Cache miss and a poisoned lock are the same thing to the caller: count it +/// over the wire again. +fn cached(key: u64) -> Option { + count_cache().lock().ok()?.get(&key).copied() +} + +fn cache(key: u64, tokens: u64) { + if let Ok(mut cache) = count_cache().lock() { + cache.insert(key, tokens); + } +} + /// One-message probe the delta counts subtract away: provider metering /// endpoints refuse an empty messages array, so category counts are /// measured as count(probe + part) - count(probe). @@ -145,31 +193,82 @@ fn probe_messages() -> Vec { })] } +/// The request part being counted. Carries everything the count needs: which +/// `count_tokens` field it fills, its cache kind, and its cache content key. +enum Part<'a> { + System(&'a str), + Tools(&'a [AgentFunction]), +} + +impl Part<'_> { + fn kind(&self) -> &'static str { + match self { + Part::System(_) => "system_prompt", + Part::Tools(_) => "tools", + } + } + + fn content_key(&self) -> String { + match self { + Part::System(prompt) => (*prompt).to_string(), + Part::Tools(tools) => serde_json::to_string(tools).unwrap_or_default(), + } + } +} + +/// One part's provider-exact token count. +#[derive(Debug, Default)] +struct Counted { + tokens: u64, + /// Which estimator answered, when this count went over the wire. `None` + /// for a part that was absent or served from cache. + estimator: Option, +} + +/// Tokens the probe alone costs, so every part's delta subtracts the same +/// base. Counted once per (model, provider) and cached for the process. +async fn probe_base(router: &RouterClient, model: &str, provider: Option<&str>) -> Option { + let key = cache_key(model, "base", ""); + if let Some(tokens) = cached(key) { + return Some(tokens); + } + let (base, _) = router + .count_tokens(model, provider, None, None, &probe_messages()) + .await?; + cache(key, base); + Some(base) +} + +/// Provider-exact tokens for one part: count(probe + part) - `base`, cached +/// by content. `None` means the provider count failed and the caller must +/// keep its heuristic numbers. async fn counted_delta( router: &RouterClient, model: &str, provider: Option<&str>, - kind: &str, - content_key: &str, - system_prompt: Option<&str>, - tools: Option<&[AgentFunction]>, -) -> Option<(u64, String)> { - let key = cache_key(model, kind, content_key); - if let Some(tokens) = count_cache().lock().ok()?.get(&key).copied() { - return Some((tokens, "provider".into())); + base: u64, + part: Part<'_>, +) -> Option { + let key = cache_key(model, part.kind(), &part.content_key()); + if let Some(tokens) = cached(key) { + return Some(Counted { + tokens, + estimator: None, + }); } - let probe = probe_messages(); - let (base, _) = router - .count_tokens(model, provider, None, None, &probe) - .await?; + let (system_prompt, tools) = match part { + Part::System(prompt) => (Some(prompt), None), + Part::Tools(tools) => (None, Some(tools)), + }; let (with_part, estimator) = router - .count_tokens(model, provider, system_prompt, tools, &probe) + .count_tokens(model, provider, system_prompt, tools, &probe_messages()) .await?; let tokens = with_part.saturating_sub(base); - if let Ok(mut cache) = count_cache().lock() { - cache.insert(key, tokens); - } - Some((tokens, estimator)) + cache(key, tokens); + Some(Counted { + tokens, + estimator: Some(estimator), + }) } /// Replace the snapshot's estimated categories with provider-exact numbers @@ -192,78 +291,61 @@ pub async fn exactify( return; } let provider = snapshot.provider.clone(); + let provider = provider.as_deref(); let model = snapshot.model.clone(); - let system_exact = match system_prompt { - Some(sp) if !sp.is_empty() => { - counted_delta( - &router.clone(), - &model, - provider.as_deref(), - "system_prompt", - sp, - Some(sp), - None, - ) - .await - } - _ => Some((0, String::new())), - }; - let tools_exact = if tools.is_empty() { - Some((0, String::new())) - } else { - let tools_key = serde_json::to_string(tools).unwrap_or_default(); - counted_delta( - &router.clone(), - &model, - provider.as_deref(), - "tools", - &tools_key, - None, - Some(tools), - ) - .await + // Every part's delta subtracts the same probe base, so count it once for + // the whole snapshot rather than once per part. + let Some(base) = probe_base(router, &model, provider).await else { + return; }; - let (Some((system_tokens, sys_est)), Some((tools_tokens, tools_est))) = - (system_exact, tools_exact) - else { + + let system_part = system_prompt.filter(|p| !p.is_empty()).map(Part::System); + let tools_part = (!tools.is_empty()).then_some(Part::Tools(tools)); + // Independent round trips: run them together. + let (counted_system, counted_tools) = tokio::join!( + async { + match system_part { + Some(part) => counted_delta(router, &model, provider, base, part).await, + None => Some(Counted::default()), + } + }, + async { + match tools_part { + Some(part) => counted_delta(router, &model, provider, base, part).await, + None => Some(Counted::default()), + } + }, + ); + let (Some(counted_system), Some(counted_tools)) = (counted_system, counted_tools) else { return; }; - let estimator = [sys_est, tools_est] - .into_iter() - .find(|e| !e.is_empty()) + let estimator = counted_system + .estimator + .or(counted_tools.estimator) .unwrap_or_else(|| "provider".into()); let remainder = billed - .saturating_sub(system_tokens) - .saturating_sub(tools_tokens); - let heuristic_messages = { - let m = &snapshot.categories.messages; - m.user + m.assistant + m.function_result + m.custom - }; + .saturating_sub(counted_system.tokens) + .saturating_sub(counted_tools.tokens); + let heuristic_messages = snapshot.categories.messages.total(); // Keep the by-role proportions from the estimate but rescale them onto // the exact remainder (providers report only the request total). - let scaled = if heuristic_messages > 0 { - let scale = remainder as f64 / heuristic_messages as f64; - let m = &snapshot.categories.messages; - SnapshotMessagesV1 { - user: (m.user as f64 * scale) as u64, - assistant: (m.assistant as f64 * scale) as u64, - function_result: (m.function_result as f64 * scale) as u64, - custom: (m.custom as f64 * scale) as u64, - } + let messages = if heuristic_messages > 0 { + snapshot + .categories + .messages + .scaled(remainder, heuristic_messages) } else { SnapshotMessagesV1 { user: remainder, - assistant: 0, - function_result: 0, - custom: 0, + ..SnapshotMessagesV1::default() } }; - snapshot.categories.system_prompt = system_tokens; - snapshot.categories.tools = tools_tokens; - snapshot.categories.messages = scaled; + snapshot.categories.system_prompt = counted_system.tokens; + snapshot.categories.tools = counted_tools.tokens; + snapshot.categories.messages = messages; snapshot.categories.overhead = 0; snapshot.categories.hook_guidance = 0; snapshot.total = billed; diff --git a/harness/src/events.rs b/harness/src/events.rs index fa29d6f2a..c3960cb78 100644 --- a/harness/src/events.rs +++ b/harness/src/events.rs @@ -420,7 +420,6 @@ impl TurnEvents { .await; } - #[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)] pub async fn emit_completed( &self, diff --git a/harness/src/functions/metrics.rs b/harness/src/functions/metrics.rs index de9c4542e..098506550 100644 --- a/harness/src/functions/metrics.rs +++ b/harness/src/functions/metrics.rs @@ -159,11 +159,9 @@ pub async fn handle( total.observe(message); } } - let context = - crate::context_snapshot::get(&deps.iii, &node.session_id, cfg.session_timeout_ms) - .await - .unwrap_or(None); - by_session.push(current.finish_session(node, context)); + // The turn record already carries the session's latest snapshot; do + // not spend a second state round trip per session re-reading it. + by_session.push(current.finish_session(node, turn.context_snapshot.clone())); } // Partial snapshots are polled as progress signals. Trace aggregation is // comparatively expensive and does not help the watchdog decide whether diff --git a/harness/src/lib.rs b/harness/src/lib.rs index 2d787ad74..e44713346 100644 --- a/harness/src/lib.rs +++ b/harness/src/lib.rs @@ -36,4 +36,3 @@ pub(crate) mod trace_tags; pub mod trigger; pub mod turn_loop; pub mod types; -pub mod ui; diff --git a/harness/src/main.rs b/harness/src/main.rs index 015352d6b..85b23d355 100644 --- a/harness/src/main.rs +++ b/harness/src/main.rs @@ -32,7 +32,11 @@ use harness::configuration::{self, ConfigCell, TriggerHandles}; use harness::deps::Deps; use harness::events::TurnEvents; use harness::hooks::HookRegistry; -use harness::{config, discovery, functions, manifest, queue, subscriptions, ui}; +use harness::{config, discovery, functions, manifest, queue, subscriptions}; + +/// Asset names + embedded bytes for the injected console UI; the registration +/// contract itself lives in the shared `iii-console-ui` crate. +mod ui; #[derive(Parser, Debug)] #[command( diff --git a/harness/src/turn_loop.rs b/harness/src/turn_loop.rs index 4e1d7c2cb..a61c0ba5b 100644 --- a/harness/src/turn_loop.rs +++ b/harness/src/turn_loop.rs @@ -612,15 +612,15 @@ pub async fn run_step( entry_id: assistant_id.clone(), turn_id: record.turn_id.clone(), }; - let snapshot_system_prompt = gen_system_prompt.clone(); - let snapshot_tools = tools.clone(); let params = ChatParams { request_id: format!("{}:{}", record.turn_id, payload.step), model: record.options.model.clone(), provider: record.options.provider.clone(), - system_prompt: gen_system_prompt, + // Cloned into the request so the originals stay available for the + // post-generation exact recount below. + system_prompt: gen_system_prompt.clone(), messages: gen_messages, - tools, + tools: tools.clone(), response_format, // Forward a cap only when the caller set one. `generation_max_output_tokens` // is the internal reservation context assembly budgets against; sending it @@ -714,13 +714,8 @@ pub async fn run_step( // must never fail a turn that generated successfully. if let Some(snapshot) = record.context_snapshot.as_mut() { snapshot.usage = outcome.message.usage.clone(); - crate::context_snapshot::exactify( - snapshot, - &router, - snapshot_system_prompt.as_deref(), - &snapshot_tools, - ) - .await; + crate::context_snapshot::exactify(snapshot, &router, gen_system_prompt.as_deref(), &tools) + .await; if let Err(error) = crate::context_snapshot::put(&deps.iii, snapshot, cfg.session_timeout_ms).await { @@ -2174,8 +2169,7 @@ async fn assemble_context( usable: out.usable, token_count: out.token_count, effective_max_output_tokens: out.effective_max_output_tokens, - compacted: out.applied.compacted, - summarized_head_tokens: out.applied.summarized_head_tokens, + applied: out.applied, breakdown: out.breakdown, }) } @@ -2195,36 +2189,30 @@ fn build_context_snapshot( final_request_tokens: u64, request_overhead_tokens: u64, ) -> crate::context_snapshot::ContextSnapshotV1 { - use crate::context_snapshot::{ContextSnapshotV1, SnapshotCategoriesV1, SnapshotMessagesV1}; - let breakdown = assembled.breakdown.as_ref(); - let messages = breakdown - .map(|b| SnapshotMessagesV1 { - user: b.by_role.user, - assistant: b.by_role.assistant, - function_result: b.by_role.function_result, - custom: b.by_role.custom, - }) - .unwrap_or_default(); + use crate::context_snapshot::{ContextSnapshotV1, SnapshotCategoriesV1}; + // A context-manager that predates the breakdown response reports nothing, + // which is what the all-zero default says. + let b = assembled.breakdown.clone().unwrap_or_default(); ContextSnapshotV1 { session_id: record.session_id.clone(), turn_id: record.turn_id.clone(), step, model: record.options.model.clone(), provider: record.options.provider.clone(), - estimator: breakdown.and_then(|b| b.estimator.clone()), + estimator: b.estimator, usable: assembled.usable, effective_max_output_tokens: assembled.effective_max_output_tokens, total: final_request_tokens, free: assembled.usable.saturating_sub(final_request_tokens), categories: SnapshotCategoriesV1 { - system_prompt: breakdown.map(|b| b.system_prompt_tokens).unwrap_or(0), - tools: breakdown.map(|b| b.tools_tokens).unwrap_or(0), - messages, + system_prompt: b.system_prompt_tokens, + tools: b.tools_tokens, + messages: b.by_role.into(), overhead: request_overhead_tokens, hook_guidance: final_request_tokens.saturating_sub(assembled.token_count), }, - compacted: assembled.compacted, - summarized_head_tokens: assembled.summarized_head_tokens, + compacted: assembled.applied.compacted, + summarized_head_tokens: assembled.applied.summarized_head_tokens, usage: None, timestamp: AgentMessage::now_ms(), } @@ -2301,8 +2289,9 @@ struct Assembled { token_count: u64, /// Model/output ceiling resolved by context-manager for this request. effective_max_output_tokens: u64, - compacted: bool, - summarized_head_tokens: Option, + /// What context-manager did to fit the window (compaction and its + /// bookkeeping), carried whole for the snapshot. + applied: crate::clients::context::Applied, breakdown: Option, } diff --git a/harness/src/ui.rs b/harness/src/ui.rs index cc4dc420a..c11964d9c 100644 --- a/harness/src/ui.rs +++ b/harness/src/ui.rs @@ -13,17 +13,11 @@ //! scoped under `[data-iii-ui="harness"]`; the console mounts it as a //! `` and link-swaps it on change, styles-before-scripts on boot. //! -//! Other workers get the registration machinery from the shared -//! `iii-console-ui` crate (`workers/crates/console-ui`), but that crate -//! pins `iii-sdk = "=0.21.6"` while the harness needs `=0.21.8` (the -//! reconnect reattach handshake) — two semver-compatible exact pins cargo -//! cannot co-resolve. This module therefore implements the same wire -//! contract against the harness's own SDK: the content function -//! `harness::ui-content` (`{path}` in, `{content, content_type}` out, -//! flagged internal), one Message-path trigger per asset (never -//! `engine::register_trigger`), and the `III_HARNESS_UI_WATCH` dev poller -//! (swap the served bytes, register a FRESH trigger for the same path, -//! THEN unregister the previous handle). +//! The registration machinery (content function `harness::ui-content`, one +//! Message-path trigger per asset, `III_HARNESS_UI_WATCH` hot-reload watcher) +//! lives in the shared `iii-console-ui` crate (path-linked from +//! `workers/crates/console-ui`); this module only names the assets and +//! embeds their bytes. //! //! The assets are compiled from `ui/` by esbuild (react + @iii-dev/console-ui //! external — they resolve through the console's import map at runtime) and @@ -33,274 +27,39 @@ //! re-registers a changed asset's trigger — every open console tab //! hot-swaps it. -use std::collections::HashMap; -use std::path::PathBuf; use std::sync::Arc; -use std::time::Duration; -use iii_sdk::errors::Error; -use iii_sdk::protocol::RegisterTriggerInput; -use iii_sdk::{IIIClient, RegisterFunction}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use tokio::sync::RwLock; +use iii_console_ui::ConsoleUi; +use iii_sdk::IIIClient; pub const PAGE_PATH: &str = "harness/page.js"; pub const STYLES_PATH: &str = "harness/styles.css"; -const CONTENT_FN: &str = "harness::ui-content"; -const WATCH_ENV: &str = "III_HARNESS_UI_WATCH"; -const WATCH_DEFAULT_DIR: &str = "ui/dist"; -const WATCH_POLL: Duration = Duration::from_millis(1000); - /// Built by `build.rs` (esbuild over `ui/`). const PAGE_JS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/page.js")); const STYLES_CSS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/styles.css")); -/// Input of the content function: the console asks for one asset by path. -#[derive(Debug, Clone, Deserialize, JsonSchema)] -pub struct UiContentInput { - pub path: String, -} - -/// Output of the content function. -#[derive(Debug, Clone, Serialize, JsonSchema)] -pub struct UiContentResult { - pub content: String, - pub content_type: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum AssetKind { - Script, - Style, -} - -impl AssetKind { - fn trigger_type(self) -> &'static str { - match self { - AssetKind::Script => "console:script", - AssetKind::Style => "console:style", - } - } - - fn content_type(self) -> &'static str { - match self { - AssetKind::Script => "text/javascript; charset=utf-8", - AssetKind::Style => "text/css; charset=utf-8", - } - } +fn console_ui() -> ConsoleUi { + ConsoleUi::new("harness") + .script(PAGE_PATH, PAGE_JS) + .style(STYLES_PATH, STYLES_CSS) } -struct AssetSpec { - path: &'static str, - kind: AssetKind, - /// File inside the watch dir the dev poller reads (last path segment). - file: &'static str, - content: &'static str, -} - -const ASSETS: [AssetSpec; 2] = [ - AssetSpec { - path: PAGE_PATH, - kind: AssetKind::Script, - file: "page.js", - content: PAGE_JS, - }, - AssetSpec { - path: STYLES_PATH, - kind: AssetKind::Style, - file: "styles.css", - content: STYLES_CSS, - }, -]; - -/// Register the harness's console UI: the content function plus one -/// Message-path trigger per asset, and the dev watcher when -/// `III_HARNESS_UI_WATCH` is set. Call after `functions::register_all`. -/// Trigger registration failures are warn-logged, not fatal — injected UI -/// is an accessory, and the SDK replays surviving registrations on -/// reconnect. +/// Register the harness's console UI. Call after `functions::register_all`. pub fn register(iii: &Arc) { - let served = Arc::new(Served::new(&ASSETS)); - - { - let served = served.clone(); - iii.register_function( - CONTENT_FN, - RegisterFunction::new_async(move |input: UiContentInput| { - let served = served.clone(); - async move { served.content_for(&input.path).await } - }) - .description( - "Serve the harness worker's injected console UI assets (content function \ - for its console:script / console:style triggers).", - ) - .metadata(serde_json::json!({ "internal": true })), - ); - } - - let mut watched = Vec::new(); - for spec in &ASSETS { - match register_asset_trigger(iii, spec.kind, spec.path) { - Ok(handle) => { - tracing::info!(path = spec.path, "registered console ui asset"); - watched.push(WatchedAsset { - path: spec.path, - kind: spec.kind, - file: spec.file, - handle, - prev: spec.content.to_string(), - }); - } - Err(e) => tracing::warn!( - error = %e, - path = spec.path, - "failed to register console ui trigger" - ), - } - } - - if let Some(dist) = watch_target() { - if watched.is_empty() { - tracing::warn!("ui watch requested but no ui trigger registered — watcher not started"); - } else { - spawn_watcher(iii.clone(), served, watched, dist); - } - } -} - -/// Serve the assets from in-memory cells so the dev watcher can swap the -/// bytes without re-registering the function. -struct Served { - content: RwLock>, -} - -struct ServedAsset { - content: String, - content_type: &'static str, -} - -impl Served { - fn new(assets: &[AssetSpec]) -> Self { - Self { - content: RwLock::new( - assets - .iter() - .map(|a| { - ( - a.path, - ServedAsset { - content: a.content.to_string(), - content_type: a.kind.content_type(), - }, - ) - }) - .collect(), - ), - } - } - - async fn content_for(&self, path: &str) -> Result { - let guard = self.content.read().await; - let asset = guard.get(path).ok_or_else(|| { - Error::Handler(format!( - "UNKNOWN_UI_ASSET: '{path}' is not one of this worker's ui assets \ - (expected one of: {PAGE_PATH}, {STYLES_PATH})" - )) - })?; - Ok(UiContentResult { - content: asset.content.clone(), - content_type: asset.content_type.to_string(), - }) - } - - async fn swap(&self, path: &str, next: String) { - if let Some(asset) = self.content.write().await.get_mut(path) { - asset.content = next; - } - } -} - -fn register_asset_trigger( - iii: &Arc, - kind: AssetKind, - path: &str, -) -> Result { - iii.register_trigger(RegisterTriggerInput { - trigger_type: kind.trigger_type().to_string(), - function_id: CONTENT_FN.to_string(), - config: serde_json::json!({ "path": path }), - metadata: None, - }) -} - -fn watch_target() -> Option { - parse_watch_target(&std::env::var(WATCH_ENV).ok()?) -} - -fn parse_watch_target(raw: &str) -> Option { - if raw.is_empty() || raw == "0" || raw.eq_ignore_ascii_case("false") { - return None; - } - if raw == "1" || raw.eq_ignore_ascii_case("true") { - return Some(PathBuf::from(WATCH_DEFAULT_DIR)); - } - Some(PathBuf::from(raw)) -} - -struct WatchedAsset { - path: &'static str, - kind: AssetKind, - file: &'static str, - handle: iii_sdk::trigger::Trigger, - prev: String, -} - -/// Dev-loop hot reload: poll the built files; on change, swap the served -/// bytes, register a fresh trigger for the same path (the console supersedes -/// + re-fetches + pushes to every tab), THEN unregister the previous handle. -fn spawn_watcher( - iii: Arc, - served: Arc, - mut watched: Vec, - dist: PathBuf, -) { - tokio::spawn(async move { - tracing::info!(dir = %dist.display(), "ui watch enabled — hot reload on rebuild"); - loop { - tokio::time::sleep(WATCH_POLL).await; - for asset in watched.iter_mut() { - let file = dist.join(asset.file); - let Ok(next) = tokio::fs::read_to_string(&file).await else { - continue; - }; - if next == asset.prev { - continue; - } - served.swap(asset.path, next.clone()).await; - asset.prev = next; - match register_asset_trigger(&iii, asset.kind, asset.path) { - Ok(next_handle) => { - let old = std::mem::replace(&mut asset.handle, next_handle); - old.unregister(); - tracing::info!(path = asset.path, "ui asset re-registered (hot reload)"); - } - Err(e) => tracing::warn!( - error = %e, - path = asset.path, - "ui hot-reload re-register failed" - ), - } - } - } - }); + console_ui().register(iii); } #[cfg(test)] mod tests { use super::*; + #[test] + fn ui_builder_accepts_the_assets() { + // The builder panics on any path/kind the console would reject. + let _ = console_ui(); + } + #[test] fn embedded_page_is_nonempty_esm() { assert!(PAGE_JS.contains("export"), "built page.js looks wrong"); @@ -315,39 +74,4 @@ mod tests { "built styles.css must be scoped under the worker's data-iii-ui attribute" ); } - - #[tokio::test] - async fn serves_registered_assets_with_content_types() { - let served = Served::new(&ASSETS); - let page = served.content_for(PAGE_PATH).await.unwrap(); - assert!(page.content_type.starts_with("text/javascript")); - let styles = served.content_for(STYLES_PATH).await.unwrap(); - assert!(styles.content_type.starts_with("text/css")); - } - - #[tokio::test] - async fn unknown_path_errors_and_names_the_known_paths() { - let err = Served::new(&ASSETS) - .content_for("harness/nope.js") - .await - .unwrap_err(); - let msg = err.to_string(); - assert!(msg.contains("UNKNOWN_UI_ASSET")); - assert!(msg.contains(PAGE_PATH)); - assert!(msg.contains(STYLES_PATH)); - } - - #[test] - fn parse_watch_target_conventions() { - assert_eq!(parse_watch_target(""), None); - assert_eq!(parse_watch_target("0"), None); - assert_eq!(parse_watch_target("false"), None); - assert_eq!(parse_watch_target("FALSE"), None); - assert_eq!(parse_watch_target("1"), Some(PathBuf::from("ui/dist"))); - assert_eq!(parse_watch_target("true"), Some(PathBuf::from("ui/dist"))); - assert_eq!( - parse_watch_target("build/out"), - Some(PathBuf::from("build/out")) - ); - } } diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index 1ef046c4a..db996a285 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -1,42 +1,29 @@ /** * The `context` session chip — a live view of the session's context window * matching the console's ContextUsage aesthetic (`ctx` label, bordered bar, - * percent, `12.3k/200k` counts). Hydrates from `harness::metrics` on mount - * and per session change; stays live over the worker's own - * `harness::turn-completed` trigger (Message-path binding, GC'd with the - * tab). Click toggles an anchored popover breaking the window down by - * category with a stacked segment bar, legend, and last-turn actuals. + * percent, `12.3k/200k` counts). Hydrates from the stored snapshot + * (`state::get` on `harness_context/`) on mount and per session + * change; stays live over the state worker's own `state` trigger for that + * key (Message-path binding, GC'd with the tab), which fires on every + * generate step. Click toggles an anchored popover breaking the window down + * by category with a stacked segment bar, legend, and last-turn actuals. */ import { useEffect, useRef, useState } from 'react' import type { Host } from '@iii-dev/console-ui' import { formatCost, formatTokens } from '../lib/format' -import { - type ContextSnapshot, - type SnapshotMessages, - isSnapshot, -} from '../lib/metrics' +import { type ContextSnapshot, isSnapshot } from '../lib/metrics' +import { TONE_COLOR, toneFor } from '../lib/tone' -/** Per-tab handler ids (host.iii.on namespaces them `::`). */ -const EVENTS_FN = 'iii::harness-ui::events' +/** Per-tab handler id (host.iii.on namespaces it `::`). */ const STATE_FN = 'iii::harness-ui::ctx-state' -const WARN_THRESHOLD = 0.75 -const DANGER_THRESHOLD = 0.9 - export interface SessionChipProps { sessionId: string modelId?: string contextWindow?: number } -interface TurnCompletedEvent { - session_id?: string - turn_id?: string - terminal?: boolean - context?: unknown -} - /** The message a `state` function trigger delivers per write (the state worker's own streaming trigger type — no polling anywhere). */ interface StateEvent { @@ -47,59 +34,135 @@ interface StateEvent { new_value?: unknown } -type Tone = 'ok' | 'warn' | 'alert' - -const TONE_COLOR: Record = { - ok: 'var(--color-accent)', - warn: 'var(--color-warn)', - alert: 'var(--color-alert)', -} - -function toneFor(ratio: number): Tone { - if (ratio >= DANGER_THRESHOLD) return 'alert' - if (ratio >= WARN_THRESHOLD) return 'warn' - return 'ok' -} - const ink = (opacity: number) => `color-mix(in srgb, var(--color-ink) ${opacity}%, transparent)` const accent = (opacity: number) => `color-mix(in srgb, var(--color-accent) ${opacity}%, transparent)` -const COLOR_SYSTEM = ink(80) -const COLOR_TOOLS = ink(55) -const COLOR_USER = accent(95) -const COLOR_ASSISTANT = accent(70) -const COLOR_RESULTS = accent(45) -const COLOR_HOOKS = ink(35) -const COLOR_OVERHEAD = ink(20) const COLOR_FREE = 'var(--color-rule-2)' -const EMPTY_MESSAGES: SnapshotMessages = { - user: 0, - assistant: 0, - function_result: 0, - custom: 0, +type CategoryKey = + | 'system' + | 'tools' + | 'user' + | 'assistant' + | 'results' + | 'hooks' + | 'overhead' + +interface Category { + key: CategoryKey + label: string + color: string + tokens: number } -function segments(snapshot: ContextSnapshot) { +/** The three categories the legend shows as one "Conversation" row. */ +const CONVERSATION_KEYS: CategoryKey[] = ['user', 'assistant', 'results'] + +/** + * Every category of the assembled window, in bar order. The single source + * for both the stacked bar (which draws each entry) and the legend (which + * merges the conversation entries into one row). + */ +function categories(snapshot: ContextSnapshot): Category[] { const cats = snapshot.categories - const messages = cats.messages ?? EMPTY_MESSAGES + const messages = cats.messages return [ - { key: 'system', tokens: cats.system_prompt ?? 0, color: COLOR_SYSTEM }, - { key: 'tools', tokens: cats.tools ?? 0, color: COLOR_TOOLS }, - { key: 'user', tokens: messages.user ?? 0, color: COLOR_USER }, - { key: 'assistant', tokens: messages.assistant ?? 0, color: COLOR_ASSISTANT }, + { + key: 'system', + label: 'System prompt', + color: ink(80), + tokens: cats.system_prompt, + }, + { + key: 'tools', + label: 'Function schemas', + color: ink(55), + tokens: cats.tools, + }, + { key: 'user', label: 'User', color: accent(95), tokens: messages.user }, + { + key: 'assistant', + label: 'Assistant', + color: accent(70), + tokens: messages.assistant, + }, { key: 'results', - tokens: (messages.function_result ?? 0) + (messages.custom ?? 0), - color: COLOR_RESULTS, + label: 'Function results', + color: accent(45), + tokens: messages.function_result + messages.custom, + }, + { + key: 'hooks', + label: 'Hook guidance', + color: ink(35), + // Optional on the wire (serde default): absent in snapshots written + // before the category existed. + tokens: cats.hook_guidance ?? 0, + }, + { + key: 'overhead', + label: 'Overhead', + color: ink(20), + tokens: cats.overhead, }, - { key: 'hooks', tokens: cats.hook_guidance ?? 0, color: COLOR_HOOKS }, - { key: 'overhead', tokens: cats.overhead ?? 0, color: COLOR_OVERHEAD }, ] } +interface LegendEntry { + key: string + label: string + /** `null` renders an invisible swatch — the row is not a bar segment. */ + color: string | null + tokens: number + badge?: string +} + +/** + * The legend, derived from the same category array the bar draws: the three + * conversation entries collapse into one row, an empty hook row is dropped, + * and the two rows that are not bar segments (the compaction summary, free + * space) take their place around the overhead row. + */ +function legendRows( + snapshot: ContextSnapshot, + cats: Category[], +): LegendEntry[] { + const conversation = cats.filter((c) => CONVERSATION_KEYS.includes(c.key)) + const rows: LegendEntry[] = [] + for (const category of cats) { + if (category.key === 'assistant') { + rows.push({ + ...category, + label: 'Conversation', + tokens: conversation.reduce((sum, entry) => sum + entry.tokens, 0), + }) + continue + } + if (CONVERSATION_KEYS.includes(category.key)) continue + if (category.key === 'hooks' && category.tokens <= 0) continue + if (category.key === 'overhead' && snapshot.compacted) { + rows.push({ + key: 'summary', + label: 'Summary', + color: null, + tokens: snapshot.summarized_head_tokens ?? 0, + badge: 'compacted', + }) + } + rows.push(category) + } + rows.push({ + key: 'free', + label: 'Free', + color: COLOR_FREE, + tokens: snapshot.free, + }) + return rows +} + function LegendRow({ color, label, @@ -138,14 +201,7 @@ function ContextPopover({ const usable = snapshot.usable const pct = usable > 0 ? Math.round(Math.min(1, snapshot.total / usable) * 100) : 0 - const messages = snapshot.categories.messages ?? EMPTY_MESSAGES - const conversation = - (messages.user ?? 0) + - (messages.assistant ?? 0) + - (messages.function_result ?? 0) + - (messages.custom ?? 0) - const hookGuidance = snapshot.categories.hook_guidance ?? 0 - const free = snapshot.free ?? Math.max(0, usable - snapshot.total) + const cats = categories(snapshot) const usage = snapshot.usage const hasActuals = usage != null && (usage.input != null || usage.cache_read != null) @@ -160,7 +216,7 @@ function ContextPopover({
- {segments(snapshot) + {cats .filter((segment) => segment.tokens > 0) .map((segment) => (
- - - - {hookGuidance > 0 ? ( + {legendRows(snapshot, cats).map((row) => ( - ) : null} - {snapshot.compacted ? ( - - ) : null} - - + ))}
@@ -290,22 +314,6 @@ export function createContextChip(host: Host) { } }, [host, sessionId]) - useEffect(() => { - const offHandler = host.iii.on(EVENTS_FN, (event) => { - if (!event || event.session_id !== sessionId) return - if (isSnapshot(event.context)) setSnapshot(event.context) - }) - const offTrigger = host.iii.registerTrigger({ - type: 'harness::turn-completed', - function_id: `${EVENTS_FN}::${host.iii.browserId}`, - config: { session_id: sessionId }, - }) - return () => { - offTrigger() - offHandler() - } - }, [host, sessionId]) - useEffect(() => { if (!open) return const onPointerDown = (event: MouseEvent) => { diff --git a/harness/ui/src/function-trigger-message/index.tsx b/harness/ui/src/function-trigger-message/index.tsx index be475310e..bcca9e363 100644 --- a/harness/ui/src/function-trigger-message/index.tsx +++ b/harness/ui/src/function-trigger-message/index.tsx @@ -18,6 +18,7 @@ import { type SessionUsage, parseMetrics, } from '../lib/metrics' +import { TONE_COLOR, toneFor } from '../lib/tone' const METRICS_ID = 'harness::metrics' @@ -56,12 +57,7 @@ function MiniUsageBar({ total, usable }: { total: number; usable: number }) { if (usable <= 0) return null const ratio = Math.min(1, total / usable) const pct = Math.round(ratio * 100) - const color = - ratio >= 0.9 - ? 'var(--color-alert)' - : ratio >= 0.75 - ? 'var(--color-warn)' - : 'var(--color-accent)' + const color = TONE_COLOR[toneFor(ratio)] return ( = { + ok: 'var(--color-accent)', + warn: 'var(--color-warn)', + alert: 'var(--color-alert)', +} + +/** Tone for a used/usable ratio (already clamped by the caller). */ +export function toneFor(ratio: number): Tone { + if (ratio >= DANGER_THRESHOLD) return 'alert' + if (ratio >= WARN_THRESHOLD) return 'warn' + return 'ok' +} From 6a5988bfb666d81c034a00a25a2aed1cc00c4394 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Tue, 4 Aug 2026 10:36:48 +0100 Subject: [PATCH 08/15] (MOT-4327) fix(harness): review fixes for snapshot accounting edges CodeRabbit findings. The count cache key now includes the provider so two providers serving one model id cannot share entries. A freshly seeded turn no longer hides the session's snapshot from harness::metrics: the record copy is preferred and the durable harness_context row is the fallback. An unchanged request subtracts the unused pre-generate hook reservation from its total instead of reporting reserved headroom as consumed. Scaled by-role buckets absorb the flooring remainder into the largest bucket so categories keep summing to the exact total. The chip keeps whichever snapshot is newest when the hydration read races the streamed trigger, and isSnapshot validates the messages object it later reads. --- harness/src/context_snapshot.rs | 29 +++++++++++++++++++++------ harness/src/functions/metrics.rs | 15 +++++++++++--- harness/src/turn_loop.rs | 5 ++++- harness/ui/src/context-chip/index.tsx | 12 +++++++++-- harness/ui/src/lib/metrics.ts | 18 ++++++++++++----- 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/harness/src/context_snapshot.rs b/harness/src/context_snapshot.rs index f6939f1c4..4b1e5e226 100644 --- a/harness/src/context_snapshot.rs +++ b/harness/src/context_snapshot.rs @@ -39,20 +39,36 @@ impl SnapshotMessagesV1 { } /// The same by-role proportions carried onto a different total - /// (`numerator / denominator`). `denominator == 0` has no proportions to - /// carry, so it yields all-zero. + /// (`numerator / denominator`), summing to exactly `numerator`: the + /// largest bucket absorbs the flooring remainder so category totals + /// keep reconciling with the exact request total. `denominator == 0` + /// has no proportions to carry, so it yields all-zero. fn scaled(&self, numerator: u64, denominator: u64) -> Self { if denominator == 0 { return Self::default(); } let scale = numerator as f64 / denominator as f64; let apply = |tokens: u64| (tokens as f64 * scale) as u64; - Self { + let mut scaled = Self { user: apply(self.user), assistant: apply(self.assistant), function_result: apply(self.function_result), custom: apply(self.custom), + }; + let shortfall = numerator.saturating_sub(scaled.total()); + let largest = [ + (&mut scaled.user, self.user), + (&mut scaled.assistant, self.assistant), + (&mut scaled.function_result, self.function_result), + (&mut scaled.custom, self.custom), + ] + .into_iter() + .max_by_key(|(_, original)| *original) + .map(|(bucket, _)| bucket); + if let Some(bucket) = largest { + *bucket += shortfall; } + scaled } } @@ -162,9 +178,10 @@ fn count_cache() -> &'static Mutex> { CACHE.get_or_init(|| Mutex::new(HashMap::new())) } -fn cache_key(model: &str, kind: &str, content: &str) -> u64 { +fn cache_key(model: &str, provider: Option<&str>, kind: &str, content: &str) -> u64 { let mut hasher = DefaultHasher::new(); model.hash(&mut hasher); + provider.hash(&mut hasher); kind.hash(&mut hasher); content.hash(&mut hasher); hasher.finish() @@ -228,7 +245,7 @@ struct Counted { /// Tokens the probe alone costs, so every part's delta subtracts the same /// base. Counted once per (model, provider) and cached for the process. async fn probe_base(router: &RouterClient, model: &str, provider: Option<&str>) -> Option { - let key = cache_key(model, "base", ""); + let key = cache_key(model, provider, "base", ""); if let Some(tokens) = cached(key) { return Some(tokens); } @@ -249,7 +266,7 @@ async fn counted_delta( base: u64, part: Part<'_>, ) -> Option { - let key = cache_key(model, part.kind(), &part.content_key()); + let key = cache_key(model, provider, part.kind(), &part.content_key()); if let Some(tokens) = cached(key) { return Some(Counted { tokens, diff --git a/harness/src/functions/metrics.rs b/harness/src/functions/metrics.rs index 098506550..19e088f46 100644 --- a/harness/src/functions/metrics.rs +++ b/harness/src/functions/metrics.rs @@ -159,9 +159,18 @@ pub async fn handle( total.observe(message); } } - // The turn record already carries the session's latest snapshot; do - // not spend a second state round trip per session re-reading it. - by_session.push(current.finish_session(node, turn.context_snapshot.clone())); + // The turn record usually carries the session's latest snapshot for + // free; a freshly seeded turn has not generated yet, so fall back to + // the durable `harness_context` row (which the previous turn wrote). + let context = match turn.context_snapshot.clone() { + Some(snapshot) => Some(snapshot), + None => { + crate::context_snapshot::get(&deps.iii, &node.session_id, cfg.session_timeout_ms) + .await + .unwrap_or(None) + } + }; + by_session.push(current.finish_session(node, context)); } // Partial snapshots are polled as progress signals. Trace aggregation is // comparatively expensive and does not help the watchdog decide whether diff --git a/harness/src/turn_loop.rs b/harness/src/turn_loop.rs index dbc76de0a..123ec9fe6 100644 --- a/harness/src/turn_loop.rs +++ b/harness/src/turn_loop.rs @@ -507,7 +507,10 @@ pub async fn run_step( &assembled.system_prompt, ); let final_request_tokens = if request_unchanged { - assembled.token_count + // `assembled.token_count` was fit against the inflated overhead + // (base + reservation); the reservation went unused on an + // unchanged request, so it is not part of the real total. + assembled.token_count.saturating_sub(extra_overhead_tokens) } else { let final_count = deps .context() diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index db996a285..d73e1aab3 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -276,6 +276,14 @@ export function createContextChip(host: Host) { const [open, setOpen] = useState(false) const rootRef = useRef(null) + // Both the hydration read and the streamed trigger write this state; + // keep whichever snapshot is newest so a slow state::get can never + // overwrite a fresher streamed step. + const acceptNewer = (value: ContextSnapshot) => + setSnapshot((current) => + current && current.timestamp >= value.timestamp ? current : value, + ) + useEffect(() => { let cancelled = false setSnapshot(null) @@ -285,7 +293,7 @@ export function createContextChip(host: Host) { .then((value) => { if (cancelled) return if (isSnapshot(value) && value.session_id === sessionId) - setSnapshot(value) + acceptNewer(value) }) .catch(() => {}) return () => { @@ -301,7 +309,7 @@ export function createContextChip(host: Host) { if (!event || event.key !== sessionId) return if (event.event_type === 'state:deleted') return if (isSnapshot(event.new_value) && event.new_value.session_id === sessionId) - setSnapshot(event.new_value) + acceptNewer(event.new_value) }) const offTrigger = host.iii.registerTrigger({ type: 'state', diff --git a/harness/ui/src/lib/metrics.ts b/harness/ui/src/lib/metrics.ts index 8a31af131..bc75ba0cf 100644 --- a/harness/ui/src/lib/metrics.ts +++ b/harness/ui/src/lib/metrics.ts @@ -107,11 +107,19 @@ export function parseMetrics(value: unknown): MetricsResponse | null { export function isSnapshot(value: unknown): value is ContextSnapshot { if (!value || typeof value !== 'object' || Array.isArray(value)) return false const snap = value as Record + if (typeof snap.total !== 'number' || typeof snap.usable !== 'number') + return false + const categories = snap.categories as Record | undefined + if (!categories || typeof categories !== 'object' || Array.isArray(categories)) + return false + const messages = categories.messages as Record | undefined return ( - typeof snap.total === 'number' && - typeof snap.usable === 'number' && - !!snap.categories && - typeof snap.categories === 'object' && - !Array.isArray(snap.categories) + !!messages && + typeof messages === 'object' && + !Array.isArray(messages) && + typeof messages.user === 'number' && + typeof messages.assistant === 'number' && + typeof messages.function_result === 'number' && + typeof messages.custom === 'number' ) } From 3b0dcd56f7c52b5175df2dce429378ac3c92edc6 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Tue, 4 Aug 2026 11:51:50 +0100 Subject: [PATCH 09/15] (MOT-4327) fix(eval): follow harness onto iii-sdk 0.21.8 eval path-depends on harness, whose exact 0.21.8 pins can never co-resolve with eval's 0.21.6 ones in a single graph; the eval CI lane trips on it for any harness-touching change. Pins aligned, the SendOptions literal gains the max_validation_retries field harness grew, and the eval goldens pick up the context snapshot schema that now flows through the embedded harness metrics response. --- eval/Cargo.lock | 11 +- eval/Cargo.toml | 4 +- eval/src/runtime.rs | 1 + .../golden/schemas/eval.assert.exact.json | 232 ++++++++++++++++++ .../schemas/eval.assert.normalized_text.json | 232 ++++++++++++++++++ eval/tests/golden/schemas/eval.result.json | 232 ++++++++++++++++++ 6 files changed, 705 insertions(+), 7 deletions(-) diff --git a/eval/Cargo.lock b/eval/Cargo.lock index cd8a235de..4be2ec8f8 100644 --- a/eval/Cargo.lock +++ b/eval/Cargo.lock @@ -532,12 +532,13 @@ dependencies = [ [[package]] name = "harness" -version = "1.6.1" +version = "1.6.7" dependencies = [ "anyhow", "async-trait", "clap", "globset", + "iii-console-ui", "iii-helpers", "iii-sdk", "jsonschema", @@ -790,9 +791,9 @@ dependencies = [ [[package]] name = "iii-helpers" -version = "0.21.6" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d84d5c149ae4404365a79feca28aa66f6a7dbed56423b4b8c4e2421e0b5add" +checksum = "84bdc7bbc3abfde934a62cdc5d3045adf52914dfc1ed6c20f8af691fc561dc55" dependencies = [ "futures-util", "opentelemetry", @@ -811,9 +812,9 @@ dependencies = [ [[package]] name = "iii-sdk" -version = "0.21.6" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dd060fddcc9153b0dd07c038a14cf172ce15ce1d4edb98155563ed55b2caba" +checksum = "4dd563a1d2f55f893d9a433b747f0bf9bc426656413b136b6ed3a699f3c757b2" dependencies = [ "async-trait", "futures-util", diff --git a/eval/Cargo.toml b/eval/Cargo.toml index 34505fcf3..388ecf0fb 100644 --- a/eval/Cargo.toml +++ b/eval/Cargo.toml @@ -16,8 +16,8 @@ path = "src/lib.rs" [dependencies] harness = { path = "../harness" } -iii-sdk = "=0.21.6" -iii-helpers = "=0.21.6" +iii-sdk = "=0.21.8" +iii-helpers = "=0.21.8" iii-console-ui = { path = "../crates/console-ui" } tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] } serde = { version = "1", features = ["derive"] } diff --git a/eval/src/runtime.rs b/eval/src/runtime.rs index cca0cdd4a..7be7d4490 100644 --- a/eval/src/runtime.rs +++ b/eval/src/runtime.rs @@ -693,6 +693,7 @@ fn send_options(job: &EvalJobRecordV1, variant: &crate::contract::EvalVariantV1) output: Some(job.request.output.clone()), functions: Some(job.request.functions.clone()), metadata: job.request.metadata.clone(), + max_validation_retries: None, } } diff --git a/eval/tests/golden/schemas/eval.assert.exact.json b/eval/tests/golden/schemas/eval.assert.exact.json index 9e3e0a983..8e583d473 100644 --- a/eval/tests/golden/schemas/eval.assert.exact.json +++ b/eval/tests/golden/schemas/eval.assert.exact.json @@ -3,6 +3,104 @@ "request_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "ContextSnapshotV1": { + "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "properties": { + "categories": { + "$ref": "#/definitions/SnapshotCategoriesV1" + }, + "compacted": { + "type": "boolean" + }, + "effective_max_output_tokens": { + "description": "Output allocation `usable` was derived against.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "estimator": { + "description": "Which estimator produced the numbers (`heuristic` until the context-manager resolves a real tokenizer). Absent when the context-manager predates the breakdown response.", + "type": [ + "string", + "null" + ] + }, + "free": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "model": { + "type": "string" + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "step": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "summarized_head_tokens": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "timestamp": { + "format": "int64", + "type": "integer" + }, + "total": { + "description": "Final request estimate: categories plus post-assembly growth.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "turn_id": { + "type": "string" + }, + "usable": { + "description": "The input budget the window was fit into.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/definitions/Usage" + }, + { + "type": "null" + } + ], + "description": "Actual provider usage for this generation, stamped after the terminal frame; absent when the provider returned none (or the generation never completed)." + } + }, + "required": [ + "categories", + "compacted", + "effective_max_output_tokens", + "free", + "model", + "session_id", + "step", + "timestamp", + "total", + "turn_id", + "usable" + ], + "type": "object" + }, "SessionMetricsResponseV1": { "additionalProperties": false, "properties": { @@ -229,6 +327,17 @@ "null" ] }, + "context": { + "anyOf": [ + { + "$ref": "#/definitions/ContextSnapshotV1" + }, + { + "type": "null" + } + ], + "description": "The session's latest per-generation context snapshot (categories, budget, usage) — absent for sessions that have not generated since snapshots landed." + }, "cost_usd": { "format": "double", "type": [ @@ -299,6 +408,129 @@ ], "type": "object" }, + "SnapshotCategoriesV1": { + "description": "Where the request's tokens sit. Categories are assembly-time estimates; `hook_guidance` is the measured growth after assembly (pre-generate hook appends and orphan-repair patches), 0 when the request left assembly unchanged.", + "properties": { + "hook_guidance": { + "default": 0, + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "messages": { + "$ref": "#/definitions/SnapshotMessagesV1" + }, + "overhead": { + "description": "Provider framing plus response_format / provider_options fields.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "system_prompt": { + "description": "Final assembled system prompt: mode paragraph, identity, per-step aids, and any compaction summary section.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "tools": { + "description": "Function schemas exposed to the model.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "messages", + "overhead", + "system_prompt", + "tools" + ], + "type": "object" + }, + "SnapshotMessagesV1": { + "description": "Estimated tokens of the assembled window's messages, by role.", + "properties": { + "assistant": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "custom": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "function_result": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "user": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "assistant", + "custom", + "function_result", + "user" + ], + "type": "object" + }, + "Usage": { + "properties": { + "cache_read": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "output": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "reasoning": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "VariantRoleV1": { "enum": [ "control", diff --git a/eval/tests/golden/schemas/eval.assert.normalized_text.json b/eval/tests/golden/schemas/eval.assert.normalized_text.json index b8cceef70..450cce370 100644 --- a/eval/tests/golden/schemas/eval.assert.normalized_text.json +++ b/eval/tests/golden/schemas/eval.assert.normalized_text.json @@ -3,6 +3,104 @@ "request_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "ContextSnapshotV1": { + "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "properties": { + "categories": { + "$ref": "#/definitions/SnapshotCategoriesV1" + }, + "compacted": { + "type": "boolean" + }, + "effective_max_output_tokens": { + "description": "Output allocation `usable` was derived against.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "estimator": { + "description": "Which estimator produced the numbers (`heuristic` until the context-manager resolves a real tokenizer). Absent when the context-manager predates the breakdown response.", + "type": [ + "string", + "null" + ] + }, + "free": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "model": { + "type": "string" + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "step": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "summarized_head_tokens": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "timestamp": { + "format": "int64", + "type": "integer" + }, + "total": { + "description": "Final request estimate: categories plus post-assembly growth.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "turn_id": { + "type": "string" + }, + "usable": { + "description": "The input budget the window was fit into.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/definitions/Usage" + }, + { + "type": "null" + } + ], + "description": "Actual provider usage for this generation, stamped after the terminal frame; absent when the provider returned none (or the generation never completed)." + } + }, + "required": [ + "categories", + "compacted", + "effective_max_output_tokens", + "free", + "model", + "session_id", + "step", + "timestamp", + "total", + "turn_id", + "usable" + ], + "type": "object" + }, "SessionMetricsResponseV1": { "additionalProperties": false, "properties": { @@ -229,6 +327,17 @@ "null" ] }, + "context": { + "anyOf": [ + { + "$ref": "#/definitions/ContextSnapshotV1" + }, + { + "type": "null" + } + ], + "description": "The session's latest per-generation context snapshot (categories, budget, usage) — absent for sessions that have not generated since snapshots landed." + }, "cost_usd": { "format": "double", "type": [ @@ -299,6 +408,129 @@ ], "type": "object" }, + "SnapshotCategoriesV1": { + "description": "Where the request's tokens sit. Categories are assembly-time estimates; `hook_guidance` is the measured growth after assembly (pre-generate hook appends and orphan-repair patches), 0 when the request left assembly unchanged.", + "properties": { + "hook_guidance": { + "default": 0, + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "messages": { + "$ref": "#/definitions/SnapshotMessagesV1" + }, + "overhead": { + "description": "Provider framing plus response_format / provider_options fields.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "system_prompt": { + "description": "Final assembled system prompt: mode paragraph, identity, per-step aids, and any compaction summary section.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "tools": { + "description": "Function schemas exposed to the model.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "messages", + "overhead", + "system_prompt", + "tools" + ], + "type": "object" + }, + "SnapshotMessagesV1": { + "description": "Estimated tokens of the assembled window's messages, by role.", + "properties": { + "assistant": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "custom": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "function_result": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "user": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "assistant", + "custom", + "function_result", + "user" + ], + "type": "object" + }, + "Usage": { + "properties": { + "cache_read": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "output": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "reasoning": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "VariantRoleV1": { "enum": [ "control", diff --git a/eval/tests/golden/schemas/eval.result.json b/eval/tests/golden/schemas/eval.result.json index fd603052e..9bd8f57bb 100644 --- a/eval/tests/golden/schemas/eval.result.json +++ b/eval/tests/golden/schemas/eval.result.json @@ -121,6 +121,104 @@ ], "type": "string" }, + "ContextSnapshotV1": { + "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "properties": { + "categories": { + "$ref": "#/definitions/SnapshotCategoriesV1" + }, + "compacted": { + "type": "boolean" + }, + "effective_max_output_tokens": { + "description": "Output allocation `usable` was derived against.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "estimator": { + "description": "Which estimator produced the numbers (`heuristic` until the context-manager resolves a real tokenizer). Absent when the context-manager predates the breakdown response.", + "type": [ + "string", + "null" + ] + }, + "free": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "model": { + "type": "string" + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "step": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "summarized_head_tokens": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "timestamp": { + "format": "int64", + "type": "integer" + }, + "total": { + "description": "Final request estimate: categories plus post-assembly growth.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "turn_id": { + "type": "string" + }, + "usable": { + "description": "The input budget the window was fit into.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/definitions/Usage" + }, + { + "type": "null" + } + ], + "description": "Actual provider usage for this generation, stamped after the terminal frame; absent when the provider returned none (or the generation never completed)." + } + }, + "required": [ + "categories", + "compacted", + "effective_max_output_tokens", + "free", + "model", + "session_id", + "step", + "timestamp", + "total", + "turn_id", + "usable" + ], + "type": "object" + }, "EvalBenchmarkV1": { "additionalProperties": false, "properties": { @@ -1251,6 +1349,17 @@ "null" ] }, + "context": { + "anyOf": [ + { + "$ref": "#/definitions/ContextSnapshotV1" + }, + { + "type": "null" + } + ], + "description": "The session's latest per-generation context snapshot (categories, budget, usage) — absent for sessions that have not generated since snapshots landed." + }, "cost_usd": { "format": "double", "type": [ @@ -1345,6 +1454,77 @@ ], "type": "object" }, + "SnapshotCategoriesV1": { + "description": "Where the request's tokens sit. Categories are assembly-time estimates; `hook_guidance` is the measured growth after assembly (pre-generate hook appends and orphan-repair patches), 0 when the request left assembly unchanged.", + "properties": { + "hook_guidance": { + "default": 0, + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "messages": { + "$ref": "#/definitions/SnapshotMessagesV1" + }, + "overhead": { + "description": "Provider framing plus response_format / provider_options fields.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "system_prompt": { + "description": "Final assembled system prompt: mode paragraph, identity, per-step aids, and any compaction summary section.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "tools": { + "description": "Function schemas exposed to the model.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "messages", + "overhead", + "system_prompt", + "tools" + ], + "type": "object" + }, + "SnapshotMessagesV1": { + "description": "Estimated tokens of the assembled window's messages, by role.", + "properties": { + "assistant": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "custom": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "function_result": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "user": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "assistant", + "custom", + "function_result", + "user" + ], + "type": "object" + }, "SystemPromptStrategy": { "description": "How a caller-supplied system prompt combines with the built-in identity prompt.", "oneOf": [ @@ -1381,6 +1561,58 @@ ], "type": "string" }, + "Usage": { + "properties": { + "cache_read": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "output": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "reasoning": { + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "VariantAggregateV1": { "additionalProperties": false, "properties": { From 6aeebd821382018613c43fb7d302b4b03d45e8b1 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Tue, 4 Aug 2026 12:42:37 +0100 Subject: [PATCH 10/15] (MOT-4327) fix(harness): teach the lifecycle wire contract the context field The integration probe's LifecycleEventV1 is an exact wire shape and the scenario floor pins the payload key set, so every turn-completed event carrying the new context snapshot was rejected and scenarios timed out waiting for a terminal status they had already received. Both contracts learn the optional context field; the snapshot's consumer-facing shape stays pinned by the harness::metrics schema golden. Full scenario suite green locally against the installed engine. --- harness/tests/integration/src/probe.rs | 1 + harness/tests/integration/src/scenario/floor.rs | 1 + harness/tests/integration/src/types/probe.rs | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/harness/tests/integration/src/probe.rs b/harness/tests/integration/src/probe.rs index 0f8958874..0a743508e 100644 --- a/harness/tests/integration/src/probe.rs +++ b/harness/tests/integration/src/probe.rs @@ -626,6 +626,7 @@ mod tests { parent: None, parent_session_id: None, reactive_depth: None, + context: None, }, }; let observations = vec![ diff --git a/harness/tests/integration/src/scenario/floor.rs b/harness/tests/integration/src/scenario/floor.rs index f3865f885..7cbb4ff12 100644 --- a/harness/tests/integration/src/scenario/floor.rs +++ b/harness/tests/integration/src/scenario/floor.rs @@ -267,6 +267,7 @@ fn lifecycle_failure(run: &RunEvidence, expected: &FloorExpectations) -> Option< "parent", "parent_session_id", "reactive_depth", + "context", ]); let shape = payloads.iter().all(|payload| { let Some(map) = payload.as_object() else { diff --git a/harness/tests/integration/src/types/probe.rs b/harness/tests/integration/src/types/probe.rs index d17def7ea..daf306864 100644 --- a/harness/tests/integration/src/types/probe.rs +++ b/harness/tests/integration/src/types/probe.rs @@ -39,6 +39,11 @@ pub struct LifecycleEventV1 { pub parent_session_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub reactive_depth: Option, + /// The generation's context snapshot (categories, budget, usage) — the + /// consumer-facing shape is pinned by the `harness::metrics` schema + /// golden, so the lifecycle sink carries it opaquely. + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] From 697007c2f56cff5f08b7fdc05737180713199a97 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Tue, 4 Aug 2026 15:16:53 +0100 Subject: [PATCH 11/15] (MOT-4327) feat(harness): show prompt cache read, write, and hit rate in the context panel The panel summed cached and fresh input into one actual figure, which hid the number that drives cost on a long session. The footer now reports fresh input separately and adds a cache line with tokens read, tokens written, and the cached share of the prompt. The line is absent when the provider reported no cache activity. --- harness/ui/src/context-chip/index.tsx | 41 ++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index d73e1aab3..31c9ffc91 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -12,7 +12,11 @@ import { useEffect, useRef, useState } from 'react' import type { Host } from '@iii-dev/console-ui' import { formatCost, formatTokens } from '../lib/format' -import { type ContextSnapshot, isSnapshot } from '../lib/metrics' +import { + type ContextSnapshot, + type SnapshotUsage, + isSnapshot, +} from '../lib/metrics' import { TONE_COLOR, toneFor } from '../lib/tone' /** Per-tab handler id (host.iii.on namespaces it `::`). */ @@ -111,6 +115,24 @@ function categories(snapshot: ContextSnapshot): Category[] { ] } +/** + * The prompt cache view of the last generation. Providers bill a cache read + * at a fraction of fresh input and a cache write at a premium, so on a long + * session the hit rate drives cost more than the window size does. `null` + * when the provider reported no cache activity at all. + */ +function cacheSummary(usage: SnapshotUsage | undefined) { + const read = usage?.cache_read ?? 0 + const write = usage?.cache_write ?? 0 + if (read === 0 && write === 0) return null + const prompt = (usage?.input ?? 0) + read + write + return { + read, + write, + hitPct: prompt > 0 ? Math.round((read / prompt) * 100) : 0, + } +} + interface LegendEntry { key: string label: string @@ -205,6 +227,7 @@ function ContextPopover({ const usage = snapshot.usage const hasActuals = usage != null && (usage.input != null || usage.cache_read != null) + const cache = cacheSummary(usage) return (
@@ -253,9 +276,19 @@ function ContextPopover({ {hasActuals ? ( - last turn actual{' '} - {formatTokens((usage?.input ?? 0) + (usage?.cache_read ?? 0))} · - output {formatTokens(usage?.output ?? 0)} + last turn {formatTokens(usage?.input ?? 0)} in · output{' '} + {formatTokens(usage?.output ?? 0)} + + ) : null} + {cache ? ( + + cache {formatTokens(cache.read)} read ·{' '} + {formatTokens(cache.write)} write · {cache.hitPct}% hit ) : null} {usage?.cost_usd != null ? ( From 850b11fa4c5564e885e30eaad682a88307e98d09 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Tue, 4 Aug 2026 15:34:28 +0100 Subject: [PATCH 12/15] (MOT-4327) style(harness): context panel footer reads at full weight The provenance, actuals, cache and cost lines sat in the ghost tint used for decoration, which made the numbers hard to read. They are findings, so they move to normal ink, and the cache hit rate carries a verdict colour: green above seventy percent, warn below thirty, plain between. A cold prefix re-bills the whole prompt at the write premium every turn, so it is worth flagging rather than dimming. --- harness/ui/src/context-chip/index.tsx | 11 +++++++++-- harness/ui/styles.css | 17 ++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index 31c9ffc91..320eb3bc6 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -126,10 +126,14 @@ function cacheSummary(usage: SnapshotUsage | undefined) { const write = usage?.cache_write ?? 0 if (read === 0 && write === 0) return null const prompt = (usage?.input ?? 0) + read + write + const hitPct = prompt > 0 ? Math.round((read / prompt) * 100) : 0 return { read, write, - hitPct: prompt > 0 ? Math.round((read / prompt) * 100) : 0, + hitPct, + // A cold or broken prefix means the whole prompt is re-billed at the + // write premium every turn, which is worth flagging rather than dimming. + tone: hitPct >= 70 ? 'ok' : hitPct < 30 ? 'warn' : 'plain', } } @@ -288,7 +292,10 @@ function ContextPopover({ } > cache {formatTokens(cache.read)} read ·{' '} - {formatTokens(cache.write)} write · {cache.hitPct}% hit + {formatTokens(cache.write)} write ·{' '} + + {cache.hitPct}% hit + ) : null} {usage?.cost_usd != null ? ( diff --git a/harness/ui/styles.css b/harness/ui/styles.css index 25b23419b..eaba102d2 100644 --- a/harness/ui/styles.css +++ b/harness/ui/styles.css @@ -162,7 +162,22 @@ padding-top: 8px; border-top: 1px solid var(--color-rule-2); font-size: 11px; - color: var(--color-ink-ghost); + /* Provenance, actuals, cache and cost are findings, not chrome: they stay + at reading weight instead of the ghost tint used for decoration. */ + color: var(--color-ink); +} + +[data-iii-ui="harness"] .harness-ui-cache-hit { + font-variant-numeric: tabular-nums; + color: var(--color-ink); +} + +[data-iii-ui="harness"] .harness-ui-cache-hit[data-tone="ok"] { + color: var(--color-ok); +} + +[data-iii-ui="harness"] .harness-ui-cache-hit[data-tone="warn"] { + color: var(--color-warn); } /* --- function-trigger message (chat / traces card) ------------------- */ From 085fac7769c2b76abd113902566f1d563990f098 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 5 Aug 2026 10:02:00 +0100 Subject: [PATCH 13/15] (MOT-4327) fix(harness): review fixes for snapshot lifetime, provenance and hot-path cost Six findings from review. `count_tokens` inherited the router's generation-sized timeout, which defaults to 320s. It runs inside `exactify`, on the path that finalizes a turn, so one slow counting endpoint could hold `turn-completed` open for as long as a whole generation. It now has its own 10s ceiling and still yields to a smaller configured timeout. The count cache stored a bare token count, so a cache hit reported no estimator and the snapshot fell back to naming the provider. Every step after the first with a stable prompt and tool set therefore mislabelled where its numbers came from. The estimator is cached alongside the count. `context_snapshot::delete` had no caller: `harness::on-session-deleted` purged filesystem grants and budget state but left the session's `harness_context` row behind. It is purged with the rest now. `ContextSnapshotV1` documented `total <= usable` as an invariant, and that sentence ships in the wire schema. Once provider usage lands, `total` is what was billed, which can exceed a budget derived beforehand from an estimate. The doc says that instead, and the golden is regenerated. `harness::metrics` read the durable snapshot inline for every session whose turn record carried none, adding a serial round trip per session to a walk that is polled as a progress signal. Those reads now happen once, for the final complete response. The context chip's state trigger was namespaced per tab but not per session, so two mounted chips registered conflicting engine-side filters under one id and either teardown took the other's stream with it. The id carries the session too. --- harness/src/clients/router.rs | 10 +++- harness/src/context_snapshot.rs | 46 +++++++++++-------- harness/src/functions/metrics.rs | 31 +++++++++---- harness/src/functions/on_session_deleted.rs | 1 + .../tests/golden/schemas/harness.metrics.json | 2 +- harness/ui/src/context-chip/index.tsx | 11 ++++- 6 files changed, 69 insertions(+), 32 deletions(-) diff --git a/harness/src/clients/router.rs b/harness/src/clients/router.rs index 36fab9a55..8f311c1df 100644 --- a/harness/src/clients/router.rs +++ b/harness/src/clients/router.rs @@ -54,6 +54,14 @@ pub struct ChatOutcome { pub error: Option, } +/// Ceiling for `count_tokens`, independent of the generation-sized router +/// timeout. Counting is a tokenizer pass, not a model call, and it runs on the +/// path that finalizes a turn: inheriting a timeout measured in minutes would +/// let one slow counting endpoint hold `turn-completed` open for as long as a +/// whole generation. A count that has not answered by now is not worth the +/// wait — the caller keeps its estimate. +const COUNT_TOKENS_TIMEOUT_MS: u64 = 10_000; + #[derive(Clone)] pub struct RouterClient { iii: Arc, @@ -479,7 +487,7 @@ impl RouterClient { function_id: "router::count_tokens".into(), payload, action: None, - timeout_ms: Some(self.timeout_ms), + timeout_ms: Some(self.timeout_ms.min(COUNT_TOKENS_TIMEOUT_MS)), }) .await .ok()?; diff --git a/harness/src/context_snapshot.rs b/harness/src/context_snapshot.rs index 4b1e5e226..032ae7462 100644 --- a/harness/src/context_snapshot.rs +++ b/harness/src/context_snapshot.rs @@ -101,8 +101,10 @@ pub struct SnapshotCategoriesV1 { pub hook_guidance: u64, } -/// One generation's context accounting. `total <= usable` always holds for -/// a generation that ran; `free = usable - total`. +/// One generation's context accounting. `free = usable - total`, floored at +/// zero: once provider usage lands, `total` is what was billed, which can +/// exceed the `usable` budget the window was fit into — that budget was +/// derived before the generation from an estimate. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ContextSnapshotV1 { pub session_id: String, @@ -161,6 +163,9 @@ pub async fn get( Ok(serde_json::from_value(v).ok()) } +/// Drop a session's snapshot row. Called when the session itself is deleted +/// (`harness::on-session-deleted`), so the state worker does not keep a row +/// for a session nobody can read any more. pub async fn delete( iii: &IIIClient, session_id: &str, @@ -169,12 +174,18 @@ pub async fn delete( crate::state::state_delete(iii, CONTEXT_SCOPE, session_id, timeout_ms).await } +/// A cached count and the estimator that answered it. Caching the estimator +/// is what keeps the provenance honest: without it every step after the first +/// would report its counts as coming from the provider's own default rather +/// than from the tokenizer that actually produced them. +type CachedCount = (u64, Option); + /// Process-lifetime cache of provider token counts keyed by /// (model, kind, content hash) — the probe base, the system prompt and the /// tool schemas are stable across a session's steps, so each is counted over /// the wire once. -fn count_cache() -> &'static Mutex> { - static CACHE: OnceLock>> = OnceLock::new(); +fn count_cache() -> &'static Mutex> { + static CACHE: OnceLock>> = OnceLock::new(); CACHE.get_or_init(|| Mutex::new(HashMap::new())) } @@ -189,13 +200,13 @@ fn cache_key(model: &str, provider: Option<&str>, kind: &str, content: &str) -> /// Cache miss and a poisoned lock are the same thing to the caller: count it /// over the wire again. -fn cached(key: u64) -> Option { - count_cache().lock().ok()?.get(&key).copied() +fn cached(key: u64) -> Option { + count_cache().lock().ok()?.get(&key).cloned() } -fn cache(key: u64, tokens: u64) { +fn cache(key: u64, tokens: u64, estimator: Option) { if let Ok(mut cache) = count_cache().lock() { - cache.insert(key, tokens); + cache.insert(key, (tokens, estimator)); } } @@ -237,8 +248,8 @@ impl Part<'_> { #[derive(Debug, Default)] struct Counted { tokens: u64, - /// Which estimator answered, when this count went over the wire. `None` - /// for a part that was absent or served from cache. + /// Which estimator answered this count, over the wire or from the cache. + /// `None` only for a part the request did not carry. estimator: Option, } @@ -246,13 +257,15 @@ struct Counted { /// base. Counted once per (model, provider) and cached for the process. async fn probe_base(router: &RouterClient, model: &str, provider: Option<&str>) -> Option { let key = cache_key(model, provider, "base", ""); - if let Some(tokens) = cached(key) { + if let Some((tokens, _)) = cached(key) { return Some(tokens); } + // The base is subtracted away from every part, so which estimator counted + // it never reaches the snapshot; only the parts' estimators do. let (base, _) = router .count_tokens(model, provider, None, None, &probe_messages()) .await?; - cache(key, base); + cache(key, base, None); Some(base) } @@ -267,11 +280,8 @@ async fn counted_delta( part: Part<'_>, ) -> Option { let key = cache_key(model, provider, part.kind(), &part.content_key()); - if let Some(tokens) = cached(key) { - return Some(Counted { - tokens, - estimator: None, - }); + if let Some((tokens, estimator)) = cached(key) { + return Some(Counted { tokens, estimator }); } let (system_prompt, tools) = match part { Part::System(prompt) => (Some(prompt), None), @@ -281,7 +291,7 @@ async fn counted_delta( .count_tokens(model, provider, system_prompt, tools, &probe_messages()) .await?; let tokens = with_part.saturating_sub(base); - cache(key, tokens); + cache(key, tokens, Some(estimator.clone())); Some(Counted { tokens, estimator: Some(estimator), diff --git a/harness/src/functions/metrics.rs b/harness/src/functions/metrics.rs index 19e088f46..6fae6b611 100644 --- a/harness/src/functions/metrics.rs +++ b/harness/src/functions/metrics.rs @@ -137,6 +137,7 @@ pub async fn handle( let mut complete = tree.complete; let mut total = UsageAccumulator::default(); let mut by_session = Vec::with_capacity(tree.sessions.len()); + let mut missing_context = Vec::new(); for node in &tree.sessions { if !session.exists(&node.session_id).await? { complete = false; @@ -160,18 +161,28 @@ pub async fn handle( } } // The turn record usually carries the session's latest snapshot for - // free; a freshly seeded turn has not generated yet, so fall back to - // the durable `harness_context` row (which the previous turn wrote). - let context = match turn.context_snapshot.clone() { - Some(snapshot) => Some(snapshot), - None => { - crate::context_snapshot::get(&deps.iii, &node.session_id, cfg.session_timeout_ms) - .await - .unwrap_or(None) - } - }; + // free. A freshly seeded turn has not generated yet, so its record has + // none and the durable `harness_context` row is the fallback — noted + // here, read after the walk. + let context = turn.context_snapshot.clone(); + if context.is_none() { + missing_context.push(by_session.len()); + } by_session.push(current.finish_session(node, context)); } + // One state read per session whose record carried no snapshot — collected + // for the final response only, so a large tree does not pay a round trip + // per session on every poll (see the note below on partial responses). + if complete { + for index in missing_context { + let session_id = by_session[index].session_id.clone(); + if let Ok(snapshot) = + crate::context_snapshot::get(&deps.iii, &session_id, cfg.session_timeout_ms).await + { + by_session[index].context = snapshot; + } + } + } // Partial snapshots are polled as progress signals. Trace aggregation is // comparatively expensive and does not help the watchdog decide whether // work advanced, so collect it only for the final complete response. diff --git a/harness/src/functions/on_session_deleted.rs b/harness/src/functions/on_session_deleted.rs index 161665c29..4fbf9b7c9 100644 --- a/harness/src/functions/on_session_deleted.rs +++ b/harness/src/functions/on_session_deleted.rs @@ -40,6 +40,7 @@ pub async fn handle( let cfg = deps.cfg().await; crate::filesystem_grants::purge(&deps.iii, &event.session_id, cfg.session_timeout_ms).await?; crate::budget::purge(deps, &event.session_id, cfg.session_timeout_ms).await?; + crate::context_snapshot::delete(&deps.iii, &event.session_id, cfg.session_timeout_ms).await?; Ok(SessionDeletedAck { ok: true, removed: swept, diff --git a/harness/tests/golden/schemas/harness.metrics.json b/harness/tests/golden/schemas/harness.metrics.json index 397119829..2d0ab07e0 100644 --- a/harness/tests/golden/schemas/harness.metrics.json +++ b/harness/tests/golden/schemas/harness.metrics.json @@ -18,7 +18,7 @@ "additionalProperties": false, "definitions": { "ContextSnapshotV1": { - "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "description": "One generation's context accounting. `free = usable - total`, floored at zero: once provider usage lands, `total` is what was billed, which can exceed the `usable` budget the window was fit into — that budget was derived before the generation from an estimate.", "properties": { "categories": { "$ref": "#/definitions/SnapshotCategoriesV1" diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index 320eb3bc6..d1840593c 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -345,7 +345,14 @@ export function createContextChip(host: Host) { // `state` trigger streams each write (engine-side scope/key filter), so // long multi-step turns tick live without any polling. useEffect(() => { - const offHandler = host.iii.on(STATE_FN, (event) => { + // The id carries the session, because the engine-side filter is keyed to + // one: two chips mounted at once (two sessions visible, or an unmount + // racing the next mount) would otherwise register conflicting filters + // under one id, and either teardown would take the other's stream with + // it. `on()` appends `::` itself, so the trigger repeats it + // to address the same handler. + const eventFn = `${STATE_FN}::${sessionId}` + const offHandler = host.iii.on(eventFn, (event) => { if (!event || event.key !== sessionId) return if (event.event_type === 'state:deleted') return if (isSnapshot(event.new_value) && event.new_value.session_id === sessionId) @@ -353,7 +360,7 @@ export function createContextChip(host: Host) { }) const offTrigger = host.iii.registerTrigger({ type: 'state', - function_id: `${STATE_FN}::${host.iii.browserId}`, + function_id: `${eventFn}::${host.iii.browserId}`, config: { scope: 'harness_context', key: sessionId }, }) return () => { From f19141fe7f6ca9277850dcd39bf8c85fa03aa879 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 5 Aug 2026 10:24:16 +0100 Subject: [PATCH 14/15] (MOT-4327) fix(eval): refresh goldens for the snapshot doc change `eval` depends on `harness` by path, so the `ContextSnapshotV1` doc rewrite in 085fac77 landed in eval's schema descriptions too. --- eval/tests/golden/schemas/eval.assert.exact.json | 2 +- eval/tests/golden/schemas/eval.assert.normalized_text.json | 2 +- eval/tests/golden/schemas/eval.result.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eval/tests/golden/schemas/eval.assert.exact.json b/eval/tests/golden/schemas/eval.assert.exact.json index 8e583d473..42071cae9 100644 --- a/eval/tests/golden/schemas/eval.assert.exact.json +++ b/eval/tests/golden/schemas/eval.assert.exact.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { "ContextSnapshotV1": { - "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "description": "One generation's context accounting. `free = usable - total`, floored at zero: once provider usage lands, `total` is what was billed, which can exceed the `usable` budget the window was fit into — that budget was derived before the generation from an estimate.", "properties": { "categories": { "$ref": "#/definitions/SnapshotCategoriesV1" diff --git a/eval/tests/golden/schemas/eval.assert.normalized_text.json b/eval/tests/golden/schemas/eval.assert.normalized_text.json index 450cce370..d5c8744df 100644 --- a/eval/tests/golden/schemas/eval.assert.normalized_text.json +++ b/eval/tests/golden/schemas/eval.assert.normalized_text.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { "ContextSnapshotV1": { - "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "description": "One generation's context accounting. `free = usable - total`, floored at zero: once provider usage lands, `total` is what was billed, which can exceed the `usable` budget the window was fit into — that budget was derived before the generation from an estimate.", "properties": { "categories": { "$ref": "#/definitions/SnapshotCategoriesV1" diff --git a/eval/tests/golden/schemas/eval.result.json b/eval/tests/golden/schemas/eval.result.json index 9bd8f57bb..814c8abbf 100644 --- a/eval/tests/golden/schemas/eval.result.json +++ b/eval/tests/golden/schemas/eval.result.json @@ -122,7 +122,7 @@ "type": "string" }, "ContextSnapshotV1": { - "description": "One generation's context accounting. `total <= usable` always holds for a generation that ran; `free = usable - total`.", + "description": "One generation's context accounting. `free = usable - total`, floored at zero: once provider usage lands, `total` is what was billed, which can exceed the `usable` budget the window was fit into — that budget was derived before the generation from an estimate.", "properties": { "categories": { "$ref": "#/definitions/SnapshotCategoriesV1" From 47fad904614093cf4b2147ff852a7e958bac09f3 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 5 Aug 2026 11:43:58 +0100 Subject: [PATCH 15/15] (MOT-4327) fix(harness): the chip's actuals row is one step, not the turn A snapshot is written after every generate step and carries that step's usage, so on a multi-step turn the row moved while the turn was still running: cost fell from one step to the next as the prompt cache warmed and more of the window was billed as cache reads. The numbers were right and the word above them was wrong. --- harness/ui/src/context-chip/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harness/ui/src/context-chip/index.tsx b/harness/ui/src/context-chip/index.tsx index d1840593c..36d9aeffe 100644 --- a/harness/ui/src/context-chip/index.tsx +++ b/harness/ui/src/context-chip/index.tsx @@ -280,7 +280,7 @@ function ContextPopover({ {hasActuals ? ( - last turn {formatTokens(usage?.input ?? 0)} in · output{' '} + last step {formatTokens(usage?.input ?? 0)} in · output{' '} {formatTokens(usage?.output ?? 0)} ) : null}