diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b99cd73..6412501 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,59 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Set up pinned Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 - name: Build and validate plugins run: make test + js-daemon-client: + name: JavaScript daemon client (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Set up pinned Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Test RPC transport contracts + run: bun test src/runtime/js-daemon-client/tests + + opencode-compatibility: + name: OpenCode package (${{ matrix.peer-version }}) + strategy: + fail-fast: false + matrix: + peer-version: ["1.0.0", "latest"] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Set up pinned Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Build validated tarball + run: make validate-opencode + - name: Install package with OpenCode peers + env: + PEER_VERSION: ${{ matrix.peer-version }} + run: | + set -euo pipefail + mkdir install-test + npm install --prefix install-test --no-audit --no-fund \ + ./dist/opencode/braintrust-trace-opencode-*.tgz \ + "@opencode-ai/plugin@${PEER_VERSION}" \ + "@opencode-ai/sdk@${PEER_VERSION}" + node -e 'import(process.argv[1]).then((m) => { if (typeof m.default !== "function") process.exit(1) })' \ + "$PWD/install-test/node_modules/@braintrust/trace-opencode/dist/index.js" + daemon: name: Daemon (${{ matrix.os }}) strategy: @@ -48,11 +98,18 @@ jobs: rustup default stable rustup component add clippy rustfmt - name: Install latest coding agents - run: npm install --prefix "${{ runner.temp }}/coding-agents" --no-save --no-package-lock --no-audit --no-fund --cache "${{ runner.temp }}/npm-cache" @openai/codex@latest @anthropic-ai/claude-code@latest + run: npm install --prefix "${{ runner.temp }}/coding-agents" --no-save --no-package-lock --no-audit --no-fund --cache "${{ runner.temp }}/npm-cache" @openai/codex@latest @anthropic-ai/claude-code@latest opencode-ai@latest + - name: Set up pinned Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Build OpenCode plugin for integration harness + run: make build-opencode - name: Report coding-agent versions run: | npm exec --prefix "${{ runner.temp }}/coding-agents" -- codex --version npm exec --prefix "${{ runner.temp }}/coding-agents" -- claude --version + npm exec --prefix "${{ runner.temp }}/coding-agents" -- opencode --version - name: Check formatting if: runner.os == 'Linux' run: cargo fmt --manifest-path bt-daemon/Cargo.toml -- --check @@ -66,6 +123,7 @@ jobs: BT_AGENT_INGEST_MODE: mock CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex${{ matrix.agent_suffix }} CLAUDE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/claude${{ matrix.agent_suffix }} + OPENCODE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/opencode${{ matrix.agent_suffix }} run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_integration -- --ignored --nocapture --test-threads=1 - name: Lint daemon run: cargo clippy --manifest-path bt-daemon/Cargo.toml --all-targets --all-features --locked -- -D warnings diff --git a/Makefile b/Makefile index 0c899f0..db0c13b 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,9 @@ DIST := dist # name would shadow it in recipe subshells. BUILD_RULES := $(addprefix build-,$(PLUGINS)) PUBLISH_RULES := $(addprefix publish-,$(PLUGINS)) +VALIDATE_RULES := $(addprefix validate-,$(PLUGINS)) -.PHONY: build test publish clean $(BUILD_RULES) $(PUBLISH_RULES) +.PHONY: build test publish clean $(BUILD_RULES) $(VALIDATE_RULES) $(PUBLISH_RULES) build: $(BUILD_RULES) @@ -38,6 +39,10 @@ test: build src/plugins/$$p/validate.sh "$(DIST)/$$p"; \ done +$(VALIDATE_RULES): validate-%: build-% + @echo "==> validate $*" + @src/plugins/$*/validate.sh "$(DIST)/$*" + # Deploy every plugin named in the PUBLISH_TARGETS env var map. Fails if unset. publish: @scripts/publish.sh diff --git a/README.md b/README.md index 41976ec..0ac7496 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ For further instructions, see the instructions for your desired coding agent |-------------|-------------------------| | Claude Code | [braintrustdata/braintrust-claude-plugin](https://github.com/braintrustdata/braintrust-claude-plugin) | | Codex | [braintrustdata/braintrust-codex-plugin](https://github.com/braintrustdata/braintrust-codex-plugin) | +| OpenCode | npm: [`@braintrust/trace-opencode`](https://www.npmjs.com/package/@braintrust/trace-opencode) | ## Development & releasing diff --git a/bt-daemon/src/dispatch.rs b/bt-daemon/src/dispatch.rs index 8b4a5ec..f759738 100644 --- a/bt-daemon/src/dispatch.rs +++ b/bt-daemon/src/dispatch.rs @@ -44,6 +44,7 @@ impl Session { pub fn spawn( session_id: String, source: String, + plugin_version: Option, journal: JournalWriter, replay: Vec, translators: Arc, @@ -57,6 +58,7 @@ impl Session { let actor = SessionActor { session_id: session_id.clone(), source: source.clone(), + plugin_version, translators, sink_factory, counters: counters.clone(), @@ -164,6 +166,7 @@ async fn hydrate_transcript_snapshot(env: &mut Envelope) { struct SessionActor { session_id: String, source: String, + plugin_version: Option, translators: Arc, sink_factory: Arc, counters: Arc, @@ -175,7 +178,11 @@ struct SessionActor { impl SessionActor { async fn run(self, mut rx: mpsc::UnboundedReceiver) { let mut translator = self.translators.create(&self.source, &self.session_id); - let mut sink = match self.sink_factory.create(&self.session_id, &self.source) { + let mut sink = match self.sink_factory.create( + &self.session_id, + &self.source, + self.plugin_version.as_deref(), + ) { Ok(s) => s, Err(e) => { self.set_error(format!("sink init failed: {e}")); diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index c246509..083d628 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -120,6 +120,7 @@ pub fn envelope_from_redacted(r: RedactedEnvelope) -> Envelope { Envelope { source: r.source, source_version: r.source_version, + plugin_version: r.plugin_version, session_id: r.session_id, event: r.event, ts_ms: r.ts_ms, diff --git a/bt-daemon/src/lib.rs b/bt-daemon/src/lib.rs index 75b755a..907607b 100644 --- a/bt-daemon/src/lib.rs +++ b/bt-daemon/src/lib.rs @@ -214,6 +214,7 @@ pub async fn run_hook( let env = Envelope { source: args.source.clone(), source_version: args.source_version.clone(), + plugin_version: None, session_id, event, ts_ms: now_ms(), @@ -612,7 +613,11 @@ impl ImportProcessor { Some(live) => live, None => { let translator = self.opts.translators.create(&env.source, &sid); - let sink = self.opts.sink_factory.create(&sid, &env.source)?; + let sink = self.opts.sink_factory.create( + &sid, + &env.source, + env.plugin_version.as_deref(), + )?; self.sessions.insert( sid.clone(), ImportLive { diff --git a/bt-daemon/src/server.rs b/bt-daemon/src/server.rs index df6c879..ad47be2 100644 --- a/bt-daemon/src/server.rs +++ b/bt-daemon/src/server.rs @@ -256,6 +256,7 @@ impl Daemon { let session = Session::spawn( env.session_id.clone(), env.source.clone(), + env.plugin_version.clone(), journal, replay, self.translators.clone(), @@ -333,6 +334,14 @@ pub async fn run(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<()> { async fn accept_loop(daemon: Arc, mut listener: Listener) -> anyhow::Result<()> { loop { + // `Notify::notify_waiters` does not retain a permit. If accepting a + // connection wins the select at the same time shutdown is requested, + // check the sticky flag before waiting again so the notification + // cannot be lost. + if daemon.shutting_down.load(Ordering::SeqCst) { + tracing::info!("shutdown requested"); + return Ok(()); + } tokio::select! { _ = daemon.shutdown.notified() => { tracing::info!("shutdown requested"); diff --git a/bt-daemon/src/sink/braintrust.rs b/bt-daemon/src/sink/braintrust.rs index f660e31..97a0a38 100644 --- a/bt-daemon/src/sink/braintrust.rs +++ b/bt-daemon/src/sink/braintrust.rs @@ -91,12 +91,17 @@ impl BraintrustSinkFactory { } impl SinkFactory for BraintrustSinkFactory { - fn create(&self, _session_id: &str, source: &str) -> anyhow::Result> { + fn create( + &self, + _session_id: &str, + source: &str, + plugin_version: Option<&str>, + ) -> anyhow::Result> { Ok(Box::new(BraintrustSink { cache: self.cache.clone(), default_api_url: self.default_api_url.clone(), default_app_url: self.default_app_url.clone(), - version: self.version.clone(), + version: plugin_version.unwrap_or(&self.version).to_string(), source: source.to_string(), creds: None, urls: None, diff --git a/bt-daemon/src/sink/debug.rs b/bt-daemon/src/sink/debug.rs index f61c534..f298047 100644 --- a/bt-daemon/src/sink/debug.rs +++ b/bt-daemon/src/sink/debug.rs @@ -13,7 +13,12 @@ pub struct DebugSinkFactory { } impl SinkFactory for DebugSinkFactory { - fn create(&self, session_id: &str, _source: &str) -> anyhow::Result> { + fn create( + &self, + session_id: &str, + _source: &str, + _plugin_version: Option<&str>, + ) -> anyhow::Result> { std::fs::create_dir_all(&self.dir)?; let path = self.dir.join(format!("{}.ndjson", sanitize(session_id))); let file = OpenOptions::new().create(true).append(true).open(&path)?; diff --git a/bt-daemon/src/sink/mod.rs b/bt-daemon/src/sink/mod.rs index f6b9a32..2338040 100644 --- a/bt-daemon/src/sink/mod.rs +++ b/bt-daemon/src/sink/mod.rs @@ -36,8 +36,14 @@ pub trait Sink: Send { } } -/// Builds a sink per session. `source` is the agent id (e.g. `codex`), used by -/// the Braintrust sink to stamp `context.span_origin`. +/// Builds a sink per session. `source` and `plugin_version` identify the +/// instrumentation that captured the events and are used to stamp +/// `context.span_origin` centrally. pub trait SinkFactory: Send + Sync { - fn create(&self, session_id: &str, source: &str) -> anyhow::Result>; + fn create( + &self, + session_id: &str, + source: &str, + plugin_version: Option<&str>, + ) -> anyhow::Result>; } diff --git a/bt-daemon/src/transcript_import.rs b/bt-daemon/src/transcript_import.rs index a8e2a24..432f3e1 100644 --- a/bt-daemon/src/transcript_import.rs +++ b/bt-daemon/src/transcript_import.rs @@ -517,6 +517,7 @@ fn envelope( Envelope { source: source.into(), source_version, + plugin_version: None, session_id: session_id.into(), event: event.into(), ts_ms, diff --git a/bt-daemon/src/translate/mod.rs b/bt-daemon/src/translate/mod.rs index 5b55567..92d978e 100644 --- a/bt-daemon/src/translate/mod.rs +++ b/bt-daemon/src/translate/mod.rs @@ -11,10 +11,12 @@ mod claude; mod codex; mod debug; mod git; +mod opencode; pub use claude::ClaudeTranslatorFactory; pub use codex::CodexTranslatorFactory; pub use debug::DebugTranslatorFactory; +pub use opencode::OpenCodeTranslatorFactory; use crate::wire::{Envelope, SessionConfig}; use serde::{Deserialize, Serialize}; @@ -122,7 +124,8 @@ impl Registry { let mut r = Registry::debug_only(); let git = Arc::new(git::GitMetadataCache::default()); r.register(Box::new(ClaudeTranslatorFactory::new(git.clone()))); - r.register(Box::new(CodexTranslatorFactory::new(git))); + r.register(Box::new(CodexTranslatorFactory::new(git.clone()))); + r.register(Box::new(OpenCodeTranslatorFactory::new(git))); r } diff --git a/bt-daemon/src/translate/opencode.rs b/bt-daemon/src/translate/opencode.rs new file mode 100644 index 0000000..62b2b2e --- /dev/null +++ b/bt-daemon/src/translate/opencode.rs @@ -0,0 +1,665 @@ +//! OpenCode translator. The JavaScript package forwards native hook payloads; +//! this module owns span construction, correlation, and recovery. + +use super::git::GitMetadataCache; +use super::{AgentTranslator, SessionCtx, SpanOp, SpanRow, SpanType, TranslatorFactory}; +use crate::ids; +use crate::wire::Envelope; +use serde_json::{json, Map, Value}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +pub struct OpenCodeTranslatorFactory { + git: Arc, +} + +impl OpenCodeTranslatorFactory { + pub(super) fn new(git: Arc) -> Self { + Self { git } + } +} + +impl TranslatorFactory for OpenCodeTranslatorFactory { + fn source(&self) -> &str { + "opencode" + } + + fn create(&self, session_id: &str) -> Box { + Box::new(OpenCodeTranslator { + daemon_session_id: session_id.to_string(), + sessions: HashMap::new(), + git: self.git.clone(), + last_ts_ms: 0, + }) + } +} + +#[derive(Default)] +struct NativeSession { + root_span_id: String, + effective_root_span_id: String, + parent_session_id: Option, + current_turn_span_id: Option, + turn_number: u32, + tool_call_count: u32, + current_input: Option, + current_output: Option, + system_prompt: Option, + output_parts: HashMap, + reasoning_parts: HashMap, + tool_calls: HashMap>, + tool_starts: HashMap, + tool_args: HashMap, + tool_outputs: HashMap, + tool_errors: HashMap, + tool_message_ids: HashMap, + denied_tools: HashSet, + completed_messages: HashSet, +} + +struct OpenCodeTranslator { + daemon_session_id: String, + sessions: HashMap, + git: Arc, + last_ts_ms: i64, +} + +impl AgentTranslator for OpenCodeTranslator { + fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { + self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); + let mut ops = match event.event.as_str() { + "session.created" => self.session_created(event, ctx), + "chat.message" => self.chat_message(event, ctx), + "experimental.chat.system.transform" => self.system_prompt(event), + "message.part.updated" => self.part_updated(event), + "message.updated" => self.message_updated(event), + "tool.execute.before" => self.tool_before(event, ctx), + "tool.execute.after" => self.tool_after(event), + "permission.asked" | "permission.replied" => self.permission(event), + "session.idle" => self.finish_session_event(event, false, None), + "session.deleted" => self.finish_session_event(event, true, None), + "session.error" => { + let error = format_error( + event + .payload + .pointer("/properties/error") + .or_else(|| event.payload.get("error")), + ); + self.finish_session_event(event, true, Some(error)) + } + _ => Vec::new(), + }; + let cwd = event + .payload + .get("directory") + .and_then(Value::as_str) + .or_else(|| event.payload.get("worktree").and_then(Value::as_str)); + self.git.enrich_rows(cwd, &mut ops); + Ok(ops) + } + + fn flush(&mut self, _ctx: &SessionCtx) -> anyhow::Result> { + let now = self.last_ts_ms; + let ids: Vec = self.sessions.keys().cloned().collect(); + let mut ops = Vec::new(); + for id in ids { + ops.extend(self.close(&id, now, true, None)); + } + Ok(ops) + } +} + +impl OpenCodeTranslator { + fn session_created(&mut self, event: &Envelope, ctx: &SessionCtx) -> Vec { + let info = event + .payload + .pointer("/properties/info") + .or_else(|| event.payload.get("info")); + let Some(native_id) = info + .and_then(|v| v.get("id")) + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| native_session_id(&event.payload)) + else { + return Vec::new(); + }; + if self.sessions.contains_key(&native_id) { + return Vec::new(); + } + let parent_id = info + .and_then(|v| v.get("parentID")) + .and_then(Value::as_str) + .map(str::to_owned); + self.open_session(&native_id, parent_id.as_deref(), info, event.ts_ms, ctx) + } + + fn open_session( + &mut self, + native_id: &str, + parent_id: Option<&str>, + info: Option<&Value>, + ts: i64, + ctx: &SessionCtx, + ) -> Vec { + if self.sessions.contains_key(native_id) { + return Vec::new(); + } + let root_span_id = ids::span_id(&self.daemon_session_id, &format!("session:{native_id}")); + let (effective_root_span_id, parent_span_ids, name) = if let Some(parent_id) = parent_id { + if let Some(parent) = self.sessions.get(parent_id) { + let title = info + .and_then(|v| v.get("title")) + .and_then(Value::as_str) + .unwrap_or("Subagent"); + ( + parent.effective_root_span_id.clone(), + parent.current_turn_span_id.clone().into_iter().collect(), + subagent_name(title), + ) + } else { + (root_span_id.clone(), Vec::new(), "Subagent".to_string()) + } + } else { + let external = ctx + .config + .as_ref() + .map(|c| c.attached_span_ids()) + .unwrap_or_default(); + ( + external.1.unwrap_or_else(|| root_span_id.clone()), + external.0.into_iter().collect(), + "OpenCode".to_string(), + ) + }; + let mut metadata = Map::new(); + metadata.insert("session_id".into(), Value::String(native_id.to_string())); + metadata.insert("source".into(), Value::String("opencode".into())); + if let Some(parent) = parent_id { + metadata.insert("parent_session_id".into(), Value::String(parent.into())); + metadata.insert("is_subagent".into(), Value::Bool(true)); + } + if let Some(extra) = ctx + .config + .as_ref() + .and_then(|c| c.additional_metadata.as_ref()) + .and_then(Value::as_object) + { + metadata.extend(extra.clone()); + } + self.sessions.insert( + native_id.to_string(), + NativeSession { + root_span_id: root_span_id.clone(), + effective_root_span_id: effective_root_span_id.clone(), + parent_session_id: parent_id.map(str::to_owned), + ..Default::default() + }, + ); + vec![SpanOp::Insert(SpanRow { + span_id: root_span_id, + root_span_id: effective_root_span_id, + parent_span_ids, + name, + span_type: SpanType::Task, + start_ms: Some(ts), + metadata: Some(Value::Object(metadata)), + ..Default::default() + })] + } + + fn chat_message(&mut self, event: &Envelope, ctx: &SessionCtx) -> Vec { + let Some(sid) = native_session_id(&event.payload) else { + return Vec::new(); + }; + let mut ops = self.open_session(&sid, None, None, event.ts_ms, ctx); + let Some(state) = self.sessions.get_mut(&sid) else { + return ops; + }; + if let Some(turn) = state.current_turn_span_id.take() { + ops.push(SpanOp::Merge(SpanRow { + span_id: turn, + root_span_id: state.effective_root_span_id.clone(), + end_ms: Some(event.ts_ms), + output: state.current_output.take().map(Value::String), + ..Default::default() + })); + } + state.turn_number += 1; + let turn_id = ids::span_id( + &self.daemon_session_id, + &format!("turn:{sid}:{}", state.turn_number), + ); + let output = event.payload.get("output").unwrap_or(&event.payload); + let input = output + .get("parts") + .and_then(Value::as_array) + .map(|parts| { + parts + .iter() + .filter(|p| p.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|p| p.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n") + }) + .unwrap_or_default(); + state.current_input = Some(input.clone()); + state.current_turn_span_id = Some(turn_id.clone()); + let model = event + .payload + .pointer("/input/model/modelID") + .or_else(|| event.payload.pointer("/model/modelID")) + .and_then(Value::as_str); + let skills = explicit_skills(&input); + ops.push(SpanOp::Insert(SpanRow { + span_id: turn_id, + root_span_id: state.effective_root_span_id.clone(), + parent_span_ids: vec![state.root_span_id.clone()], + name: format!("Turn {}", state.turn_number), + span_type: SpanType::Task, + start_ms: Some(event.ts_ms), + input: (!input.is_empty()).then_some(Value::String(input)), + metadata: Some( + json!({"turn_number":state.turn_number,"model":model,"loaded_skill_names":skills}), + ), + ..Default::default() + })); + ops + } + + fn system_prompt(&mut self, event: &Envelope) -> Vec { + if let Some(sid) = native_session_id(&event.payload) { + if let Some(state) = self.sessions.get_mut(&sid) { + state.system_prompt = event + .payload + .pointer("/output/system") + .and_then(Value::as_array) + .map(|p| { + p.iter() + .filter_map(Value::as_str) + .collect::>() + .join("\n\n") + }); + } + } + Vec::new() + } + + fn part_updated(&mut self, event: &Envelope) -> Vec { + let part = event + .payload + .pointer("/properties/part") + .or_else(|| event.payload.get("part")); + let Some(part) = part else { return Vec::new() }; + let Some(sid) = part.get("sessionID").and_then(Value::as_str) else { + return Vec::new(); + }; + let Some(state) = self.sessions.get_mut(sid) else { + return Vec::new(); + }; + let message_id = part.get("messageID").and_then(Value::as_str).unwrap_or(""); + match part.get("type").and_then(Value::as_str) { + Some("text") => { + if let Some(text) = part.get("text").and_then(Value::as_str) { + state.output_parts.insert(message_id.into(), text.into()); + if part.pointer("/time/end").is_some() { + state.current_output = Some(text.into()); + } + } + } + Some("reasoning") => { + if let Some(text) = part.get("text").and_then(Value::as_str) { + state.reasoning_parts.insert(message_id.into(), text.into()); + } + } + Some("tool") => { + let call_id = part.get("callID").and_then(Value::as_str).unwrap_or(""); + let tool = part.get("tool").and_then(Value::as_str).unwrap_or("tool"); + if let Some(input) = part.pointer("/state/input") { + let call = json!({"id":call_id,"type":"function","function":{"name":tool,"arguments":serde_json::to_string(input).unwrap_or_default()}}); + let calls = state.tool_calls.entry(message_id.into()).or_default(); + if let Some(i) = calls + .iter() + .position(|v| v.get("id").and_then(Value::as_str) == Some(call_id)) + { + calls[i] = call + } else { + calls.push(call) + }; + state + .tool_message_ids + .insert(call_id.into(), message_id.into()); + } + match part.pointer("/state/status").and_then(Value::as_str) { + Some("completed") => { + if let Some(v) = part.pointer("/state/output") { + state.tool_outputs.insert(call_id.into(), v.clone()); + } + } + Some("error") => { + state + .tool_errors + .insert(call_id.into(), format_error(part.pointer("/state/error"))); + } + _ => {} + } + } + _ => {} + } + Vec::new() + } + + fn message_updated(&mut self, event: &Envelope) -> Vec { + let info = event + .payload + .pointer("/properties/info") + .or_else(|| event.payload.get("info")); + let Some(info) = info else { return Vec::new() }; + if info.get("role").and_then(Value::as_str) != Some("assistant") + || info.pointer("/time/completed").is_none() + { + return Vec::new(); + } + let Some(sid) = info.get("sessionID").and_then(Value::as_str) else { + return Vec::new(); + }; + let Some(mid) = info.get("id").and_then(Value::as_str) else { + return Vec::new(); + }; + let Some(state) = self.sessions.get_mut(sid) else { + return Vec::new(); + }; + if !state.completed_messages.insert(mid.into()) { + return Vec::new(); + } + let Some(turn) = state.current_turn_span_id.clone() else { + return Vec::new(); + }; + let cache_read = num(info, "/tokens/cache/read"); + let cache_write = num(info, "/tokens/cache/write"); + let prompt = num(info, "/tokens/input") + cache_read + cache_write; + let completion = num(info, "/tokens/output"); + let reasoning = num(info, "/tokens/reasoning"); + let mut assistant = json!({"role":"assistant","content":state.output_parts.get(mid).cloned().unwrap_or_default()}); + if let Some(calls) = state.tool_calls.get(mid) { + assistant["tool_calls"] = Value::Array(calls.clone()) + } + if let Some(reason) = state.reasoning_parts.get(mid) { + assistant["reasoning"] = json!([{"id":"reasoning","content":reason}]) + } + let mut input = Vec::new(); + if let Some(system) = &state.system_prompt { + input.push(json!({"role":"system","content":system})) + } + if let Some(user) = &state.current_input { + input.push(json!({"role":"user","content":user})) + } + let provider = info + .get("providerID") + .and_then(Value::as_str) + .unwrap_or("unknown"); + let model = info + .get("modelID") + .and_then(Value::as_str) + .unwrap_or("unknown"); + vec![SpanOp::Insert(SpanRow { + span_id: ids::span_id(&self.daemon_session_id, &format!("llm:{sid}:{mid}")), + root_span_id: state.effective_root_span_id.clone(), + parent_span_ids: vec![turn], + name: format!("{provider}/{model}"), + span_type: SpanType::Llm, + start_ms: info + .pointer("/time/created") + .and_then(Value::as_i64) + .or(Some(event.ts_ms)), + end_ms: info + .pointer("/time/completed") + .and_then(Value::as_i64) + .or(Some(event.ts_ms)), + input: Some(Value::Array(input)), + output: Some(Value::Array(vec![assistant])), + metadata: Some(json!({"model":model,"provider":provider,"message_id":mid})), + metrics: Some( + json!({"prompt_tokens":prompt,"completion_tokens":completion,"tokens":prompt+completion+reasoning,"prompt_cached_tokens":cache_read,"prompt_cache_creation_tokens":cache_write,"reasoning_tokens":reasoning}), + ), + error: info.get("error").map(|e| format_error(Some(e))), + ..Default::default() + })] + } + + fn tool_before(&mut self, event: &Envelope, _ctx: &SessionCtx) -> Vec { + let Some(sid) = native_session_id(&event.payload) else { + return vec![]; + }; + let Some(call) = event + .payload + .pointer("/input/callID") + .or_else(|| event.payload.get("callID")) + .and_then(Value::as_str) + else { + return vec![]; + }; + if let Some(s) = self.sessions.get_mut(&sid) { + s.tool_starts.insert(call.into(), event.ts_ms); + if let Some(a) = event.payload.pointer("/output/args") { + s.tool_args.insert(call.into(), a.clone()); + } + } + vec![] + } + + fn tool_after(&mut self, event: &Envelope) -> Vec { + let Some(sid) = native_session_id(&event.payload) else { + return vec![]; + }; + let call = event + .payload + .pointer("/input/callID") + .or_else(|| event.payload.get("callID")) + .and_then(Value::as_str) + .unwrap_or(""); + let tool = event + .payload + .pointer("/input/tool") + .or_else(|| event.payload.get("tool")) + .and_then(Value::as_str) + .unwrap_or("tool"); + let Some(s) = self.sessions.get_mut(&sid) else { + return vec![]; + }; + if s.denied_tools.remove(call) { + return vec![]; + } + let Some(turn) = s.current_turn_span_id.clone() else { + return vec![]; + }; + s.tool_call_count += 1; + let output = s + .tool_outputs + .remove(call) + .or_else(|| event.payload.pointer("/result/output").cloned()) + .or_else(|| event.payload.get("output").cloned()); + let error = s.tool_errors.remove(call); + let args = s.tool_args.remove(call); + let name = if tool == "skill" { + args.as_ref() + .and_then(|v| v.get("name")) + .and_then(Value::as_str) + .map(|n| format!("skill: {n}")) + .unwrap_or_else(|| "skill".into()) + } else { + event + .payload + .pointer("/result/title") + .and_then(Value::as_str) + .unwrap_or(tool) + .to_string() + }; + let mut metadata = json!({"tool_name":tool,"call_id":call,"tool_outcome":if error.is_some(){"error"}else{"success"}}); + if tool == "skill" { + metadata["tool_kind"] = json!("skill"); + metadata["skill_name"] = args + .as_ref() + .and_then(|v| v.get("name")) + .cloned() + .unwrap_or(Value::Null) + } + vec![SpanOp::Insert(SpanRow { + span_id: ids::span_id(&self.daemon_session_id, &format!("tool:{sid}:{call}")), + root_span_id: s.effective_root_span_id.clone(), + parent_span_ids: vec![turn], + name, + span_type: SpanType::Tool, + start_ms: s.tool_starts.remove(call).or(Some(event.ts_ms)), + end_ms: Some(event.ts_ms), + input: args, + output, + error, + metadata: Some(metadata), + ..Default::default() + })] + } + + fn permission(&mut self, event: &Envelope) -> Vec { + let props = event.payload.get("properties").unwrap_or(&event.payload); + let Some(sid) = native_session_id(props) else { + return vec![]; + }; + let call = props + .get("callID") + .or_else(|| props.get("id")) + .and_then(Value::as_str) + .unwrap_or(""); + let reply = props + .get("response") + .or_else(|| props.get("status")) + .and_then(Value::as_str) + .unwrap_or(""); + if matches!(reply, "reject" | "denied" | "deny") { + let Some(s) = self.sessions.get_mut(&sid) else { + return vec![]; + }; + let Some(turn) = s.current_turn_span_id.clone() else { + return vec![]; + }; + s.denied_tools.insert(call.into()); + let tool = props.get("tool").and_then(Value::as_str).unwrap_or("tool"); + return vec![SpanOp::Insert(SpanRow { + span_id: ids::span_id(&self.daemon_session_id, &format!("tool:{sid}:{call}")), + root_span_id: s.effective_root_span_id.clone(), + parent_span_ids: vec![turn], + name: tool.into(), + span_type: SpanType::Tool, + start_ms: s.tool_starts.remove(call).or(Some(event.ts_ms)), + end_ms: Some(event.ts_ms), + input: s.tool_args.remove(call), + metadata: Some( + json!({"tool_name":tool,"call_id":call,"tool_approval":"denied","tool_outcome":"denied"}), + ), + error: Some("Permission denied".into()), + ..Default::default() + })]; + } + vec![] + } + + fn finish_session_event( + &mut self, + event: &Envelope, + close_root: bool, + error: Option, + ) -> Vec { + let Some(sid) = native_session_id(&event.payload) else { + return vec![]; + }; + self.close(&sid, event.ts_ms, close_root, error) + } + fn close( + &mut self, + sid: &str, + ts: i64, + close_root: bool, + error: Option, + ) -> Vec { + let Some(mut s) = self.sessions.remove(sid) else { + return vec![]; + }; + let mut ops = vec![]; + if let Some(turn) = s.current_turn_span_id.take() { + ops.push(SpanOp::Merge(SpanRow { + span_id: turn, + root_span_id: s.effective_root_span_id.clone(), + end_ms: Some(ts), + output: s.current_output.take().map(Value::String), + error: error.clone(), + ..Default::default() + })); + } + if close_root || s.parent_session_id.is_some() { + ops.push(SpanOp::Merge(SpanRow { + span_id: s.root_span_id, + root_span_id: s.effective_root_span_id, + end_ms: Some(ts), + metadata: Some( + json!({"total_turns":s.turn_number,"total_tool_calls":s.tool_call_count}), + ), + error, + ..Default::default() + })); + } else { + self.sessions.insert(sid.into(), s); + } + ops + } +} + +fn native_session_id(v: &Value) -> Option { + v.pointer("/input/sessionID") + .or_else(|| v.get("sessionID")) + .or_else(|| v.pointer("/properties/sessionID")) + .or_else(|| v.pointer("/properties/info/id")) + .or_else(|| v.pointer("/part/sessionID")) + .and_then(Value::as_str) + .map(str::to_owned) +} +fn num(v: &Value, p: &str) -> i64 { + v.pointer(p).and_then(Value::as_i64).unwrap_or(0) +} +fn format_error(v: Option<&Value>) -> String { + let Some(v) = v else { + return "UnknownError".into(); + }; + if let Some(s) = v.as_str() { + return s.lines().next().unwrap_or(s).into(); + } + let name = v + .get("name") + .and_then(Value::as_str) + .unwrap_or("UnknownError"); + let msg = v + .pointer("/data/message") + .or_else(|| v.get("message")) + .and_then(Value::as_str) + .unwrap_or(name); + format!("{msg}\n\ntype: {name}") +} +fn subagent_name(title: &str) -> String { + let Some((description, tail)) = title.split_once(" (@") else { + return title.into(); + }; + let agent = tail.strip_suffix(" subagent)").unwrap_or(tail); + format!("{agent}: {description}") +} +fn explicit_skills(input: &str) -> Vec { + input + .split_whitespace() + .filter_map(|s| { + s.strip_prefix("/skills:") + .or_else(|| s.strip_prefix("/skills")) + }) + .map(|s| { + s.trim_matches(|c: char| matches!(c, ',' | ')' | '.' | ';')) + .to_string() + }) + .filter(|s| !s.is_empty()) + .collect() +} diff --git a/bt-daemon/src/wire/envelope.rs b/bt-daemon/src/wire/envelope.rs index 67f3fd6..0455b7f 100644 --- a/bt-daemon/src/wire/envelope.rs +++ b/bt-daemon/src/wire/envelope.rs @@ -13,6 +13,10 @@ pub struct Envelope { /// The agent version, for payload-drift handling. Optional. #[serde(default, skip_serializing_if = "Option::is_none")] pub source_version: Option, + /// Version of the Braintrust instrumentation package that captured the + /// event. Distinct from the coding agent's `source_version`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub plugin_version: Option, /// Per-session queue + state key. pub session_id: String, /// Agent-native hook event name (not normalized). @@ -190,6 +194,7 @@ impl Envelope { RedactedEnvelope { source: self.source.clone(), source_version: self.source_version.clone(), + plugin_version: self.plugin_version.clone(), session_id: self.session_id.clone(), event: self.event.clone(), ts_ms: self.ts_ms, @@ -206,6 +211,8 @@ pub struct RedactedEnvelope { pub source: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub source_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub plugin_version: Option, pub session_id: String, pub event: String, pub ts_ms: i64, @@ -222,6 +229,7 @@ mod tests { Envelope { source: "codex".into(), source_version: Some("1.2.3".into()), + plugin_version: Some("0.4.0".into()), session_id: "sess-1".into(), event: "PostToolUse".into(), ts_ms: 1_753_639_552_123, diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index e5d48e4..15ee95f 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -3,7 +3,7 @@ mod support; use axum::http::StatusCode; use serde_json::{json, Value}; use support::agent_process::AgentTestWorld; -use support::agents::{ClaudeAgent, ClaudeRun, CodexAgent, CodexRun}; +use support::agents::{ClaudeAgent, ClaudeRun, CodexAgent, CodexRun, OpenCodeAgent, OpenCodeRun}; use support::inference::{ AnthropicMock, AnthropicRequest, AnthropicTurn, MockReply, OpenAiMock, OpenAiRequest, OpenAiTurn, @@ -37,6 +37,32 @@ fn codex_tool_call(request: &OpenAiRequest) -> OpenAiTurn { panic!("Codex offered no supported shell tool; offered tools: {names:?}"); } +fn shell_tool_call(request: &OpenAiRequest, call_id: &str, command: &str) -> OpenAiTurn { + let names = request.tool_names(); + let name = ["bash", "shell", "shell_command", "exec_command"] + .into_iter() + .find(|name| names.contains(name)) + .unwrap_or_else(|| { + panic!("agent offered no supported shell tool; offered tools: {names:?}") + }); + let arguments = match name { + "exec_command" => json!({"cmd": command, "login": false}), + _ => json!({"command": command}), + }; + OpenAiTurn::tool_call(call_id, name, arguments) +} + +fn tool_command(marker: &str) -> String { + #[cfg(unix)] + { + format!("printf {marker}") + } + #[cfg(windows)] + { + format!("Write-Output {marker}") + } +} + fn codex_tool_command() -> &'static str { #[cfg(unix)] { @@ -229,6 +255,78 @@ async fn claude_session_emits_traces() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the OpenCode CLI and built plugin"] +async fn opencode_session_emits_traces() { + let inference = OpenAiMock::new(|_context, request| { + assert_eq!(request.model(), Some("mock-model")); + if request.tool_names().is_empty() { + return MockReply::response(OpenAiTurn::text("OpenCode harness")); + } + if request.has_function_output("call_opencode_1") { + return MockReply::response(OpenAiTurn::text("OPENCODE_MOCK_OK")); + } + MockReply::response(shell_tool_call( + &request, + "call_opencode_1", + &tool_command("OPENCODE_TOOL_OK"), + )) + }); + let inference_server = TestServer::start(inference.router()).await; + let world = AgentTestWorld::start().await; + let opencode = OpenCodeAgent::new(&world); + + let output = opencode + .run( + &world, + OpenCodeRun::new( + "Run a shell command that prints OPENCODE_TOOL_OK, then reply with OPENCODE_MOCK_OK.", + ) + .mock_inference(inference_server.uri()), + ) + .await; + output.assert_success(); + if world.uses_mock_inference() { + output.assert_contains("OPENCODE_MOCK_OK"); + assert!( + inference + .requests() + .iter() + .any(|request| request.has_function_output("call_opencode_1")), + "OpenCode did not return the tool result: {}", + output.text() + ); + } + + let rows = world.wait_for_trace_delivery().await; + if world.uses_mock_ingest() { + assert!( + rows.iter() + .any(|row| row_contains(row, &["braintrust.plugin.opencode", "0.1.0"])), + "OpenCode trace origin metadata was not emitted" + ); + } + if world.uses_mock_inference() && world.uses_mock_ingest() { + let scenario = IngestScenario::new() + .expect("OpenCode trace origin", |row| { + row_contains(row, &["braintrust.plugin.opencode", "0.1.0"]) + }) + .expect("OpenCode turn input", |row| { + row_contains(row, &["Turn 1", "OPENCODE_TOOL_OK"]) + }) + .expect("OpenCode tool span", |row| { + row_contains(row, &[r#""type":"tool""#, "OPENCODE_TOOL_OK"]) + }) + .expect("OpenCode LLM span", |row| { + row_contains(row, &[r#""type":"llm""#, "call_opencode_1"]) + }) + .expect("OpenCode final LLM span", |row| { + row_contains(row, &[r#""type":"llm""#, "OPENCODE_MOCK_OK"]) + }); + world.wait_for_mock_ingest_scenario(&scenario).await; + } +} + #[test] fn request_helpers_recognize_tool_results_and_advertised_tools() { let openai = OpenAiRequest { diff --git a/bt-daemon/tests/braintrust_sink.rs b/bt-daemon/tests/braintrust_sink.rs index 331aa26..d3aeee6 100644 --- a/bt-daemon/tests/braintrust_sink.rs +++ b/bt-daemon/tests/braintrust_sink.rs @@ -105,7 +105,7 @@ async fn multi_profile_sessions_route_to_their_own_backend() { version: "test".into(), }); - let mut sink_a = factory.create("sess-a", "codex").unwrap(); + let mut sink_a = factory.create("sess-a", "codex", None).unwrap(); sink_a.configure(&session_config(&base_a)); sink_a .emit(&[SpanOp::Insert(row( @@ -121,7 +121,7 @@ async fn multi_profile_sessions_route_to_their_own_backend() { .unwrap(); sink_a.flush().await.unwrap(); - let mut sink_b = factory.create("sess-b", "codex").unwrap(); + let mut sink_b = factory.create("sess-b", "codex", None).unwrap(); sink_b.configure(&session_config(&base_b)); sink_b .emit(&[SpanOp::Insert(row( @@ -157,7 +157,7 @@ async fn merge_with_empty_name_does_not_clobber_the_original_name() { app_url: Some(base.clone()), version: "test".into(), }); - let mut sink = factory.create("sess-1", "codex").unwrap(); + let mut sink = factory.create("sess-1", "codex", None).unwrap(); sink.configure(&session_config(&base)); let named = row("s1", "s1", &[], "codex: myapp", SpanType::Task, 1, None); @@ -185,7 +185,7 @@ async fn attached_trace_children_keep_the_external_root() { app_url: Some(base.clone()), version: "test".into(), }); - let mut sink = factory.create("sess-1", "codex").unwrap(); + let mut sink = factory.create("sess-1", "codex", None).unwrap(); let mut config = session_config(&base); let mut components = SpanComponents::new(SpanObjectType::ProjectLogs); components.object_id = Some("proj-parent".into()); @@ -222,7 +222,7 @@ async fn exported_parent_preserves_object_root_and_propagated_event() { app_url: Some(base.clone()), version: "test".into(), }); - let mut sink = factory.create("sess-parent", "codex").unwrap(); + let mut sink = factory.create("sess-parent", "codex", None).unwrap(); let mut config = session_config(&base); let mut components = SpanComponents::new(SpanObjectType::Experiment); components.object_id = Some("exp-parent".into()); @@ -287,7 +287,7 @@ async fn braintrust_sink_delivers_spans_to_collector() { version: "test".into(), }); - let mut sink = factory.create("sess-1", "codex").unwrap(); + let mut sink = factory.create("sess-1", "codex", Some("0.9.0")).unwrap(); sink.configure(&session_config(&base)); // A session root (task) and a child tool span under it. @@ -343,6 +343,14 @@ async fn braintrust_sink_delivers_spans_to_collector() { ); assert!(bodies.contains("codex: sess-1"), "root span name missing"); assert!(bodies.contains("\"command\""), "tool input missing"); + assert!( + bodies.contains("braintrust.plugin.codex"), + "shared span origin name missing" + ); + assert!( + bodies.contains("0.9.0"), + "plugin version missing from shared span origin" + ); // Project registration happened (org_name path, no login). assert!( @@ -367,7 +375,7 @@ async fn experiment_sessions_use_experiment_object_type_and_id() { app_url: Some(base.clone()), version: "test".into(), }); - let mut sink = factory.create("sess-exp", "claude-code").unwrap(); + let mut sink = factory.create("sess-exp", "claude-code", None).unwrap(); let mut config = session_config(&base); config.destination = Some(TraceDestination::Experiment { experiment_id: "exp-42".into(), diff --git a/bt-daemon/tests/claude_translator.rs b/bt-daemon/tests/claude_translator.rs index 2ed2c02..96bb8c8 100644 --- a/bt-daemon/tests/claude_translator.rs +++ b/bt-daemon/tests/claude_translator.rs @@ -46,6 +46,7 @@ fn replay(name: &str) -> Vec { let env = Envelope { source: "claude-code".into(), source_version: None, + plugin_version: None, session_id: session_id.into(), event: record["hook"].as_str().unwrap().into(), ts_ms, @@ -254,6 +255,7 @@ fn claude_permission_denied_and_failed_tools_are_first_class_spans() { let event = |name: &str, payload: Value| Envelope { source: "claude-code".into(), source_version: None, + plugin_version: None, session_id: "s".into(), event: name.into(), ts_ms: 1, @@ -341,6 +343,7 @@ fn claude_pairs_tool_lifecycle_and_marks_explicit_skills_and_stop_failures() { let event = |name: &str, ts_ms: i64, payload: Value| Envelope { source: "claude-code".into(), source_version: Some("2.0.0".into()), + plugin_version: None, session_id: "lifecycle".into(), event: name.into(), ts_ms, @@ -458,6 +461,7 @@ fn claude_groups_streamed_rows_and_reads_late_final_output_at_session_end() { let event = |name: &str, ts_ms: i64, payload: Value| Envelope { source: "claude-code".into(), source_version: None, + plugin_version: None, session_id: "streamed".into(), event: name.into(), ts_ms, diff --git a/bt-daemon/tests/codex_translator.rs b/bt-daemon/tests/codex_translator.rs index fdb795e..dc0eae3 100644 --- a/bt-daemon/tests/codex_translator.rs +++ b/bt-daemon/tests/codex_translator.rs @@ -57,6 +57,7 @@ fn envelope(session: &str, event: &str, transcript_path: &str, extra: Value) -> Envelope { source: "codex".into(), source_version: None, + plugin_version: None, session_id: session.into(), event: event.into(), ts_ms: 0, diff --git a/bt-daemon/tests/inference_mocks.rs b/bt-daemon/tests/inference_mocks.rs index 03f3990..fc6b832 100644 --- a/bt-daemon/tests/inference_mocks.rs +++ b/bt-daemon/tests/inference_mocks.rs @@ -22,7 +22,8 @@ async fn openai_mock_streams_text_and_captures_requests() { .unwrap(); let body = response.text().await.unwrap(); - assert!(body.contains("response.output_item.done")); + assert!(body.contains("response.output_text.done")); + assert!(body.contains("response.completed")); assert!(body.contains("deterministic")); assert_eq!(mock.requests().len(), 1); } diff --git a/bt-daemon/tests/opencode_translator.rs b/bt-daemon/tests/opencode_translator.rs new file mode 100644 index 0000000..ad6d1bf --- /dev/null +++ b/bt-daemon/tests/opencode_translator.rs @@ -0,0 +1,149 @@ +use bt_daemon::wire::Envelope; +use bt_daemon::{Registry, SessionCtx, SpanOp, SpanRow, SpanType}; +use serde_json::json; +use std::collections::HashMap; + +fn event(name: &str, ts_ms: i64, payload: serde_json::Value) -> Envelope { + Envelope { + source: "opencode".into(), + source_version: Some("1.1.14".into()), + plugin_version: Some("0.1.0".into()), + session_id: "root-session".into(), + event: name.into(), + ts_ms, + payload, + route: None, + config: None, + } +} + +fn reduce(ops: Vec) -> HashMap { + let mut rows = HashMap::new(); + for op in ops { + match op { + SpanOp::Insert(row) => { + rows.insert(row.span_id.clone(), row); + } + SpanOp::Merge(update) => { + let row: &mut SpanRow = rows.entry(update.span_id.clone()).or_default(); + if update.end_ms.is_some() { + row.end_ms = update.end_ms; + } + if update.output.is_some() { + row.output = update.output; + } + if update.error.is_some() { + row.error = update.error; + } + if update.metadata.is_some() { + row.metadata = update.metadata; + } + } + } + } + rows +} + +#[test] +fn opencode_builds_turn_llm_tool_and_closes_the_session() { + let registry = Registry::default_agents(); + assert!(registry.sources().contains(&"opencode".to_string())); + let mut translator = registry.create("opencode", "root-session"); + let ctx = SessionCtx { + session_id: "root-session".into(), + config: None, + }; + let events = vec![ + event( + "session.created", + 1, + json!({"properties":{"info":{"id":"native"}}}), + ), + event( + "chat.message", + 2, + json!({"input":{"sessionID":"native","model":{"modelID":"gpt-5"}},"output":{"parts":[{"type":"text","text":"hello"}]}}), + ), + event( + "tool.execute.before", + 3, + json!({"input":{"sessionID":"native","callID":"call-1","tool":"read"},"output":{"args":{"path":"README.md"}}}), + ), + event( + "message.part.updated", + 4, + json!({"properties":{"part":{"sessionID":"native","messageID":"m1","type":"text","text":"world","time":{"end":4}}}}), + ), + event( + "message.updated", + 5, + json!({"properties":{"info":{"id":"m1","sessionID":"native","role":"assistant","providerID":"openai","modelID":"gpt-5","time":{"created":2,"completed":5},"tokens":{"input":5,"output":2,"reasoning":1,"cache":{"read":3,"write":4}}}}}), + ), + event( + "tool.execute.after", + 6, + json!({"input":{"sessionID":"native","callID":"call-1","tool":"read"},"result":{"title":"Read","output":"contents"}}), + ), + event( + "session.deleted", + 7, + json!({"properties":{"sessionID":"native"}}), + ), + ]; + let mut ops = Vec::new(); + for event in events { + ops.extend(translator.handle(&event, &ctx).unwrap()); + } + let rows = reduce(ops); + assert_eq!( + rows.values() + .filter(|r| r.span_type == SpanType::Task) + .count(), + 2 + ); + let llm = rows + .values() + .find(|r| r.span_type == SpanType::Llm) + .unwrap(); + assert_eq!(llm.metrics.as_ref().unwrap()["prompt_tokens"], 12); + assert_eq!(llm.output.as_ref().unwrap()[0]["content"], "world"); + let tool = rows + .values() + .find(|r| r.span_type == SpanType::Tool) + .unwrap(); + assert_eq!(tool.metadata.as_ref().unwrap()["tool_outcome"], "success"); + assert!(rows + .values() + .filter(|r| r.span_type == SpanType::Task) + .all(|r| r.end_ms.is_some())); +} + +#[test] +fn opencode_child_sessions_share_the_parent_trace_root() { + let registry = Registry::default_agents(); + let mut translator = registry.create("opencode", "root-session"); + let ctx = SessionCtx { + session_id: "root-session".into(), + config: None, + }; + let mut ops = translator + .handle( + &event( + "session.created", + 1, + json!({"properties":{"info":{"id":"parent"}}}), + ), + &ctx, + ) + .unwrap(); + ops.extend(translator.handle(&event("chat.message", 2, json!({"input":{"sessionID":"parent"},"output":{"parts":[{"type":"text","text":"delegate"}]}})), &ctx).unwrap()); + ops.extend(translator.handle(&event("session.created", 3, json!({"properties":{"info":{"id":"child","parentID":"parent","title":"find docs (@research subagent)"}}})), &ctx).unwrap()); + let rows = reduce(ops); + let parent = rows.values().find(|r| r.name == "OpenCode").unwrap(); + let child = rows + .values() + .find(|r| r.name == "research: find docs") + .unwrap(); + assert_eq!(child.root_span_id, parent.root_span_id); + assert_eq!(child.parent_span_ids.len(), 1); +} diff --git a/bt-daemon/tests/pipeline.rs b/bt-daemon/tests/pipeline.rs index f035718..520ed27 100644 --- a/bt-daemon/tests/pipeline.rs +++ b/bt-daemon/tests/pipeline.rs @@ -27,6 +27,7 @@ fn envelope(session_id: &str, event: &str, ts_ms: i64) -> Envelope { Envelope { source: "debug".into(), source_version: Some("0.0.0".into()), + plugin_version: None, session_id: session_id.into(), event: event.into(), ts_ms, diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index 744c1ff..29423b1 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -37,6 +37,18 @@ The world controls inference and ingest independently: - `BT_AGENT_INGEST_MODE=mock|live` selects captured local ingest or the normal Braintrust backend. +Mock ingest launches the feature-gated standalone daemon with test +credentials. Live ingest instead launches the profile-aware daemon embedded in +`bt`, selected by: + +- `BT_AGENT_BT_BIN` — `bt` executable to test (defaults to `bt` on `PATH`); +- `BT_AGENT_PROFILE` — optional saved OAuth or API-key profile; +- `BT_AGENT_ORG` — optional organization constraint; +- `BT_AGENT_PROJECT` — destination project name (defaults to `agent-e2e`). + +Only those non-secret selections are written to the harness route. The `bt` +daemon host resolves credentials and refreshes OAuth leases internally. + This allows deterministic inference to drive real Braintrust ingest without paying for model inference. Every test uses ordinary assertions for stable process behavior and trace delivery regardless of mode. When ingest is mocked, diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index f240494..dcdf1bd 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -1,6 +1,6 @@ use crate::support::ingest::{IngestMock, IngestScenario}; use crate::support::server::TestServer; -use bt_daemon::{run_status, StatusArgs}; +use bt_daemon::{flush_session, run_status, StatusArgs}; use serde_json::{json, Value}; use std::path::{Path, PathBuf}; use std::process::Stdio; @@ -12,6 +12,10 @@ use uuid::Uuid; const INFERENCE_MODE_ENV: &str = "BT_AGENT_INFERENCE_MODE"; const INGEST_MODE_ENV: &str = "BT_AGENT_INGEST_MODE"; +const BT_BIN_ENV: &str = "BT_AGENT_BT_BIN"; +const PROFILE_ENV: &str = "BT_AGENT_PROFILE"; +const ORG_ENV: &str = "BT_AGENT_ORG"; +const PROJECT_ENV: &str = "BT_AGENT_PROJECT"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum TestBackendMode { @@ -54,6 +58,9 @@ impl AgentTestWorld { let data_dir = root.path().join("daemon"); let socket = test_endpoint(root.path()); let config_path = data_dir.join("config.json"); + let profile = selection(PROFILE_ENV, "BRAINTRUST_PROFILE"); + let org = selection(ORG_ENV, "BRAINTRUST_ORG_NAME"); + let project = std::env::var(PROJECT_ENV).unwrap_or_else(|_| "agent-e2e".into()); std::fs::create_dir_all(&wrapper_dir).expect("create wrapper directory"); std::fs::create_dir_all(&data_dir).expect("create daemon data directory"); std::fs::write( @@ -61,7 +68,8 @@ impl AgentTestWorld { serde_json::to_vec_pretty(&json!({ "traceToBraintrust": true, "route": { - "destination": {"type": "project_logs", "project_name": "agent-e2e"}, + "auth": {"profile": profile, "org_name": org}, + "destination": {"type": "project_logs", "project_name": project}, "flush_mode": "flush_on_turn_end", "additional_metadata": {"test_harness": true} } @@ -71,11 +79,25 @@ impl AgentTestWorld { .expect("write daemon config"); let daemon_binary = Path::new(env!("CARGO_BIN_EXE_bt-daemon")); - write_bt_wrapper(&wrapper_dir, daemon_binary); - - let mut command = Command::new(daemon_binary); + let mut command = if ingest_mode == TestBackendMode::Live { + let bt_binary = std::env::var_os(BT_BIN_ENV).unwrap_or_else(|| "bt".into()); + write_bt_host_wrapper(&wrapper_dir, Path::new(&bt_binary)); + let mut command = Command::new(bt_binary); + command.args(["trace", "daemon"]); + if let Some(profile) = selection(PROFILE_ENV, "BRAINTRUST_PROFILE") { + command.arg("--profile").arg(profile); + } + if let Some(org) = selection(ORG_ENV, "BRAINTRUST_ORG_NAME") { + command.arg("--org").arg(org); + } + command + } else { + write_bt_wrapper(&wrapper_dir, daemon_binary); + let mut command = Command::new(daemon_binary); + command.arg("serve"); + command + }; command - .arg("serve") .arg("--socket") .arg(&socket) .arg("--data-dir") @@ -90,11 +112,12 @@ impl AgentTestWorld { command .env("BRAINTRUST_API_KEY", "test-key") .env("BRAINTRUST_API_URL", collector_server.uri()) - .env("BRAINTRUST_APP_URL", collector_server.uri()); + .env("BRAINTRUST_APP_URL", collector_server.uri()) + .env("BRAINTRUST_PROJECT", "agent-e2e"); } let daemon = command.spawn().expect("start daemon"); - wait_for_daemon(daemon_binary, &socket).await; + wait_for_daemon(&socket).await; Self { inference_mode, ingest_mode, @@ -142,7 +165,17 @@ impl AgentTestWorld { .env("BT_DAEMON_DATA_DIR", &self.data_dir) .env("BT_DAEMON_CONFIG", &self.config_path) .env("BRAINTRUST_FLUSH_ON_TURN_END", "true") + .env("BRAINTRUST_ADDITIONAL_METADATA", r#"{"test_harness":true}"#) .stdin(Stdio::null()); + if let Ok(profile) = std::env::var(PROFILE_ENV) { + command.env("BRAINTRUST_PROFILE", profile); + } + if let Ok(org) = std::env::var(ORG_ENV) { + command.env("BRAINTRUST_ORG_NAME", org); + } + if let Ok(project) = std::env::var(PROJECT_ENV) { + command.env("BRAINTRUST_PROJECT", project); + } if self.uses_mock_ingest() { command .env("BRAINTRUST_API_KEY", "test-key") @@ -243,6 +276,40 @@ impl AgentTestWorld { "live ingest reported daemon sink errors: {errors:?}" ); if emitted { + for session in &status.sessions { + let result = flush_session(&session.session_id, &self.socket, 10_000) + .await + .unwrap_or_else(|error| { + panic!( + "failed to flush live ingest session {}: {error}", + session.session_id + ) + }); + assert!( + result.flushed && result.pending == 0, + "live ingest session {} did not flush: {result:?}", + session.session_id + ); + if let Some(permalink) = &session.permalink { + eprintln!("Braintrust trace: {permalink}"); + } + } + let flushed_status = run_status(StatusArgs { + socket: Some(self.socket.clone()), + session_id: None, + }) + .await + .expect("query daemon status after live ingest flush") + .expect("daemon stopped during live ingest flush"); + let flush_errors = flushed_status + .sessions + .iter() + .filter_map(|session| session.last_error.as_deref()) + .collect::>(); + assert!( + flush_errors.is_empty(), + "live ingest flush reported daemon sink errors: {flush_errors:?}" + ); return Vec::new(); } } @@ -255,6 +322,13 @@ impl AgentTestWorld { } } +fn selection(primary: &str, fallback: &str) -> Option { + std::env::var(primary) + .ok() + .or_else(|| std::env::var(fallback).ok()) + .filter(|value| !value.trim().is_empty()) +} + impl Drop for AgentTestWorld { fn drop(&mut self) { let _ = self.daemon.start_kill(); @@ -276,6 +350,18 @@ fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { std::fs::set_permissions(&path, permissions).expect("make bt wrapper executable"); } +#[cfg(unix)] +fn write_bt_host_wrapper(directory: &Path, bt_binary: &Path) { + use std::os::unix::fs::PermissionsExt; + + let path = directory.join("bt"); + let script = format!("#!/bin/sh\nexec '{}' \"$@\"\n", bt_binary.display()); + std::fs::write(&path, script).expect("write bt host wrapper"); + let mut permissions = std::fs::metadata(&path).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&path, permissions).expect("make bt host wrapper executable"); +} + #[cfg(windows)] fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { let powershell = directory.join("bt-wrapper.ps1"); @@ -306,6 +392,25 @@ fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { std::fs::write(directory.join("bt"), shell).expect("write bt Git Bash wrapper"); } +#[cfg(windows)] +fn write_bt_host_wrapper(directory: &Path, bt_binary: &Path) { + let powershell = directory.join("bt-wrapper.ps1"); + let script = format!("& '{}' @args\nexit $LASTEXITCODE\n", bt_binary.display()); + std::fs::write(&powershell, script).expect("write bt host PowerShell wrapper"); + std::fs::write( + directory.join("bt.cmd"), + "@echo off\r\npowershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File \"%~dp0bt-wrapper.ps1\" %*\r\n", + ) + .expect("write bt host command wrapper"); + + let shell_binary = bt_binary.to_string_lossy().replace('\\', "/"); + std::fs::write( + directory.join("bt"), + format!("#!/bin/sh\nexec '{}' \"$@\"\n", shell_binary), + ) + .expect("write bt host Git Bash wrapper"); +} + #[cfg(unix)] fn test_endpoint(root: &Path) -> PathBuf { root.join("daemon.sock") @@ -319,20 +424,17 @@ fn test_endpoint(_root: &Path) -> PathBuf { )) } -async fn wait_for_daemon(daemon_binary: &Path, endpoint: &Path) { +async fn wait_for_daemon(endpoint: &Path) { for _ in 0..100 { - let output = Command::new(daemon_binary) - .arg("status") - .arg("--socket") - .arg(endpoint) - .output() - .await; - if let Ok(output) = output { - if output.status.success() - && !String::from_utf8_lossy(&output.stdout).contains("not running") - { - return; - } + if matches!( + run_status(StatusArgs { + socket: Some(endpoint.to_path_buf()), + session_id: None, + }) + .await, + Ok(Some(_)) + ) { + return; } tokio::time::sleep(Duration::from_millis(50)).await; } diff --git a/bt-daemon/tests/support/agents/mod.rs b/bt-daemon/tests/support/agents/mod.rs index 47447bd..ab06630 100644 --- a/bt-daemon/tests/support/agents/mod.rs +++ b/bt-daemon/tests/support/agents/mod.rs @@ -1,30 +1,35 @@ mod claude; mod codex; +mod opencode; mod test_plugin; #[allow(unused_imports)] pub use claude::{ClaudeAgent, ClaudeRun}; #[allow(unused_imports)] pub use codex::{CodexAgent, CodexRun}; +#[allow(unused_imports)] +pub use opencode::{OpenCodeAgent, OpenCodeRun}; use std::ffi::OsString; use std::path::PathBuf; use tokio::process::Command; pub struct AgentOutput { - output: std::process::Output, + success: bool, + stdout: Vec, + stderr: Vec, } impl AgentOutput { pub fn success(&self) -> bool { - self.output.status.success() + self.success } pub fn text(&self) -> String { format!( "stdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&self.output.stdout), - String::from_utf8_lossy(&self.output.stderr) + String::from_utf8_lossy(&self.stdout), + String::from_utf8_lossy(&self.stderr) ) } @@ -43,11 +48,30 @@ impl AgentOutput { self.text() ); } + + fn from_http_result(result: Result) -> Self { + match result { + Ok(stdout) => Self { + success: true, + stdout: stdout.into_bytes(), + stderr: Vec::new(), + }, + Err(stderr) => Self { + success: false, + stdout: Vec::new(), + stderr: stderr.into_bytes(), + }, + } + } } impl From for AgentOutput { fn from(output: std::process::Output) -> Self { - Self { output } + Self { + success: output.status.success(), + stdout: output.stdout, + stderr: output.stderr, + } } } @@ -71,6 +95,10 @@ impl ProcessOptions { .args(&self.args) .envs(self.env.iter().map(|(k, v)| (k, v))); } + + fn apply_env(&self, command: &mut Command) { + command.envs(self.env.iter().map(|(k, v)| (k, v))); + } } fn command_from_env(name: &str, fallback: &str) -> Command { diff --git a/bt-daemon/tests/support/agents/opencode.rs b/bt-daemon/tests/support/agents/opencode.rs new file mode 100644 index 0000000..8a1fecb --- /dev/null +++ b/bt-daemon/tests/support/agents/opencode.rs @@ -0,0 +1,272 @@ +use super::{command_from_env, AgentOutput, ProcessOptions}; +use crate::support::agent_process::AgentTestWorld; +use serde_json::json; +use std::ffi::OsString; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; +use tokio::process::Child; + +pub struct OpenCodeAgent { + home: PathBuf, + config_home: PathBuf, + data_home: PathBuf, + cache_home: PathBuf, + plugin: PathBuf, +} + +pub struct OpenCodeRun { + prompt: OsString, + mock_inference: Option, + options: ProcessOptions, +} + +impl OpenCodeRun { + pub fn new(prompt: impl Into) -> Self { + Self { + prompt: prompt.into(), + mock_inference: None, + options: ProcessOptions::default(), + } + } + + pub fn mock_inference(mut self, base_url: impl Into) -> Self { + self.mock_inference = Some(base_url.into()); + self + } + + pub fn arg(mut self, value: impl Into) -> Self { + self.options.arg(value); + self + } + + pub fn env(mut self, key: impl Into, value: impl Into) -> Self { + self.options.env(key, value); + self + } +} + +impl OpenCodeAgent { + pub fn new(world: &AgentTestWorld) -> Self { + let home = world.temp_path("opencode-home"); + let config_home = world.temp_path("opencode-config"); + let data_home = world.temp_path("opencode-data"); + let cache_home = world.temp_path("opencode-cache"); + for directory in [&home, &config_home, &data_home, &cache_home] { + std::fs::create_dir_all(directory).expect("create OpenCode test directory"); + } + let plugin = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("repository root") + .join("dist/opencode/dist/index.js"); + assert!( + plugin.is_file(), + "build the OpenCode plugin before running integration tests: {}", + plugin.display() + ); + Self { + home, + config_home, + data_home, + cache_home, + plugin, + } + } + + pub async fn run(&self, world: &AgentTestWorld, run: OpenCodeRun) -> AgentOutput { + let workspace = world.workspace(); + let mut config = json!({ + "$schema": "https://opencode.ai/config.json", + "plugin": [path_to_file_url(&self.plugin)], + "permission": {"*": "allow"} + }); + if world.uses_mock_inference() { + let base_url = run + .mock_inference + .as_ref() + .expect("mock OpenCode runs require a mock inference endpoint"); + config["model"] = json!("mock/mock-model"); + config["provider"] = json!({ + "mock": { + "npm": "@ai-sdk/openai", + "name": "Harness Mock", + "options": { + "baseURL": format!("{base_url}/v1"), + "apiKey": "test-key" + }, + "models": {"mock-model": {"name": "Mock Model"}} + } + }); + } + std::fs::write( + workspace.join("opencode.json"), + serde_json::to_vec_pretty(&config).unwrap(), + ) + .expect("write OpenCode config"); + + // Keep the plugin-hosting process alive until OpenCode has published + // the final message and session.idle events. The one-shot `run` server + // tears itself down as soon as it has printed the final response, which + // can drop those last plugin events before the daemon receives them. + let port = available_port(); + let server_url = format!("http://127.0.0.1:{port}"); + let server_log = world.temp_path("opencode-server.log"); + let server_log_file = + std::fs::File::create(&server_log).expect("create OpenCode server diagnostic log"); + let mut server = command_from_env("OPENCODE_BIN", "opencode"); + server + .args([ + "serve", + "--hostname", + "127.0.0.1", + "--port", + &port.to_string(), + ]) + .current_dir(&workspace) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::from(server_log_file)) + .kill_on_drop(true); + configure_environment(&mut server, self); + world.configure(&mut server); + run.options.apply_env(&mut server); + let mut server = server.spawn().expect("start OpenCode server"); + if let Err(error) = wait_for_server(&server_url, &mut server, &server_log).await { + let _ = server.kill().await; + return AgentOutput::from_http_result(Err(error)); + } + + let output = run_session(&server_url, &workspace, &run.prompt) + .await + .map_err(|error| format!("{error}\n{}", server_diagnostics(&mut server, &server_log))); + + // The synchronous message endpoint returns once the final assistant + // message is persisted. Give the server event bus a bounded window to + // deliver completion and idle callbacks before stopping the plugin host. + tokio::time::sleep(Duration::from_millis(500)).await; + let _ = server.kill().await; + AgentOutput::from_http_result(output) + } +} + +fn configure_environment(command: &mut tokio::process::Command, agent: &OpenCodeAgent) { + command + .env("HOME", &agent.home) + .env("USERPROFILE", &agent.home) + .env("XDG_CONFIG_HOME", &agent.config_home) + .env("XDG_DATA_HOME", &agent.data_home) + .env("XDG_CACHE_HOME", &agent.cache_home) + .env("TRACE_TO_BRAINTRUST", "true") + .env("BRAINTRUST_OPENCODE_ENABLE_TOOLS", "false") + .env("OPENCODE_DISABLE_MODELS_FETCH", "true"); + for key in [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "GOOGLE_GENERATIVE_AI_API_KEY", + ] { + command.env_remove(key); + } +} + +fn available_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("reserve OpenCode server port") + .local_addr() + .expect("read OpenCode server port") + .port() +} + +async fn wait_for_server(url: &str, server: &mut Child, log: &Path) -> Result<(), String> { + let client = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_millis(500)) + .build() + .map_err(|error| format!("build OpenCode health client: {error}"))?; + for _ in 0..60 { + if let Some(status) = server + .try_wait() + .map_err(|error| format!("query OpenCode server process: {error}"))? + { + return Err(format!( + "OpenCode server exited before becoming healthy with {status}\n{}", + server_diagnostics(server, log) + )); + } + if let Ok(response) = client.get(format!("{url}/global/health")).send().await { + if let Ok(response) = response.error_for_status() { + if response + .json::() + .await + .is_ok_and(|body| body["healthy"] == true) + { + return Ok(()); + } + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(format!( + "OpenCode server did not become healthy at {url}/global/health\n{}", + server_diagnostics(server, log) + )) +} + +async fn run_session(url: &str, workspace: &Path, prompt: &OsString) -> Result { + let client = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(90)) + .build() + .map_err(|error| format!("build OpenCode HTTP client: {error}"))?; + let directory = workspace.to_string_lossy().into_owned(); + let session = client + .post(format!("{url}/session")) + .query(&[("directory", &directory)]) + .json(&json!({})) + .send() + .await + .map_err(|error| format!("create OpenCode session: {error}"))? + .error_for_status() + .map_err(|error| format!("create OpenCode session: {error}"))? + .json::() + .await + .map_err(|error| format!("decode OpenCode session: {error}"))?; + let session_id = session["id"] + .as_str() + .ok_or_else(|| format!("OpenCode session response omitted id: {session}"))?; + let response = client + .post(format!("{url}/session/{session_id}/message")) + .query(&[("directory", &directory)]) + .json(&json!({ + "parts": [{"type": "text", "text": prompt.to_string_lossy()}] + })) + .send() + .await + .map_err(|error| format!("run OpenCode session: {error}"))? + .error_for_status() + .map_err(|error| format!("run OpenCode session: {error}"))? + .text() + .await + .map_err(|error| format!("read OpenCode response: {error}"))?; + Ok(response) +} + +fn server_diagnostics(server: &mut Child, log: &Path) -> String { + let status = match server.try_wait() { + Ok(Some(status)) => format!("exited with {status}"), + Ok(None) => "still running".into(), + Err(error) => format!("status unavailable: {error}"), + }; + let log = std::fs::read_to_string(log).unwrap_or_else(|error| format!("")); + format!("OpenCode server is {status}; stderr:\n{log}") +} + +fn path_to_file_url(path: &Path) -> String { + let path = path.to_string_lossy().replace('\\', "/"); + if path.starts_with('/') { + format!("file://{path}") + } else { + format!("file:///{path}") + } +} diff --git a/bt-daemon/tests/support/inference/README.md b/bt-daemon/tests/support/inference/README.md index 92567d3..a03e643 100644 --- a/bt-daemon/tests/support/inference/README.md +++ b/bt-daemon/tests/support/inference/README.md @@ -73,6 +73,13 @@ behavior while reporting traces to the normal Braintrust backend: ```console BT_AGENT_INFERENCE_MODE=mock BT_AGENT_INGEST_MODE=live \ +BT_AGENT_BT_BIN=/path/to/bt BT_AGENT_PROFILE=work \ +BT_AGENT_PROJECT=agent-e2e \ cargo test --manifest-path bt-daemon/Cargo.toml \ --all-features --test agent_integration -- --ignored --test-threads=1 ``` + +The live-ingest path starts the daemon through `bt`, so OAuth access-token +refresh and profile resolution remain inside the long-lived CLI host. The +standalone daemon is intentionally limited to mock-ingest and explicit API-key +development scenarios. diff --git a/bt-daemon/tests/support/inference/openai.rs b/bt-daemon/tests/support/inference/openai.rs index 92e3ec4..69ea7f9 100644 --- a/bt-daemon/tests/support/inference/openai.rs +++ b/bt-daemon/tests/support/inference/openai.rs @@ -93,13 +93,62 @@ impl OpenAiTurn { output_tokens, } => vec![ created, + json!({ + "type": "response.output_item.added", + "output_index": 0, + "item": { + "type": "message", + "role": "assistant", + "id": format!("msg_mock_{response_index}"), + "status": "in_progress", + "content": [] + } + }), + json!({ + "type": "response.content_part.added", + "item_id": format!("msg_mock_{response_index}"), + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text", "text": "", "annotations": []} + }), + json!({ + "type": "response.output_text.delta", + "item_id": format!("msg_mock_{response_index}"), + "output_index": 0, + "content_index": 0, + "delta": text.clone() + }), + json!({ + "type": "response.output_text.done", + "item_id": format!("msg_mock_{response_index}"), + "output_index": 0, + "content_index": 0, + "text": text.clone() + }), + json!({ + "type": "response.content_part.done", + "item_id": format!("msg_mock_{response_index}"), + "output_index": 0, + "content_index": 0, + "part": { + "type": "output_text", + "text": text.clone(), + "annotations": [] + } + }), json!({ "type": "response.output_item.done", + "output_index": 0, "item": { "type": "message", "role": "assistant", "id": format!("msg_mock_{response_index}"), - "content": [{"type": "output_text", "text": text}] + "status": "completed", + "content": [{ + "type": "output_text", + "text": text, + "annotations": [] + }] } }), completed(&response_id, input_tokens, output_tokens), @@ -110,19 +159,50 @@ impl OpenAiTurn { arguments, input_tokens, output_tokens, - } => vec![ - created, - json!({ - "type": "response.output_item.done", - "item": { - "type": "function_call", - "call_id": call_id, - "name": name, - "arguments": arguments.to_string() - } - }), - completed(&response_id, input_tokens, output_tokens), - ], + } => { + let item_id = format!("fc_mock_{response_index}"); + let arguments = arguments.to_string(); + vec![ + created, + json!({ + "type": "response.output_item.added", + "output_index": 0, + "item": { + "type": "function_call", + "id": item_id, + "call_id": call_id, + "name": name, + "arguments": "", + "status": "in_progress" + } + }), + json!({ + "type": "response.function_call_arguments.delta", + "item_id": item_id, + "output_index": 0, + "delta": arguments + }), + json!({ + "type": "response.function_call_arguments.done", + "item_id": item_id, + "output_index": 0, + "arguments": arguments + }), + json!({ + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "function_call", + "id": item_id, + "call_id": call_id, + "name": name, + "arguments": arguments, + "status": "completed" + } + }), + completed(&response_id, input_tokens, output_tokens), + ] + } Self::Events(events) => events, } } @@ -174,6 +254,7 @@ impl OpenAiMock { Router::new() .route("/v1/models", get(models)) .route("/v1/responses", post(responses)) + .route("/v1/chat/completions", post(chat_completions)) .route("/backend-api/plugins/featured", get(featured_plugins)) .with_state(Arc::clone(&self.state)) } @@ -229,3 +310,92 @@ async fn responses( } => raw_response(status, content_type, body), } } + +async fn chat_completions( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> axum::response::Response { + let body = match decode_json_body(&headers, &body) { + Ok(body) => body, + Err(error) => return json_response(StatusCode::BAD_REQUEST, json!({"error": error})), + }; + let request = OpenAiRequest { body }; + state + .requests + .lock() + .expect("request lock") + .push(request.clone()); + let index = state.next_index.fetch_add(1, Ordering::SeqCst); + match (state.handler)( + RequestContext { + request_index: index, + }, + request, + ) { + MockReply::Response(OpenAiTurn::Text { + text, + input_tokens, + output_tokens, + }) => { + let id = format!("chatcmpl_mock_{index}"); + let chunks = [ + json!({ + "id": id, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}}] + }), + json!({ + "id": id, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens + } + }), + ]; + let mut body = chunks + .iter() + .map(|chunk| format!("data: {chunk}\n\n")) + .collect::(); + body.push_str("data: [DONE]\n\n"); + raw_response(StatusCode::OK, "text/event-stream", body.into_bytes()) + } + MockReply::Response(OpenAiTurn::ToolCall { + call_id, + name, + arguments, + .. + }) => { + let body = json!({ + "id": format!("chatcmpl_mock_{index}"), + "object": "chat.completion", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "tool_calls": [{ + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": arguments.to_string()} + }] + }, + "finish_reason": "tool_calls" + }] + }); + json_response(StatusCode::OK, body) + } + MockReply::Response(OpenAiTurn::Events(_)) => json_response( + StatusCode::INTERNAL_SERVER_ERROR, + json!({"error": "raw response events are unsupported on chat completions"}), + ), + MockReply::HttpError { status, body } => json_response(status, body), + MockReply::Raw { + status, + content_type, + body, + } => raw_response(status, content_type, body), + } +} diff --git a/src/plugins/opencode/build.sh b/src/plugins/opencode/build.sh new file mode 100755 index 0000000..562fcee --- /dev/null +++ b/src/plugins/opencode/build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_DIR="${1:?usage: build.sh }" +PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$PLUGIN_DIR/../../.." && pwd)" +STAGE="$(mktemp -d)" +trap 'rm -rf "$STAGE"' EXIT + +rm -rf "$TARGET_DIR" +mkdir -p "$TARGET_DIR" "$STAGE/src/runtime" +TARGET_DIR="$(cd "$TARGET_DIR" && pwd)" +tar --exclude=node_modules --exclude=dist -cf - -C "$PLUGIN_DIR/content" . \ + | tar -xf - -C "$STAGE" +cp "$REPO_ROOT/src/runtime/js-daemon-client/src/index.ts" "$STAGE/src/runtime/daemon-client.ts" + +(cd "$STAGE" && bun install --frozen-lockfile && bun run build) +rm -rf "$STAGE/node_modules" +cp -R "$STAGE/." "$TARGET_DIR/" +(cd "$TARGET_DIR" && npm pack --pack-destination "$TARGET_DIR" >/dev/null) + +echo "Built OpenCode npm package and tarball in $TARGET_DIR" diff --git a/src/plugins/opencode/content/.env.example b/src/plugins/opencode/content/.env.example new file mode 100644 index 0000000..c3c92c7 --- /dev/null +++ b/src/plugins/opencode/content/.env.example @@ -0,0 +1,24 @@ +# Braintrust API Key (required) +# Get your API key from https://www.braintrust.dev/app/settings +BRAINTRUST_API_KEY=your-api-key-here + +# bt auth profile used by daemon-backed tracing (optional) +# BRAINTRUST_PROFILE=work + +# Project name for tracing (optional, default: opencode) +BRAINTRUST_PROJECT=opencode + +# Enable/disable automatic tracing (optional, default: false) +TRACE_TO_BRAINTRUST=true + +# Enable debug logging (optional, default: false) +BRAINTRUST_DEBUG=false + +# Custom Braintrust app URL (optional, default: https://www.braintrust.dev) +# BRAINTRUST_APP_URL=https://www.braintrust.dev + +# Organization name if you belong to multiple orgs (optional) +# BRAINTRUST_ORG_NAME=your-org-name + +# Custom API URL (optional, will be auto-discovered from login) +# BRAINTRUST_API_URL=https://api.braintrust.dev diff --git a/src/plugins/opencode/content/.gitignore b/src/plugins/opencode/content/.gitignore new file mode 100644 index 0000000..8ed2b7c --- /dev/null +++ b/src/plugins/opencode/content/.gitignore @@ -0,0 +1,13 @@ +node_modules/ +dist/ +*.log +.env +.env.local +.DS_Store +*.swp +*.swo +*~ +.vscode/ +.idea/ +*.iml +bun.lockb diff --git a/src/plugins/opencode/content/AGENTS.md b/src/plugins/opencode/content/AGENTS.md new file mode 100644 index 0000000..1cfa09f --- /dev/null +++ b/src/plugins/opencode/content/AGENTS.md @@ -0,0 +1,9 @@ +# OpenCode package guidance + +- Tracing code is a fail-open event adapter only. It forwards raw events to the + shared daemon client and contains no span construction or Braintrust delivery. +- `bt-daemon` owns translation, correlation, journaling, recovery, and delivery. +- The package never calls the Braintrust API or manages credentials. Its four + data-access tools delegate to non-interactive `bt` CLI commands. +- Preserve independent `trace_to_braintrust` and `enable_tools` controls. +- Run `make validate-opencode` from the monorepo root after package changes. diff --git a/src/plugins/opencode/content/CONTRIBUTING.md b/src/plugins/opencode/content/CONTRIBUTING.md new file mode 100644 index 0000000..c53e7ee --- /dev/null +++ b/src/plugins/opencode/content/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Development Guide + +OpenCode tracing is deliberately split into two independent runtime paths: + +- `src/tracing/daemon.ts` forwards raw OpenCode events to `bt` over the shared daemon client. +- `src/tools/` delegates the four optional Braintrust data-access tools to the + installed `bt` CLI. + +JavaScript must not construct spans, queue trace delivery, persist trace state, +manage credentials, or call the Braintrust API. Translation and delivery belong +to `bt-daemon`; data-access tools invoke `bt` with profile/org/project selection. + +## Local checks + +```bash +bun install --frozen-lockfile +bun run check +bun run typecheck +bun test +bun run build +``` + +From the monorepo root, `make validate-opencode` additionally builds and checks +the npm tarball. The real-agent integration test runs OpenCode with deterministic +inference through the daemon and mock Braintrust ingest. + +## Adding hooks + +Forward the unmodified native input/output in `src/tracing/daemon.ts`, then add +or update the corresponding Rust translator behavior and fixtures under +`bt-daemon/`. Do not add JavaScript processing state. + +## Adding tools + +Add tool definitions under `src/tools/` and expose the required operation in +`src/tools/bt-cli.ts`. Use argument arrays with `execFile`; never invoke a shell, +read credentials, or implement Braintrust HTTP requests in the package. diff --git a/src/plugins/opencode/content/LICENSE b/src/plugins/opencode/content/LICENSE new file mode 100644 index 0000000..8eec230 --- /dev/null +++ b/src/plugins/opencode/content/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Braintrust Data Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/plugins/opencode/content/README.md b/src/plugins/opencode/content/README.md new file mode 100644 index 0000000..f674b42 --- /dev/null +++ b/src/plugins/opencode/content/README.md @@ -0,0 +1,139 @@ +# @braintrust/trace-opencode + +Braintrust tracing plugin for [OpenCode](https://opencode.ai). The JavaScript +adapter forwards native OpenCode events to the installed `bt` daemon, which +constructs and delivers the trace. + +- **Session spans**: Root span for each OpenCode session with metadata (workspace, hostname, etc.) +- **Turn spans**: Captures each user-assistant interaction +- **Tool spans**: Records individual tool executions with inputs and outputs + +## Quick Start + +Add to your OpenCode configuration (`opencode.json` or `~/.config/opencode/opencode.json`): + +```json +{ + "plugin": ["@braintrust/trace-opencode@0.0.x"], <--- replace with latest version +} +``` + +Then, + +```bash +# Authenticate the installed bt CLI used by the tracing daemon +bt auth login +export TRACE_TO_BRAINTRUST="true" + +# Run OpenCode +opencode + +# View traces at: +# https://www.braintrust.dev/app/projects/opencode/logs +``` + +## Configuration + +You can configure the plugin using a config file or environment variables. + +### Config File + +Create a `braintrust.json` file in one of these locations: + +- `.opencode/braintrust.json` - Project-level config +- `~/.config/opencode/braintrust.json` - Global config + +```json +{ + "trace_to_braintrust": true, + "enable_tools": true, + "profile": "work", + "project": "my-project", + "debug": true +} +``` + +### Config Options + +| Config Key | Env Var | Type | Default | Description | +|------------|---------|------|---------|-------------| +| `trace_to_braintrust` | `TRACE_TO_BRAINTRUST` | boolean | `false` | Enable/disable tracing | +| `enable_tools` | `BRAINTRUST_OPENCODE_ENABLE_TOOLS` | boolean | `true` | Register Braintrust tools in OpenCode | +| `profile` | `BRAINTRUST_PROFILE` | string | current `bt` profile | Select the `bt` auth profile used by tracing and tools | +| `project` | `BRAINTRUST_PROJECT` | string | `"opencode"` | Project name for traces and project-scoped tools | +| `debug` | `BRAINTRUST_DEBUG` | boolean | `false` | Enable debug logging | +| `org_name` | `BRAINTRUST_ORG_NAME` | string | profile default | Organization selected within the tracing profile and for tools | +| `additional_metadata` | `BRAINTRUST_ADDITIONAL_METADATA` | string | | JSON object of additional metadata to attach to the root span. Standard metadata keys take precedence on conflict. | + +### Precedence + +Configuration is loaded with the following precedence (later overrides earlier): + +1. Default values +2. `~/.config/opencode/braintrust.json` (global config) +3. `.opencode/braintrust.json` (project config) +4. Environment variables (highest priority) + +## Disabling Braintrust Tools + +Set `enable_tools` to `false` to trace OpenCode sessions without registering Braintrust-branded tools (`braintrust_query_logs`, `braintrust_list_projects`, `braintrust_log_data`, `braintrust_get_experiments`): + +```json +{ + "trace_to_braintrust": true, + "enable_tools": false, + "project": "my-project" +} +``` + +Or use the environment variable: + +```bash +BRAINTRUST_OPENCODE_ENABLE_TOOLS=false TRACE_TO_BRAINTRUST=true opencode +``` + +## Adding Dynamic Metadata + +Use `BRAINTRUST_ADDITIONAL_METADATA` to attach custom key-value pairs to the root span. This is useful for tagging traces in CI or linking them back to a specific run. + +```bash +BRAINTRUST_ADDITIONAL_METADATA='{"ci": true, "run_id": "abc-123"}' opencode run "do the thing" +``` + +You can also set it via the config file: + +```json +{ + "additional_metadata": { + "team": "platform" + } +} +``` + +The value must be a JSON object. Any keys that conflict with standard root span metadata (`session_id`, `workspace`, `directory`, `hostname`, `username`, `os`) will be overridden by the standard values. + +## Trace Structure + +Sessions are traced with the following hierarchy: + +``` +Session (task span) +├── metadata: session_id, workspace, hostname, username, os +├── Turn 1 (task span) +│ ├── input: "user message" +│ ├── metadata: turn_number, agent, model +│ ├── Tool 1 (tool span) +│ │ ├── input: tool arguments +│ │ └── output: tool result +│ └── Tool 2 (tool span) +├── Turn 2 (task span) +│ └── ... +└── metrics: total_turns, total_tool_calls +``` + +## Runtime architecture + +The package never calls the Braintrust API from JavaScript. Tracing forwards +native events over local JSON-RPC to `bt-daemon`. The four optional data-access +tools invoke non-interactive `bt` CLI commands. In both cases, `bt` owns profile +selection, credential storage, refresh, backend resolution, and API transport. diff --git a/src/plugins/opencode/content/biome.json b/src/plugins/opencode/content/biome.json new file mode 100644 index 0000000..1f116a2 --- /dev/null +++ b/src/plugins/opencode/content/biome.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.3.11/schema.json", + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "suspicious": { + "noExplicitAny": "off" + }, + "style": { + "noNonNullAssertion": "off" + } + } + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2, + "lineWidth": 100 + }, + "javascript": { + "formatter": { + "quoteStyle": "double", + "semicolons": "asNeeded" + } + }, + "files": { + "includes": ["src/**/*.ts"] + } +} diff --git a/src/plugins/opencode/content/bun.lock b/src/plugins/opencode/content/bun.lock new file mode 100644 index 0000000..333310c --- /dev/null +++ b/src/plugins/opencode/content/bun.lock @@ -0,0 +1,55 @@ +{ + "lockfileVersion": 1, + "configVersion": 0, + "workspaces": { + "": { + "name": "opencode-braintrust", + "devDependencies": { + "@biomejs/biome": "^2.3.11", + "@opencode-ai/plugin": "^1.1.14", + "@opencode-ai/sdk": "^1.1.14", + "@types/bun": "^1.1.14", + "typescript": "^5.7.2", + }, + "peerDependencies": { + "@opencode-ai/plugin": ">=1.0.0", + "@opencode-ai/sdk": ">=1.0.0", + }, + }, + }, + "packages": { + "@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="], + + "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.3.11", "", { "os": "darwin", "cpu": "arm64" }, "sha512-/uXXkBcPKVQY7rc9Ys2CrlirBJYbpESEDme7RKiBD6MmqR2w3j0+ZZXRIL2xiaNPsIMMNhP1YnA+jRRxoOAFrA=="], + + "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.3.11", "", { "os": "darwin", "cpu": "x64" }, "sha512-fh7nnvbweDPm2xEmFjfmq7zSUiox88plgdHF9OIW4i99WnXrAC3o2P3ag9judoUMv8FCSUnlwJCM1B64nO5Fbg=="], + + "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-l4xkGa9E7Uc0/05qU2lMYfN1H+fzzkHgaJoy98wO+b/7Gl78srbCRRgwYSW+BTLixTBrM6Ede5NSBwt7rd/i6g=="], + + "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-XPSQ+XIPZMLaZ6zveQdwNjbX+QdROEd1zPgMwD47zvHV+tCGB88VH+aynyGxAHdzL+Tm/+DtKST5SECs4iwCLg=="], + + "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-/1s9V/H3cSe0r0Mv/Z8JryF5x9ywRxywomqZVLHAoa/uN0eY7F8gEngWKNS5vbbN/BsfpCG5yeBT5ENh50Frxg=="], + + "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-vU7a8wLs5C9yJ4CB8a44r12aXYb8yYgBn+WeyzbMjaCMklzCv1oXr8x+VEyWodgJt9bDmhiaW/I0RHbn7rsNmw=="], + + "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.3.11", "", { "os": "win32", "cpu": "arm64" }, "sha512-PZQ6ElCOnkYapSsysiTy0+fYX+agXPlWugh6+eQ6uPKI3vKAqNp6TnMhoM3oY2NltSB89hz59o8xIfOdyhi9Iw=="], + + "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.3.11", "", { "os": "win32", "cpu": "x64" }, "sha512-43VrG813EW+b5+YbDbz31uUsheX+qFKCpXeY9kfdAx+ww3naKxeVkTD9zLIWxUPfJquANMHrmW3wbe/037G0Qg=="], + + "@opencode-ai/plugin": ["@opencode-ai/plugin@1.1.14", "", { "dependencies": { "@opencode-ai/sdk": "1.1.14", "zod": "4.1.8" } }, "sha512-tfF4bEjeF7Gm0W0ViQUhzy77AaZfRxQ/kcPa7/Bc/YM9HddzjEqz0wOJ6ePG8UdUYc0dkKSJOJVhapUbAn/tOw=="], + + "@opencode-ai/sdk": ["@opencode-ai/sdk@1.1.14", "", {}, "sha512-PJFu2QPxnOk0VZzlPm+IxhD1wSA41PJyCG6gkxAMI767gfAO96A0ukJJN7VK/gO6MbxLF5oTFaxBX5rAGcBRVw=="], + + "@types/bun": ["@types/bun@1.3.5", "", { "dependencies": { "bun-types": "1.3.5" } }, "sha512-RnygCqNrd3srIPEWBd5LFeUYG7plCoH2Yw9WaZGyNmdTEei+gWaHqydbaIRkIkcbXwhBT94q78QljxN0Sk838w=="], + + "@types/node": ["@types/node@25.0.6", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-NNu0sjyNxpoiW3YuVFfNz7mxSQ+S4X2G28uqg2s+CzoqoQjLPsWSbsFFyztIAqt2vb8kfEAsJNepMGPTxFDx3Q=="], + + "bun-types": ["bun-types@1.3.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + + "zod": ["zod@4.1.8", "", {}, "sha512-5R1P+WwQqmmMIEACyzSvo4JXHY5WiAFHRMg+zBZKgKS+Q1viRa0C1hmUKtHltoIFKtIdki3pRxkmpP74jnNYHQ=="], + } +} diff --git a/src/plugins/opencode/content/docs/PUBLISHING.md b/src/plugins/opencode/content/docs/PUBLISHING.md new file mode 100644 index 0000000..6cd422b --- /dev/null +++ b/src/plugins/opencode/content/docs/PUBLISHING.md @@ -0,0 +1,21 @@ +# Publishing + +`@braintrust/trace-opencode` is built and released from the plugin monorepo. +The standalone repository workflow described by older versions of this file is +not used by this package. + +From the monorepo root: + +```bash +make validate-opencode +DRY_RUN=1 src/plugins/opencode/publish.sh dist/opencode +``` + +Validation builds the ESM package, bundles the shared daemon client, runs the +tests and static checks, verifies the tarball contents, and performs +`npm pack --dry-run`. + +Publishing a real package is a separate release-only action. It requires an +already validated tarball and the explicit safeguards enforced by +`src/plugins/opencode/publish.sh`. Normal CI and integration work must use only +the dry-run path. diff --git a/src/plugins/opencode/content/install.sh b/src/plugins/opencode/content/install.sh new file mode 100755 index 0000000..2365206 --- /dev/null +++ b/src/plugins/opencode/content/install.sh @@ -0,0 +1,48 @@ +#!/bin/bash +### +# Installation script for opencode-braintrust plugin +### + +set -e + +echo "Installing opencode-braintrust plugin..." + +# Check if bun is available +if ! command -v bun &> /dev/null; then + echo "Error: bun is required but not found. Install it from https://bun.sh" + exit 1 +fi + +# Install dependencies +echo "Installing dependencies..." +bun install + +# Build the plugin +echo "Building plugin..." +bun run build + +# Create OpenCode plugin directory if it doesn't exist +PLUGIN_DIR="$HOME/.config/opencode/plugin" +mkdir -p "$PLUGIN_DIR" + +# Copy plugin to OpenCode +echo "Installing plugin to $PLUGIN_DIR/trace-opencode.js" +cp dist/index.js "$PLUGIN_DIR/trace-opencode.js" + +echo "" +echo "✓ Plugin installed successfully!" +echo "" +echo "Next steps:" +echo "1. Authenticate the bt CLI:" +echo " bt auth login" +echo "" +echo "2. (Optional) Configure project name:" +echo " export BRAINTRUST_PROJECT='my-project'" +echo "" +echo "3. Run OpenCode:" +echo " opencode" +echo "" +echo "4. Your sessions will be traced to Braintrust automatically!" +# TODO: add org to url to fix the link +# echo " View at: https://www.braintrust.dev/app/projects/${BRAINTRUST_PROJECT:-opencode}/logs" +echo "" diff --git a/src/plugins/opencode/content/package.json b/src/plugins/opencode/content/package.json new file mode 100644 index 0000000..3290958 --- /dev/null +++ b/src/plugins/opencode/content/package.json @@ -0,0 +1,57 @@ +{ + "name": "@braintrust/trace-opencode", + "version": "0.1.0", + "description": "Automatically trace OpenCode conversations to Braintrust. Captures user messages, assistant responses, and tool calls for observability.", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "packageManager": "bun@1.3.14", + "scripts": { + "build": "bun build src/index.ts --outdir dist --target bun --format esm", + "dev": "bun run --watch src/index.ts", + "test": "bun test", + "test:watch": "bun test --watch", + "typecheck": "tsc --noEmit", + "lint": "biome lint src", + "lint:fix": "biome lint --write src", + "format": "biome format --write src", + "check": "biome check src", + "check:fix": "biome check --write src", + "check:fix-unsafe": "biome check --write --unsafe src", + "clean": "rm -rf dist", + "prepublishOnly": "bun run build" + }, + "keywords": [ + "opencode", + "braintrust", + "llm", + "tracing", + "observability", + "evaluation" + ], + "author": "Braintrust", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/braintrustdata/plugin-monorepo.git", + "directory": "src/plugins/opencode/content" + }, + "publishConfig": { + "access": "public" + }, + "files": [ + "dist", + "README.md" + ], + "devDependencies": { + "@biomejs/biome": "^2.3.11", + "@opencode-ai/plugin": "^1.1.14", + "@opencode-ai/sdk": "^1.1.14", + "@types/bun": "^1.1.14", + "typescript": "^5.7.2" + }, + "peerDependencies": { + "@opencode-ai/plugin": ">=1.0.0", + "@opencode-ai/sdk": ">=1.0.0" + } +} diff --git a/src/plugins/opencode/content/src/config.test.ts b/src/plugins/opencode/content/src/config.test.ts new file mode 100644 index 0000000..aca9ad2 --- /dev/null +++ b/src/plugins/opencode/content/src/config.test.ts @@ -0,0 +1,101 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test" +import { loadConfig, type PluginConfig, parseBooleanEnv } from "./config" + +describe("parseBooleanEnv", () => { + it("accepts true and 1 case-insensitively", () => { + expect(parseBooleanEnv("true")).toBe(true) + expect(parseBooleanEnv("TRUE")).toBe(true) + expect(parseBooleanEnv("1")).toBe(true) + }) + + it("rejects other and missing values", () => { + expect(parseBooleanEnv(undefined)).toBe(false) + expect(parseBooleanEnv("false")).toBe(false) + expect(parseBooleanEnv("yes")).toBe(false) + }) +}) + +describe("loadConfig", () => { + const keys = [ + "TRACE_TO_BRAINTRUST", + "BRAINTRUST_DEBUG", + "BRAINTRUST_ORG_NAME", + "BRAINTRUST_PROFILE", + "BRAINTRUST_PROJECT", + "BRAINTRUST_ADDITIONAL_METADATA", + "BRAINTRUST_OPENCODE_ENABLE_TOOLS", + ] + const original: Record = {} + + beforeEach(() => { + for (const key of keys) { + original[key] = process.env[key] + delete process.env[key] + } + }) + + afterEach(() => { + for (const key of keys) { + if (original[key] === undefined) delete process.env[key] + else process.env[key] = original[key] + } + }) + + it("defaults to the bt default profile and the opencode project", () => { + expect(loadConfig()).toEqual({ + profile: undefined, + orgName: undefined, + projectName: "opencode", + tracingEnabled: false, + enableTools: true, + debug: false, + additionalMetadata: undefined, + }) + }) + + it("loads routing and behavior from plugin configuration", () => { + const pluginConfig: PluginConfig = { + profile: "work", + org_name: "acme", + project: "agents", + trace_to_braintrust: true, + enable_tools: false, + debug: true, + additional_metadata: { team: "platform" }, + } + expect(loadConfig(pluginConfig)).toEqual({ + profile: "work", + orgName: "acme", + projectName: "agents", + tracingEnabled: true, + enableTools: false, + debug: true, + additionalMetadata: { team: "platform" }, + }) + }) + + it("lets environment selection override plugin configuration", () => { + process.env.BRAINTRUST_PROFILE = "personal" + process.env.BRAINTRUST_ORG_NAME = "braintrust" + process.env.BRAINTRUST_PROJECT = "opencode-runs" + process.env.TRACE_TO_BRAINTRUST = "true" + process.env.BRAINTRUST_OPENCODE_ENABLE_TOOLS = "false" + process.env.BRAINTRUST_ADDITIONAL_METADATA = '{"ci":true}' + + expect(loadConfig({ profile: "work", project: "other" })).toMatchObject({ + profile: "personal", + orgName: "braintrust", + projectName: "opencode-runs", + tracingEnabled: true, + enableTools: false, + additionalMetadata: { ci: true }, + }) + }) + + it("ignores invalid optional metadata", () => { + process.env.BRAINTRUST_ADDITIONAL_METADATA = "not-json" + expect(loadConfig({ additional_metadata: { fallback: true } }).additionalMetadata).toEqual({ + fallback: true, + }) + }) +}) diff --git a/src/plugins/opencode/content/src/config.ts b/src/plugins/opencode/content/src/config.ts new file mode 100644 index 0000000..1a15dfb --- /dev/null +++ b/src/plugins/opencode/content/src/config.ts @@ -0,0 +1,99 @@ +/** Shared OpenCode plugin configuration. */ + +export interface BraintrustConfig { + profile?: string + orgName?: string + projectName: string + tracingEnabled: boolean + enableTools: boolean + debug: boolean + additionalMetadata?: Record +} + +/** + * Plugin config from opencode.json `braintrust` section. + * Uses snake_case to match environment variable naming. + */ +export interface PluginConfig { + profile?: string + org_name?: string + project?: string + trace_to_braintrust?: boolean + enable_tools?: boolean + debug?: boolean + additional_metadata?: Record +} + +/** + * Parse a boolean environment variable. + * Accepts: "true", "TRUE", "1", "tRuE" (case-insensitive) as truthy. + * All other values (including undefined, "false", "0", "no") are falsy. + */ +export function parseBooleanEnv(value: string | undefined): boolean { + if (!value) return false + const normalized = value.toLowerCase() + return normalized === "true" || normalized === "1" +} + +/** + * Load Braintrust config with the following precedence (later overrides earlier): + * 1. Default values + * 2. opencode.json `braintrust` section (pluginConfig) + * 3. Environment variables (highest priority) + */ +export function loadConfig(pluginConfig?: PluginConfig): BraintrustConfig { + // Defaults + const defaults: BraintrustConfig = { + profile: undefined, + orgName: undefined, + projectName: "opencode", + tracingEnabled: false, + enableTools: true, + debug: false, + } + + // Layer 1: Apply opencode.json config (if provided) + if (pluginConfig) { + if (pluginConfig.profile) defaults.profile = pluginConfig.profile + if (pluginConfig.org_name) defaults.orgName = pluginConfig.org_name + if (pluginConfig.project) defaults.projectName = pluginConfig.project + if (pluginConfig.trace_to_braintrust !== undefined) { + defaults.tracingEnabled = pluginConfig.trace_to_braintrust + } + if (pluginConfig.enable_tools !== undefined) { + defaults.enableTools = pluginConfig.enable_tools + } + if (pluginConfig.debug !== undefined) { + defaults.debug = pluginConfig.debug + } + if (pluginConfig.additional_metadata) { + defaults.additionalMetadata = pluginConfig.additional_metadata + } + } + + // Layer 2: Apply environment variables (override opencode.json) + let additionalMetadata = defaults.additionalMetadata + if (process.env.BRAINTRUST_ADDITIONAL_METADATA) { + try { + additionalMetadata = JSON.parse(process.env.BRAINTRUST_ADDITIONAL_METADATA) + } catch { + // Invalid JSON in env var — ignore and keep config file value (if any) + } + } + + return { + profile: process.env.BRAINTRUST_PROFILE || defaults.profile, + orgName: process.env.BRAINTRUST_ORG_NAME || defaults.orgName, + projectName: process.env.BRAINTRUST_PROJECT || defaults.projectName, + tracingEnabled: process.env.TRACE_TO_BRAINTRUST + ? parseBooleanEnv(process.env.TRACE_TO_BRAINTRUST) + : defaults.tracingEnabled, + enableTools: process.env.BRAINTRUST_OPENCODE_ENABLE_TOOLS + ? parseBooleanEnv(process.env.BRAINTRUST_OPENCODE_ENABLE_TOOLS) + : defaults.enableTools, + debug: process.env.BRAINTRUST_DEBUG + ? parseBooleanEnv(process.env.BRAINTRUST_DEBUG) + : defaults.debug, + additionalMetadata, + } +} diff --git a/src/plugins/opencode/content/src/config/index.ts b/src/plugins/opencode/content/src/config/index.ts new file mode 100644 index 0000000..94b8f17 --- /dev/null +++ b/src/plugins/opencode/content/src/config/index.ts @@ -0,0 +1,2 @@ +export type { BraintrustConfig, PluginConfig } from "../config" +export { loadConfig, parseBooleanEnv } from "../config" diff --git a/src/plugins/opencode/content/src/index.test.ts b/src/plugins/opencode/content/src/index.test.ts new file mode 100644 index 0000000..76650bc --- /dev/null +++ b/src/plugins/opencode/content/src/index.test.ts @@ -0,0 +1,116 @@ +/** + * Tests for OpenCode plugin registration behavior + */ + +import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import type { PluginInput } from "@opencode-ai/plugin" + +mock.module("@opencode-ai/plugin", () => ({ + tool: Object.assign((definition: unknown) => definition, { + schema: { + string: () => ({ + optional() { + return this + }, + describe() { + return this + }, + }), + number: () => ({ + optional() { + return this + }, + describe() { + return this + }, + }), + }, + }), +})) + +function createInput(directory: string): PluginInput { + return { + directory, + worktree: directory, + project: "test-project", + client: { + app: { + log: async () => {}, + }, + }, + } as unknown as PluginInput +} + +describe("BraintrustPlugin", () => { + const originalEnv: Record = {} + const envVars = [ + "TRACE_TO_BRAINTRUST", + "BRAINTRUST_PROFILE", + "BRAINTRUST_OPENCODE_ENABLE_TOOLS", + "HOME", + ] + let directory: string + + beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), "braintrust-opencode-plugin-")) + for (const key of envVars) { + originalEnv[key] = process.env[key] + delete process.env[key] + } + process.env.HOME = directory + process.env.TRACE_TO_BRAINTRUST = "false" + process.env.BRAINTRUST_OPENCODE_ENABLE_TOOLS = "true" + }) + + afterEach(() => { + rmSync(directory, { recursive: true, force: true }) + for (const key of envVars) { + if (originalEnv[key] !== undefined) { + process.env[key] = originalEnv[key] + } else { + delete process.env[key] + } + } + }) + + it("registers Braintrust tools when enabled", async () => { + const { BraintrustPlugin } = await import("./index") + const hooks = await BraintrustPlugin(createInput(directory)) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(hooks.tool).toBeDefined() + expect(Object.keys(hooks.tool ?? {}).sort()).toEqual([ + "braintrust_get_experiments", + "braintrust_list_projects", + "braintrust_log_data", + "braintrust_query_logs", + ]) + }) + + it("does not register Braintrust tools when disabled", async () => { + process.env.BRAINTRUST_OPENCODE_ENABLE_TOOLS = "false" + + const { BraintrustPlugin } = await import("./index") + const hooks = await BraintrustPlugin(createInput(directory)) + await Promise.resolve() + + expect(hooks.tool).toBeUndefined() + }) + + it("registers daemon tracing independently of the tools", async () => { + process.env.TRACE_TO_BRAINTRUST = "true" + process.env.BRAINTRUST_OPENCODE_ENABLE_TOOLS = "false" + + const { BraintrustPlugin } = await import("./index") + const hooks = await BraintrustPlugin(createInput(directory)) + + expect(hooks.event).toBeDefined() + expect(hooks["chat.message"]).toBeDefined() + expect(hooks["tool.execute.before"]).toBeDefined() + expect(hooks["tool.execute.after"]).toBeDefined() + expect(hooks.tool).toBeUndefined() + }) +}) diff --git a/src/plugins/opencode/content/src/index.ts b/src/plugins/opencode/content/src/index.ts new file mode 100644 index 0000000..4fb9303 --- /dev/null +++ b/src/plugins/opencode/content/src/index.ts @@ -0,0 +1,107 @@ +/** + * Braintrust plugin for OpenCode + * + * Provides two main capabilities: + * 1. Tracing - Automatically traces OpenCode sessions to Braintrust + * 2. Data Access - Tools to query and interact with Braintrust data + */ + +import type { Hooks, Plugin, PluginInput } from "@opencode-ai/plugin" +import { loadConfig, type PluginConfig } from "./config" +import { createBraintrustTools } from "./tools" +import { BtCliToolsClient } from "./tools/bt-cli" +import { createDaemonTracingHooks } from "./tracing/daemon" + +export const BraintrustPlugin: Plugin = async (input: PluginInput) => { + const { client } = input + + // Load plugin config from config files + // Precedence: global config -> project config (project overrides global) + let pluginConfig: PluginConfig | undefined + try { + const fs = await import("node:fs") + const path = await import("node:path") + const os = await import("node:os") + + // Load configs in order: global first, then project (so project overrides global) + const configPaths = [ + path.join(os.homedir(), ".config", "opencode", "braintrust.json"), // global + path.join(input.directory, ".opencode", "braintrust.json"), // project + ] + + for (const configPath of configPaths) { + try { + if (fs.existsSync(configPath)) { + const content = fs.readFileSync(configPath, "utf-8") + const parsed = JSON.parse(content) as PluginConfig + // Merge: later config overrides earlier + pluginConfig = pluginConfig ? { ...pluginConfig, ...parsed } : parsed + } + } catch { + // Continue to next path + } + } + } catch { + // Config loading failed, proceed with env vars only + } + + const config = loadConfig(pluginConfig) + + const toolsClient = config.enableTools ? new BtCliToolsClient(config) : undefined + + const hooks: Hooks = {} + + // Add tracing hooks if enabled + if (config.tracingEnabled) { + const tracingHooks = createDaemonTracingHooks(input, config, (message, extra) => { + client.app + .log({ body: { service: "braintrust-trace", level: "warn", message, extra } }) + .catch(() => {}) + }) + Object.assign(hooks, tracingHooks) + + client.app + .log({ + body: { + service: "braintrust", + level: "info", + message: `Tracing hooks registered: ${Object.keys(tracingHooks).join(", ")}`, + }, + }) + .catch(() => {}) + } + + if (toolsClient) { + hooks.tool = createBraintrustTools(toolsClient) + } + + if (config.tracingEnabled || toolsClient) { + client.app + .log({ + body: { + service: "braintrust", + level: "info", + message: `Braintrust plugin enabled for project "${config.projectName}"`, + }, + }) + .catch(() => {}) + } else { + client.app + .log({ + body: { + service: "braintrust", + level: "info", + message: "Braintrust tracing and tools are disabled.", + }, + }) + .catch(() => {}) + } + + return hooks +} + +// Default export for OpenCode plugin loading +export default BraintrustPlugin + +// Re-export types only (not the class, since OpenCode will try to call all exports as plugins) +export type { BraintrustConfig, PluginConfig } from "./config" diff --git a/src/plugins/opencode/content/src/tools/bt-cli.test.ts b/src/plugins/opencode/content/src/tools/bt-cli.test.ts new file mode 100644 index 0000000..3276826 --- /dev/null +++ b/src/plugins/opencode/content/src/tools/bt-cli.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "bun:test" +import { existsSync, readFileSync } from "node:fs" +import type { BraintrustConfig } from "../config" +import { BtCliToolsClient } from "./bt-cli" + +const config: BraintrustConfig = { + profile: "work", + orgName: "acme", + projectName: "agents", + tracingEnabled: true, + enableTools: true, + debug: false, +} + +describe("BtCliToolsClient", () => { + it("passes profile and org selection to bt for project listing", async () => { + const calls: string[][] = [] + const client = new BtCliToolsClient(config, async (args) => { + calls.push(args) + return '[{"id":"project-1","name":"agents"}]' + }) + + expect(await client.listProjects()).toEqual([{ id: "project-1", name: "agents" }]) + expect(calls).toEqual([ + [ + "projects", + "list", + "--json", + "--no-input", + "--prefer-profile", + "--profile", + "work", + "--org", + "acme", + ], + ]) + }) + + it("resolves the selected project and delegates SQL to bt", async () => { + const calls: string[][] = [] + const client = new BtCliToolsClient(config, async (args) => { + calls.push(args) + return args[0] === "projects" + ? '[{"id":"project-1","name":"agents"}]' + : '{"data":[{"id":"row-1"}]}' + }) + + expect(await client.queryLogs("SELECT * FROM logs LIMIT 1")).toEqual({ + data: [{ id: "row-1" }], + }) + expect(calls[1]).toContain("SELECT * FROM project_logs('project-1') LIMIT 1") + expect(calls[1]).toContain("--project") + expect(calls[1]).toContain("agents") + }) + + it("delegates experiment listing and applies the requested limit", async () => { + const client = new BtCliToolsClient(config, async () => '[{"id":"one"},{"id":"two"}]') + expect(await client.listExperiments(1)).toEqual([{ id: "one" }]) + }) + + it("delegates manual logs through bt sync push and removes the temporary input", async () => { + let inputPath = "" + let inputContents = "" + const client = new BtCliToolsClient(config, async (args) => { + if (args[0] === "projects") return '[{"id":"project-1","name":"agents"}]' + inputPath = args[args.indexOf("--in") + 1] ?? "" + inputContents = readFileSync(inputPath, "utf8") + return '{"uploaded_rows":1}' + }) + + const id = await client.logData({ + id: "row-1", + span_id: "span-1", + root_span_id: "span-1", + input: "hello", + span_attributes: { name: "Manual Log", type: "task" }, + }) + + expect(id).toBe("row-1") + expect(JSON.parse(inputContents)).toMatchObject({ id: "row-1", input: "hello" }) + expect(inputPath).toContain("bt-opencode-tools-") + expect(existsSync(inputPath)).toBe(false) + }) +}) diff --git a/src/plugins/opencode/content/src/tools/bt-cli.ts b/src/plugins/opencode/content/src/tools/bt-cli.ts new file mode 100644 index 0000000..c80f9fa --- /dev/null +++ b/src/plugins/opencode/content/src/tools/bt-cli.ts @@ -0,0 +1,125 @@ +import { execFile } from "node:child_process" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { promisify } from "node:util" +import type { BraintrustConfig } from "../config" + +const execFileAsync = promisify(execFile) + +export interface ProjectInfo { + id: string + name: string +} + +export interface ToolLogData { + id: string + span_id: string + root_span_id: string + input?: string + output?: string + expected?: string + scores?: Record + metadata?: Record + tags?: string[] + span_attributes: { name: string; type: "task" } +} + +export type BtCliRunner = (args: string[]) => Promise + +async function defaultRunner(args: string[]): Promise { + const executable = process.env.BT_EXECUTABLE || "bt" + try { + const { stdout } = await execFileAsync(executable, args, { + encoding: "utf8", + timeout: 30_000, + maxBuffer: 10 * 1024 * 1024, + }) + return stdout + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new Error(`bt CLI command failed: ${message}`) + } +} + +/** Runs OpenCode data-access tools through bt so bt exclusively owns auth and API access. */ +export class BtCliToolsClient { + private readonly config: BraintrustConfig + private readonly runner: BtCliRunner + + constructor(config: BraintrustConfig, runner: BtCliRunner = defaultRunner) { + this.config = config + this.runner = runner + } + + private selection(includeProject = false): string[] { + return [ + "--json", + "--no-input", + "--prefer-profile", + ...(this.config.profile ? ["--profile", this.config.profile] : []), + ...(this.config.orgName ? ["--org", this.config.orgName] : []), + ...(includeProject ? ["--project", this.config.projectName] : []), + ] + } + + private async json(args: string[]): Promise { + const stdout = await this.runner(args) + try { + return JSON.parse(stdout) as T + } catch { + throw new Error("bt CLI returned invalid JSON") + } + } + + async listProjects(): Promise { + return this.json(["projects", "list", ...this.selection()]) + } + + private async project(): Promise { + const existing = (await this.listProjects()).find( + (project) => project.name === this.config.projectName, + ) + if (existing) return existing + return this.json([ + "projects", + "create", + this.config.projectName, + ...this.selection(), + ]) + } + + async queryLogs(sql: string): Promise { + const project = await this.project() + const query = sql.replace(/\bFROM\s+logs\b/gi, `FROM project_logs('${project.id}')`) + return this.json(["sql", query, "--non-interactive", ...this.selection(true)]) + } + + async listExperiments(limit: number): Promise { + const result = await this.json(["experiments", "list", ...this.selection(true)]) + return result.slice(0, limit) + } + + async logData(data: ToolLogData): Promise { + await this.project() + const directory = await mkdtemp(join(tmpdir(), "bt-opencode-tools-")) + const input = join(directory, "event.jsonl") + try { + await writeFile(input, `${JSON.stringify(data)}\n`, { encoding: "utf8", mode: 0o600 }) + await this.json([ + "sync", + "push", + `project_logs:${this.config.projectName}`, + "--in", + input, + "--root", + directory, + "--fresh", + ...this.selection(true), + ]) + return data.id + } finally { + await rm(directory, { recursive: true, force: true }) + } + } +} diff --git a/src/plugins/opencode/content/src/tools/index.ts b/src/plugins/opencode/content/src/tools/index.ts new file mode 100644 index 0000000..4df1298 --- /dev/null +++ b/src/plugins/opencode/content/src/tools/index.ts @@ -0,0 +1,148 @@ +/** + * Braintrust tools for OpenCode + * + * Provides tools for: + * - Querying logs + * - Listing projects + * - Logging data + */ + +import type { ToolDefinition } from "@opencode-ai/plugin" +import { tool } from "@opencode-ai/plugin" +import type { BtCliToolsClient, ToolLogData } from "./bt-cli" + +/** + * Create Braintrust tools + */ +export function createBraintrustTools(client: BtCliToolsClient): Record { + return { + braintrust_query_logs: tool({ + description: `Query Braintrust logs using SQL. +Use "FROM logs" in your query - it will be automatically rewritten. + +SQL dialect notes: +- Use hour(timestamp_column), day(timestamp_column) instead of date_trunc +- Use "interval 1 day" (singular unit, no quotes) for intervals +- Use dot notation for nested fields: metadata.key +- Common columns: id, input, output, expected, scores, metadata, created + +Example queries: +- SELECT * FROM logs ORDER BY created DESC LIMIT 10 +- SELECT * FROM logs WHERE scores.Factuality < 0.5 +- SELECT * FROM logs WHERE created > now() - interval 1 hour`, + args: { + query: tool.schema.string().describe("SQL query to execute against Braintrust logs"), + }, + async execute(args) { + try { + const results = await client.queryLogs(args.query) + return JSON.stringify(results, null, 2) + } catch (error) { + return `Error executing query: ${error}` + } + }, + }), + + braintrust_list_projects: tool({ + description: "List all projects in your Braintrust organization", + args: {}, + async execute() { + try { + const projects = await client.listProjects() + if (projects.length === 0) { + return "No projects found." + } + return projects.map((p) => `- ${p.name} (${p.id})`).join("\n") + } catch (error) { + return `Error listing projects: ${error}` + } + }, + }), + + braintrust_log_data: tool({ + description: `Log data to Braintrust for evaluation or tracking. +You can log input/output pairs, scores, and metadata. + +This is useful for: +- Recording important decisions or outputs for review +- Creating evaluation datasets +- Tracking model performance over time`, + args: { + input: tool.schema.string().optional().describe("The input that was given (optional)"), + output: tool.schema.string().optional().describe("The output that was produced (optional)"), + expected: tool.schema.string().optional().describe("The expected/ideal output (optional)"), + scores: tool.schema + .string() + .optional() + .describe('JSON object of scores, e.g. {"accuracy": 0.95, "relevance": 0.8}'), + metadata: tool.schema + .string() + .optional() + .describe('JSON object of additional metadata, e.g. {"task_type": "code_review"}'), + tags: tool.schema.string().optional().describe("Comma-separated list of tags"), + }, + async execute(args) { + try { + const spanId = crypto.randomUUID() + + const data: ToolLogData = { + id: crypto.randomUUID(), + span_id: spanId, + root_span_id: spanId, + span_attributes: { + name: "Manual Log", + type: "task", + }, + } + + if (args.input) data.input = args.input + if (args.output) data.output = args.output + if (args.expected) data.expected = args.expected + + if (args.scores) { + try { + data.scores = JSON.parse(args.scores) + } catch { + return "Error: scores must be valid JSON" + } + } + + if (args.metadata) { + try { + data.metadata = JSON.parse(args.metadata) + } catch { + return "Error: metadata must be valid JSON" + } + } + + if (args.tags) { + data.tags = args.tags.split(",").map((t) => t.trim()) + } + + const rowId = await client.logData(data) + return `Successfully logged data with ID: ${rowId}` + } catch (error) { + return `Error logging data: ${error}` + } + }, + }), + + braintrust_get_experiments: tool({ + description: "List recent experiments for the current project", + args: { + limit: tool.schema + .number() + .optional() + .describe("Maximum number of experiments to return (default: 10)"), + }, + async execute(args) { + const limit = args.limit || 10 + try { + return JSON.stringify(await client.listExperiments(limit), null, 2) + } catch (error) { + return `Error getting experiments: ${error}` + } + }, + }), + } +} diff --git a/src/plugins/opencode/content/src/tracing/daemon.ts b/src/plugins/opencode/content/src/tracing/daemon.ts new file mode 100644 index 0000000..dcbd296 --- /dev/null +++ b/src/plugins/opencode/content/src/tracing/daemon.ts @@ -0,0 +1,94 @@ +import { randomUUID } from "node:crypto" +import type { Hooks, PluginInput } from "@opencode-ai/plugin" +import type { Event } from "@opencode-ai/sdk" +import { DaemonClient } from "../runtime/daemon-client" +import { PLUGIN_VERSION } from "../version" + +type Logger = (message: string, extra?: Record) => void + +interface TracingRouteConfig { + profile?: string + orgName?: string + projectName: string + additionalMetadata?: Record +} + +const FORWARDED_NATIVE_EVENTS = new Set([ + "session.created", + "session.idle", + "session.deleted", + "session.error", + "message.part.updated", + "message.updated", + "permission.asked", + "permission.replied", +]) + +export function createDaemonTracingHooks( + input: PluginInput, + config: TracingRouteConfig, + log: Logger, +): Partial { + // One transport stream per plugin instance. Native OpenCode session IDs and + // parent relationships stay untouched in each payload for the daemon to + // interpret. + const daemonSessionId = randomUUID() + const daemon = new DaemonClient({ + source: "opencode", + pluginVersion: PLUGIN_VERSION, + warn: (message) => log("Braintrust tracing unavailable", { message }), + }) + + const forward = async (event: string, payload: unknown) => { + await daemon.log({ + source: "opencode", + source_version: process.env.OPENCODE_VERSION, + session_id: daemonSessionId, + event, + ts_ms: Date.now(), + payload: { + ...(payload as Record), + directory: input.directory, + worktree: input.worktree, + }, + route: { + auth: { + ...(config.profile ? { profile: config.profile } : {}), + ...(config.orgName ? { org_name: config.orgName } : {}), + }, + destination: { + type: "project_logs", + project_name: config.projectName, + }, + flush_mode: "fire_and_forget", + ...(config.additionalMetadata ? { additional_metadata: config.additionalMetadata } : {}), + }, + }) + if (event === "session.idle" || event === "session.deleted" || event === "session.error") { + await daemon.flush(daemonSessionId) + } + } + + return { + event: async ({ event }: { event: Event }) => { + if (event.type === "server.instance.disposed") { + await daemon.flush(daemonSessionId) + await daemon.close() + return + } + if (!FORWARDED_NATIVE_EVENTS.has(event.type)) return + await forward(event.type, { properties: event.properties }) + }, + "chat.message": async (hookInput, hookOutput) => + forward("chat.message", { input: hookInput, output: hookOutput }), + "experimental.chat.system.transform": async (hookInput, hookOutput) => + forward("experimental.chat.system.transform", { + input: hookInput, + output: hookOutput, + }), + "tool.execute.before": async (hookInput, hookOutput) => + forward("tool.execute.before", { input: hookInput, output: hookOutput }), + "tool.execute.after": async (hookInput, hookOutput) => + forward("tool.execute.after", { input: hookInput, result: hookOutput }), + } +} diff --git a/src/plugins/opencode/content/src/version.ts b/src/plugins/opencode/content/src/version.ts new file mode 100644 index 0000000..b01a794 --- /dev/null +++ b/src/plugins/opencode/content/src/version.ts @@ -0,0 +1,5 @@ +// Read the package version at build/bundle time so package.json remains the +// single source of truth for span origin provenance. +import manifest from "../package.json" with { type: "json" } + +export const PLUGIN_VERSION: string = manifest.version diff --git a/src/plugins/opencode/content/tsconfig.json b/src/plugins/opencode/content/tsconfig.json new file mode 100644 index 0000000..87caaf5 --- /dev/null +++ b/src/plugins/opencode/content/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM"], + "strict": false, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "declarationDir": "dist", + "outDir": "dist", + "rootDir": "src", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "types": ["bun-types"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/src/plugins/opencode/content/uninstall.sh b/src/plugins/opencode/content/uninstall.sh new file mode 100755 index 0000000..1d4cbe7 --- /dev/null +++ b/src/plugins/opencode/content/uninstall.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +PLUGIN_DIR="$HOME/.config/opencode/plugin" +mkdir -p "$PLUGIN_DIR" + +echo "Uninstalling plugin at $PLUGIN_DIR/trace-opencode.js" +rm "$PLUGIN_DIR/trace-opencode.js" + +echo "" +echo "✓ Plugin uninstalled successfully!" +echo "" diff --git a/src/plugins/opencode/publish.sh b/src/plugins/opencode/publish.sh new file mode 100755 index 0000000..c81d9bd --- /dev/null +++ b/src/plugins/opencode/publish.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_DIR="${BUILD_DIR:-$PLUGIN_DIR/../../../dist/opencode}" +NPM_TAG="${NPM_TAG:-latest}" +case "$NPM_TAG" in latest|rc|next|beta) ;; *) echo "unsupported NPM_TAG: $NPM_TAG" >&2; exit 1;; esac + +tarball="$(find "$BUILD_DIR" -maxdepth 1 -name 'braintrust-trace-opencode-*.tgz' -print -quit)" +[[ -n "$tarball" ]] || { echo "build OpenCode before publishing" >&2; exit 1; } +version="$(node -p "require('$BUILD_DIR/package.json').version")" + +if npm view "@braintrust/trace-opencode@$version" version >/dev/null 2>&1; then + echo "@braintrust/trace-opencode@$version is already published" >&2 + exit 1 +fi + +if [[ "${DRY_RUN:-}" == "1" ]]; then + npm publish "$tarball" --tag "$NPM_TAG" --dry-run + exit 0 +fi + +[[ "${RELEASE_CONFIRM:-}" == "publish-@braintrust/trace-opencode@$version" ]] || { + echo "set RELEASE_CONFIRM=publish-@braintrust/trace-opencode@$version for a release-approved publish" >&2 + exit 1 +} +npm publish "$tarball" --tag "$NPM_TAG" diff --git a/src/plugins/opencode/validate.sh b/src/plugins/opencode/validate.sh new file mode 100755 index 0000000..36e1bc8 --- /dev/null +++ b/src/plugins/opencode/validate.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_DIR="${1:?usage: validate.sh }" +fail() { echo "validate: $*" >&2; exit 1; } + +[[ -f "$TARGET_DIR/package.json" ]] || fail "missing package.json" +[[ -f "$TARGET_DIR/dist/index.js" ]] || fail "missing production entrypoint" +tarball="$(find "$TARGET_DIR" -maxdepth 1 -name 'braintrust-trace-opencode-*.tgz' -print -quit)" +[[ -n "$tarball" ]] || fail "missing npm tarball" + +(cd "$TARGET_DIR" && bun install --frozen-lockfile && bun run check && bun run typecheck && bun test && bun run build) +node -e "import(process.argv[1])" "$(cd "$TARGET_DIR" && pwd)/dist/index.js" +(cd "$TARGET_DIR" && npm pack --dry-run >/dev/null) + +contents="$(tar -tzf "$tarball")" +grep -q '^package/dist/index.js$' <<<"$contents" || fail "tarball omits dist/index.js" +grep -q '^package/README.md$' <<<"$contents" || fail "tarball omits README.md" +grep -q '^package/LICENSE$' <<<"$contents" || fail "tarball omits LICENSE" +for removed in client.ts tracing.ts event-processor.ts replay.ts span-queue.ts span-sink.ts; do + [[ ! -e "$TARGET_DIR/src/$removed" ]] || fail "old JavaScript tracing runtime remains: src/$removed" +done +if grep -R -n -E "fetch\(|/v1/|/btql|apikey/login|from [\"']\.\./tools" "$TARGET_DIR/src/tracing"; then + fail "daemon tracing performs API access or imports the tools runtime" +fi +if grep -R -n -E \ + "fetch\(|https?://api\.|/v1/|/btql|apikey/login|BRAINTRUST_API_(KEY|URL)|BRAINTRUST_APP_URL" \ + "$TARGET_DIR/src"; then + fail "OpenCode package contains direct Braintrust API or credential handling" +fi +grep -q 'BtCliToolsClient' "$TARGET_DIR/src/tools/index.ts" \ + || fail "data-access tools no longer delegate to bt" +grep -q '"--prefer-profile"' "$TARGET_DIR/src/tools/bt-cli.ts" \ + || fail "bt tool delegation does not prefer managed profiles" +if grep -q '"braintrust"[[:space:]]*:' "$TARGET_DIR/package.json"; then + fail "OpenCode package still depends on the Braintrust JavaScript SDK" +fi + +echo "validate: OpenCode npm package OK ($TARGET_DIR)" diff --git a/src/runtime/js-daemon-client/package.json b/src/runtime/js-daemon-client/package.json new file mode 100644 index 0000000..94fc1a6 --- /dev/null +++ b/src/runtime/js-daemon-client/package.json @@ -0,0 +1,6 @@ +{ + "name": "@braintrust/coding-agent-daemon-client", + "version": "0.1.0", + "private": true, + "type": "module" +} diff --git a/src/runtime/js-daemon-client/src/index.ts b/src/runtime/js-daemon-client/src/index.ts new file mode 100644 index 0000000..6aef0bc --- /dev/null +++ b/src/runtime/js-daemon-client/src/index.ts @@ -0,0 +1,336 @@ +import { spawn } from "node:child_process" +import { createHash } from "node:crypto" +import { createConnection, type Socket } from "node:net" +import { homedir } from "node:os" +import { join } from "node:path" + +export const DAEMON_PROTOCOL_VERSION = 1 + +export interface DaemonSessionRoute { + auth?: { + profile?: string + org_name?: string + } + destination: unknown + flush_mode?: "fire_and_forget" | "flush_on_turn_end" + additional_metadata?: Record +} + +export interface DaemonEnvelope { + source: string + source_version?: string + plugin_version?: string + session_id: string + event: string + ts_ms: number + payload: unknown + route?: DaemonSessionRoute +} + +export interface DaemonSessionStatus { + session_id: string + source: string + queued: number + spans_emitted: number + permalink?: string + last_error?: string +} + +export interface DaemonStatus { + daemon_version: string + uptime_ms: number + sessions: DaemonSessionStatus[] +} + +export interface DaemonClientOptions { + source: string + pluginVersion?: string + socketPath?: string + btExecutable?: string + startArguments?: string[] + connectAttempts?: number + connectDelayMs?: number + requestTimeoutMs?: number + warn?: (message: string) => void +} + +interface RpcResponse { + jsonrpc: "2.0" + id: number + result?: unknown + error?: { code: number; message: string; data?: unknown } +} + +interface PendingRequest { + resolve: (value: unknown) => void + reject: (error: Error) => void + timer: ReturnType +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +export function daemonSocketPath(env: NodeJS.ProcessEnv = process.env): string { + if (env.BT_DAEMON_SOCKET) return env.BT_DAEMON_SOCKET + if (process.platform === "win32") { + const identity = `${env.USERDOMAIN ?? ""}\\${env.USERNAME ?? ""}` + const suffix = createHash("sha256").update(identity).digest("hex").slice(0, 16) + return `\\\\.\\pipe\\braintrust-bt-daemon-${suffix}` + } + if (env.XDG_RUNTIME_DIR) return join(env.XDG_RUNTIME_DIR, "braintrust", "daemon.sock") + return join(env.HOME ?? env.USERPROFILE ?? homedir(), ".braintrust", "run", "daemon.sock") +} + +export class DaemonClient { + private readonly options: Required< + Pick< + DaemonClientOptions, + "source" | "btExecutable" | "connectAttempts" | "connectDelayMs" | "requestTimeoutMs" + > + > & + DaemonClientOptions + private socket?: Socket + private input = "" + private nextId = 1 + private pending = new Map() + private connecting?: Promise + private queue: Promise = Promise.resolve() + private warned = new Set() + + constructor(options: DaemonClientOptions) { + this.options = { + btExecutable: "bt", + connectAttempts: 50, + connectDelayMs: 20, + requestTimeoutMs: 10_000, + ...options, + } + } + + async log(envelope: DaemonEnvelope): Promise { + return this.serial(async () => { + const event = { + ...envelope, + ...(this.options.pluginVersion ? { plugin_version: this.options.pluginVersion } : {}), + } + try { + const result = (await this.request("event.log", event)) as { accepted?: boolean } + return result.accepted === true + } catch (error) { + this.disconnect(error) + try { + const result = (await this.request("event.log", event)) as { accepted?: boolean } + return result.accepted === true + } catch (retryError) { + this.warnOnce(`event:${String(retryError)}`) + return false + } + } + }) + } + + async flush(sessionId: string, timeoutMs = 10_000): Promise { + return this.serial(async () => { + try { + const result = (await this.request("session.flush", { + session_id: sessionId, + timeout_ms: timeoutMs, + })) as { flushed?: boolean } + return result.flushed === true + } catch (error) { + this.warnOnce(`flush:${String(error)}`) + return false + } + }) + } + + async status(sessionId?: string): Promise { + return this.serial(async () => { + try { + return (await this.request("status.get", { + ...(sessionId ? { session_id: sessionId } : {}), + })) as DaemonStatus + } catch (error) { + this.warnOnce(`status:${String(error)}`) + return undefined + } + }) + } + + async close(): Promise { + await this.queue + this.disconnect(new Error("daemon client closed")) + } + + private serial(operation: () => Promise): Promise { + const next = this.queue.then(operation, operation) + this.queue = next.then( + () => undefined, + () => undefined, + ) + return next + } + + private async request(method: string, params: unknown): Promise { + await this.ensureConnected() + const socket = this.socket + if (!socket || socket.destroyed) throw new Error("daemon socket is unavailable") + + const id = this.nextId++ + const response = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id) + reject(new Error(`daemon request timed out: ${method}`)) + }, this.options.requestTimeoutMs) + this.pending.set(id, { resolve, reject, timer }) + }) + socket.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`) + return response + } + + private async ensureConnected(): Promise { + if (this.socket && !this.socket.destroyed) return + if (!this.connecting) { + this.connecting = this.connectWithStartup().finally(() => { + this.connecting = undefined + }) + } + return this.connecting + } + + private async connectWithStartup(): Promise { + try { + await this.connectOnce() + } catch { + this.startDaemon() + let lastError: unknown + for (let attempt = 0; attempt < this.options.connectAttempts; attempt++) { + await sleep(this.options.connectDelayMs) + try { + await this.connectOnce() + return + } catch (error) { + lastError = error + } + } + throw new Error(`could not connect to Braintrust tracing daemon: ${String(lastError)}`) + } + } + + private connectOnce(): Promise { + return new Promise((resolve, reject) => { + const socket = createConnection(this.options.socketPath ?? daemonSocketPath()) + const onError = (error: Error) => { + socket.destroy() + reject(error) + } + socket.once("error", onError) + socket.once("connect", async () => { + socket.off("error", onError) + this.attach(socket) + try { + const result = (await this.requestOnConnectedSocket("initialize", { + protocol_version: DAEMON_PROTOCOL_VERSION, + client: { + source: this.options.source, + plugin_version: this.options.pluginVersion, + pid: process.pid, + }, + })) as { + protocol_version: number + capabilities?: { sources?: string[] } + } + if (result.protocol_version !== DAEMON_PROTOCOL_VERSION) { + throw new Error(`unsupported daemon protocol ${result.protocol_version}`) + } + if (!result.capabilities?.sources?.includes(this.options.source)) { + throw new Error(`daemon does not support ${this.options.source}; update bt`) + } + resolve() + } catch (error) { + this.disconnect(error) + reject(error) + } + }) + }) + } + + private requestOnConnectedSocket(method: string, params: unknown): Promise { + const socket = this.socket + if (!socket) return Promise.reject(new Error("daemon socket is unavailable")) + const id = this.nextId++ + const response = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id) + reject(new Error(`daemon request timed out: ${method}`)) + }, this.options.requestTimeoutMs) + this.pending.set(id, { resolve, reject, timer }) + }) + socket.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`) + return response + } + + private attach(socket: Socket): void { + this.socket = socket + this.input = "" + socket.setEncoding("utf8") + socket.on("data", (chunk: string) => this.onData(chunk)) + socket.on("error", (error) => this.disconnect(error)) + socket.on("close", () => this.disconnect(new Error("daemon connection closed"))) + } + + private onData(chunk: string): void { + this.input += chunk + while (true) { + const newline = this.input.indexOf("\n") + if (newline < 0) return + const line = this.input.slice(0, newline) + this.input = this.input.slice(newline + 1) + if (!line) continue + let response: RpcResponse + try { + response = JSON.parse(line) as RpcResponse + } catch { + continue + } + const pending = this.pending.get(response.id) + if (!pending) continue + this.pending.delete(response.id) + clearTimeout(pending.timer) + if (response.error) pending.reject(new Error(response.error.message)) + else pending.resolve(response.result) + } + } + + private startDaemon(): void { + try { + const child = spawn( + this.options.btExecutable, + this.options.startArguments ?? ["trace", "daemon"], + { detached: true, stdio: "ignore", windowsHide: true }, + ) + child.once("error", (error) => this.warnOnce(`start:${String(error)}`)) + child.unref() + } catch (error) { + this.warnOnce(`start:${String(error)}`) + } + } + + private disconnect(reason: unknown): void { + const socket = this.socket + this.socket = undefined + if (socket && !socket.destroyed) socket.destroy() + for (const pending of this.pending.values()) { + clearTimeout(pending.timer) + pending.reject(reason instanceof Error ? reason : new Error(String(reason))) + } + this.pending.clear() + } + + private warnOnce(message: string): void { + if (this.warned.has(message)) return + this.warned.add(message) + this.options.warn?.(message) + } +} diff --git a/src/runtime/js-daemon-client/tests/client.test.ts b/src/runtime/js-daemon-client/tests/client.test.ts new file mode 100644 index 0000000..7dfe3b6 --- /dev/null +++ b/src/runtime/js-daemon-client/tests/client.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "bun:test" +import { createHash } from "node:crypto" +import { mkdtempSync, rmSync } from "node:fs" +import { createServer } from "node:net" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { DaemonClient, daemonSocketPath } from "../src/index" + +describe("daemonSocketPath", () => { + test("prefers the explicit environment override", () => { + expect(daemonSocketPath({ BT_DAEMON_SOCKET: "/tmp/custom.sock" })).toBe("/tmp/custom.sock") + }) + + test("matches the Unix runtime-directory contract", () => { + if (process.platform === "win32") return + expect(daemonSocketPath({ XDG_RUNTIME_DIR: "/run/user/1" })).toBe( + "/run/user/1/braintrust/daemon.sock", + ) + }) + + test("documents the Windows identity hash contract", () => { + const identity = "ACME\\alice" + expect(createHash("sha256").update(identity).digest("hex").slice(0, 16)).toHaveLength(16) + }) +}) + +test("serializes initialize, events, flush, and status over one connection", async () => { + const temp = mkdtempSync(join(tmpdir(), "bt-js-client-")) + const endpoint = process.platform === "win32" + ? `\\\\.\\pipe\\bt-js-client-${process.pid}-${Date.now()}` + : join(temp, "daemon.sock") + const methods: string[] = [] + const eventParams: Array> = [] + const server = createServer((socket) => { + socket.setEncoding("utf8") + let input = "" + socket.on("data", (chunk: string) => { + input += chunk + while (input.includes("\n")) { + const newline = input.indexOf("\n") + const request = JSON.parse(input.slice(0, newline)) + input = input.slice(newline + 1) + methods.push(request.method) + if (request.method === "event.log") eventParams.push(request.params) + const result = request.method === "initialize" + ? { protocol_version: 1, capabilities: { sources: ["opencode"] } } + : request.method === "event.log" + ? { accepted: true } + : request.method === "session.flush" + ? { flushed: true, pending: 0 } + : { daemon_version: "test", uptime_ms: 1, sessions: [] } + socket.write(`${JSON.stringify({ jsonrpc: "2.0", id: request.id, result })}\n`) + } + }) + }) + await new Promise((resolve, reject) => server.listen(endpoint, resolve).once("error", reject)) + const client = new DaemonClient({ + source: "opencode", + pluginVersion: "0.1.0", + socketPath: endpoint, + }) + const envelope = (name: string) => ({ + source: "opencode", + session_id: "session", + event: name, + ts_ms: Date.now(), + payload: {}, + }) + expect(await Promise.all([client.log(envelope("one")), client.log(envelope("two"))])).toEqual([ + true, + true, + ]) + expect(await client.flush("session")).toBe(true) + expect((await client.status("session"))?.daemon_version).toBe("test") + expect(methods).toEqual(["initialize", "event.log", "event.log", "session.flush", "status.get"]) + expect(eventParams.map((params) => params.plugin_version)).toEqual(["0.1.0", "0.1.0"]) + await client.close() + await new Promise((resolve) => server.close(() => resolve())) + rmSync(temp, { recursive: true, force: true }) +}) + +test("fails open when the daemon and bt executable are absent", async () => { + const warnings: string[] = [] + const endpoint = process.platform === "win32" + ? `\\\\.\\pipe\\bt-js-client-missing-${process.pid}-${Date.now()}` + : join(tmpdir(), `bt-js-client-missing-${process.pid}-${Date.now()}.sock`) + const client = new DaemonClient({ + source: "opencode", + socketPath: endpoint, + btExecutable: `bt-does-not-exist-${Date.now()}`, + connectAttempts: 1, + connectDelayMs: 1, + warn: (message) => warnings.push(message), + }) + expect( + await client.log({ + source: "opencode", + session_id: "session", + event: "session.created", + ts_ms: Date.now(), + payload: {}, + }), + ).toBe(false) + expect(warnings.length).toBeGreaterThan(0) + await client.close() +})