diff --git a/.changeset/local-thread-history.md b/.changeset/local-thread-history.md new file mode 100644 index 0000000..72fadf8 --- /dev/null +++ b/.changeset/local-thread-history.md @@ -0,0 +1,5 @@ +--- +"grok-bot-cli": minor +--- + +Persist sent prompts and fetched thread messages in local JSONL history, with offline search, a configurable directory, and recording opt-outs. diff --git a/README.md b/README.md index cb0df33..b91b627 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,53 @@ gbot bots delete Writer Run `gbot --help` for every command. +## Local history + +Successful `send` commands save the prompt locally. `thread` and `chat` save the +messages returned by the gateway, including replies, without the terminal display's +400-character truncation. History is plain-text JSONL at +`~/.grok-bot-cli/history.jsonl`: one message observation per line, with the target +ID/name, role, text, command, and recording time. Message IDs, message timestamps, +and the requested thread root are included when available. + +```bash +gbot send Researcher "Investigate the startup timeout" +gbot thread Researcher +gbot history Researcher --search timeout +gbot history --search timeout --limit 100 --json +grep -in 'timeout' "$(gbot history --path)" +``` + +`history` works offline, without app credentials. It shows the last 40 matching +records in recording order; `--limit` changes that count. The optional target +matches an exact ID or a case-insensitive full name. `--search` is a +case-insensitive literal text search. `--json` returns an array with full text. +JSONL escapes embedded newlines, so multiline prompts remain one searchable line. + +Use `--history-dir DIR` or `GROK_BOT_HISTORY_DIR` to choose another directory (the +flag takes precedence). New directories are created with mode `0700` and new +files with mode `0600`. Existing directory permissions are left unchanged. +History contains the conversation text, including any sensitive information you +put in messages. Gateway credentials and raw response metadata are not recorded. +Disable recording for a command with `--no-history`, or for all commands with +`GROK_BOT_HISTORY=off` (`false` and `0` also work): + +```bash +gbot --no-history send Researcher "Do not record this prompt locally" +export GROK_BOT_HISTORY=off +``` + +The opt-out affects local recording only; the gateway still receives the command, +and existing local history remains readable. To remove local history, delete the +file reported by `gbot history --path`. There is no automatic expiry. + +This is a record of CLI observations, not a background sync or a conversation +resume mechanism. Replies are captured when you run `thread`/`chat`; the default +fetch is the latest 40 messages (`thread --limit N` requests more). Repeated fetches +append repeated observations, and a sent prompt can appear again when fetched. +Messages never fetched by this CLI are not backed up. Local write failures warn +on stderr without turning a successful gateway operation into a failed command. + ## License MIT diff --git a/src/cli.js b/src/cli.js index 58fe5be..fa4306d 100755 --- a/src/cli.js +++ b/src/cli.js @@ -3,6 +3,8 @@ import { AVATAR_COLORS, AVATAR_SHAPES, MAX_GROUP_MEMBERS, StoreError, defaultCan import { hasGatewayAuth } from "./gateway.js"; import { openBackend } from "./commands.js"; import { inspectGrokBotGatewaySession } from "./app-session.js"; +import { entryText, transcriptEntries } from "./transcript.js"; +import { historyPath, readHistory, saveHistory } from "./history.js"; function print(value) { if (typeof value === "string") process.stdout.write(value + "\n"); @@ -47,6 +49,8 @@ function usage() { " send ", " thread [--limit N] [--root MESSAGE_ID]", " chat alias for thread", + " history [bot-or-group] [--search TEXT] [--limit N] (offline)", + " history --path print the local JSONL file path", "", "Max group members: " + MAX_GROUP_MEMBERS, "--description / --instructions is the UI Instructions field (same key).", @@ -55,6 +59,9 @@ function usage() { "Flags: --gateway --files --dir DIR --json", "Auth: GROK_BOT_GATEWAY_URL + GROK_BOT_GATEWAY_TOKEN, or the Grok Bot app session, or CURSOR_ACCESS_TOKEN", "File fallback: GROK_BOT_AGENTS_DIR", + "History: ~/.grok-bot-cli/history.jsonl (plain text; saved after send/thread/chat)", + " --history-dir DIR / GROK_BOT_HISTORY_DIR to relocate", + " --no-history / GROK_BOT_HISTORY=off to disable recording", ].join("\n"); } @@ -184,27 +191,9 @@ function formatRecord(rec, all) { return kind + " " + rec.name + title + "\n " + rec.id + desc + avatar + settingsLine + extra; } -function entryText(e) { - if (!e || typeof e !== "object") return ""; - const direct = e.text || e.prompt || e.message || e.preview; - if (typeof direct === "string" && direct) return direct; - const content = e.content; - if (typeof content === "string") return content; - if (Array.isArray(content)) { - return content.map((part) => { - if (typeof part === "string") return part; - if (part && typeof part === "object") return part.text || part.content || ""; - return ""; - }).filter(Boolean).join("\n"); - } - if (content && typeof content === "object") return content.text || JSON.stringify(content); - return ""; -} - function formatTranscript(out) { const rec = out.target; - const payload = out.transcript || out.thread || {}; - const entries = payload.entries || payload.messages || payload.items || (Array.isArray(payload) ? payload : []); + const entries = transcriptEntries(out); const header = (rec.isGroup ? "group" : "bot") + " " + rec.name + "\n " + rec.id; if (!Array.isArray(entries) || entries.length === 0) { return header + "\n (no messages)"; @@ -226,6 +215,8 @@ async function main(argv) { return; } + const noHistory = hasFlag(args, "--no-history"); + const historyDir = takeFlag(args, "--history-dir"); const json = hasFlag(args, "--json"); const gateway = hasFlag(args, "--gateway"); const filesMode = hasFlag(args, "--files"); @@ -238,6 +229,28 @@ async function main(argv) { return; } + if (cmd === "history") { + const options = args.slice(1); + const showPath = hasFlag(options, "--path"); + const search = takeFlag(options, "--search"); + const limitRaw = takeFlag(options, "--limit"); + const limit = limitRaw === undefined ? 40 : Number(limitRaw); + if (!Number.isSafeInteger(limit) || limit < 1) throw new StoreError("--limit must be a positive integer"); + if (options.length > 1 || options[0]?.startsWith("-") || (showPath && (options.length || search !== undefined || limitRaw !== undefined))) { + throw new StoreError("gbot history [bot-or-group] [--search TEXT] [--limit N], or history --path"); + } + const path = historyPath(historyDir); + if (showPath) print(json ? { path } : path); + else { + const rows = await readHistory(path, { ref: options[0], search, limit }); + if (json) print(rows); + else print(rows.length ? rows.map((row) => + "[" + row.recordedAt + "] " + row.target.name + " (" + row.target.id + ") [" + row.role + "] " + row.text + ).join("\n") : "No local history."); + } + return; + } + if (cmd === "doctor") { const candidates = defaultCandidateRoots(); const found = candidates.filter(looksLikeAgentsRoot); @@ -379,6 +392,7 @@ async function main(argv) { const message = rest.join(" ").trim(); if (!ref || !message) throw new StoreError("gbot send "); const out = await backend.send(ref, message); + saveHistory(out, { dir: historyDir, disabled: noHistory, event: "send", prompt: message }); if (json) print({ id: out.target.id, name: out.target.name, kind: out.target.isGroup ? "group" : "bot", result: out.result }); else print("Sent to " + (out.target.isGroup ? "group" : "bot") + " " + out.target.name + " (" + out.target.id + ")"); return; @@ -391,6 +405,7 @@ async function main(argv) { const rootId = takeFlag(rest, "--root"); const limit = limitRaw ? Number(limitRaw) : 40; const out = rootId ? await backend.thread(ref, rootId) : await backend.transcript(ref, limit); + saveHistory(out, { dir: historyDir, disabled: noHistory, event: cmd, rootId }); if (json) print(out); else print(formatTranscript(out)); return; diff --git a/src/history.js b/src/history.js new file mode 100644 index 0000000..cf83ac2 --- /dev/null +++ b/src/history.js @@ -0,0 +1,81 @@ +import { appendFileSync, closeSync, constants, createReadStream, fstatSync, mkdirSync, openSync, readSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { entryText, transcriptEntries } from "./transcript.js"; + +export function historyPath(dir) { + return join(resolve(dir || process.env.GROK_BOT_HISTORY_DIR || join(homedir(), ".grok-bot-cli")), "history.jsonl"); +} + +// Keep only conversation fields, never gateway responses, session credentials or bot instructions. +export function saveHistory(out, { dir, disabled, event, prompt, rootId } = {}) { + if (disabled || /^(off|false|0)$/i.test(process.env.GROK_BOT_HISTORY || "")) return; + try { + const recordedAt = new Date().toISOString(); + const target = { id: out.target.id, name: out.target.name, kind: out.target.isGroup ? "group" : "bot" }; + const entries = event === "send" ? [{ role: "user", text: prompt }] : transcriptEntries(out); + const rows = entries.map((entry) => ({ + version: 1, + recordedAt, + event, + target, + ...(rootId ? { rootId } : {}), + role: String(entry.role || entry.kind || entry.sender || entry.type || "msg"), + ...(entry.id || entry.messageId ? { messageId: String(entry.id || entry.messageId) } : {}), + ...(entry.timestamp || entry.createdAt ? { timestamp: String(entry.timestamp || entry.createdAt) } : {}), + text: entryText(entry), + })); + if (!rows.length) return; + const path = historyPath(dir); + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const fd = openSync(path, constants.O_CREAT | constants.O_APPEND | constants.O_RDWR | constants.O_NOFOLLOW, 0o600); + try { + // Separate a previous interrupted append from the next complete record. + const size = fstatSync(fd).size; + const last = Buffer.alloc(1); + if (size) readSync(fd, last, 0, 1, size - 1); + const prefix = size && last[0] !== 10 ? "\n" : ""; + appendFileSync(fd, prefix + rows.map((row) => JSON.stringify(row)).join("\n") + "\n"); + } finally { + closeSync(fd); + } + } catch { + // A successful remote send must not look failed and invite an accidental resend. + process.stderr.write("Warning: could not save local history. Check the history directory and permissions.\n"); + } +} + +export async function readHistory(path, { ref, search, limit = 40 } = {}) { + const rows = []; + let malformed = 0; + const input = createReadStream(path, { encoding: "utf8" }); + const lines = createInterface({ input, crlfDelay: Infinity }); + try { + for await (const line of lines) { + if (!line.trim()) continue; + let row; + try { + row = JSON.parse(line); + if (row?.version !== 1 || typeof row.text !== "string" || typeof row.role !== "string" || + typeof row.recordedAt !== "string" || typeof row.target?.id !== "string" || typeof row.target?.name !== "string") { + throw new Error("Invalid history record"); + } + } catch { + malformed++; + continue; + } + if (ref && row.target.id !== ref && row.target.name.toLowerCase() !== ref.toLowerCase()) continue; + if (search !== undefined && !row.text.toLowerCase().includes(search.toLowerCase())) continue; + rows.push(row); + if (rows.length > limit) rows.shift(); + } + } catch (err) { + if (err.code !== "ENOENT") throw err; + } finally { + lines.close(); + input.destroy(); + } + if (malformed) process.stderr.write("Warning: skipped " + malformed + " malformed local history record(s).\n"); + return rows; +} diff --git a/src/transcript.js b/src/transcript.js new file mode 100644 index 0000000..588bc2d --- /dev/null +++ b/src/transcript.js @@ -0,0 +1,22 @@ +export function entryText(e) { + if (!e || typeof e !== "object") return ""; + const direct = e.text || e.prompt || e.message; + if (typeof direct === "string" && direct) return direct; + const content = e.content; + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content.map((part) => { + if (typeof part === "string") return part; + if (part && typeof part === "object") return part.text || part.content || ""; + return ""; + }).filter(Boolean).join("\n"); + } + if (content && typeof content === "object") return content.text || JSON.stringify(content); + return typeof e.preview === "string" ? e.preview : ""; +} + +export function transcriptEntries(out) { + const payload = out.transcript || out.thread || {}; + const entries = Array.isArray(payload) ? payload : payload.entries || payload.messages || payload.items || []; + return Array.isArray(entries) ? entries.filter((entry) => entry && typeof entry === "object") : []; +} diff --git a/test/history.test.js b/test/history.test.js new file mode 100644 index 0000000..3327f9e --- /dev/null +++ b/test/history.test.js @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { once } from "node:events"; +import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); +const CLI = fileURLToPath(new URL("../src/cli.js", import.meta.url)); +const target = { id: "bot-1", name: "Researcher", description: "PRIVATE_INSTRUCTIONS" }; +const group = { id: "group-1", name: "Launch", isGroup: true, memberIds: [target.id] }; +const reply = "A long reply: " + "x".repeat(500) + "\nneedle at the end 🔧"; + +async function fixture(t) { + const home = mkdtempSync(join(tmpdir(), "gbot-history-test-")); + t.after(() => rmSync(home, { recursive: true, force: true })); + const env = { ...process.env, HOME: home, USERPROFILE: home }; + for (const key of Object.keys(env)) { + if (/^(GROK_BOT_|CURSOR_|SAND_)/.test(key)) delete env[key]; + } + const calls = []; + const state = { failSend: false, payload: { entries: [ + { id: "reply-1", role: "assistant", preview: "A long reply: ...", content: [{ type: "text", text: reply }], createdAt: "2026-09-05T10:00:00Z", token: "PRIVATE_ENTRY_METADATA" }, + ], gatewayToken: "PRIVATE_RESPONSE_METADATA" } }; + const server = createServer(async (req, res) => { + let text = ""; + for await (const chunk of req) text += chunk; + const body = JSON.parse(text); + calls.push({ method: req.url, body }); + res.setHeader("Content-Type", "application/json"); + if (req.url === "/api/listAgents") res.end(JSON.stringify({ agents: [target, group] })); + else if (req.url === "/api/sendPrompt") { + res.statusCode = state.failSend ? 500 : 200; + res.end(JSON.stringify(state.failSend ? { error: "rejected" } : { ok: true, gatewayToken: "PRIVATE_SEND_METADATA" })); + } else res.end(JSON.stringify(state.payload)); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + t.after(() => new Promise((resolve) => server.close(resolve))); + const onlineEnv = { ...env, GROK_BOT_GATEWAY_URL: "http://127.0.0.1:" + server.address().port, GROK_BOT_GATEWAY_TOKEN: "PRIVATE_AUTH" }; + const run = (args, extra = {}, online = true) => exec(process.execPath, [CLI, ...args], { env: { ...(online ? onlineEnv : env), ...extra } }); + const path = join(home, ".grok-bot-cli", "history.jsonl"); + const rows = () => readFileSync(path, "utf8").trim().split("\n").map(JSON.parse); + return { home, path, rows, run, calls, state }; +} + +test("send persists a full multiline prompt across processes, searchable offline and with grep", async (t) => { + const f = await fixture(t); + const prompt = 'Investigate timeout\nwith "quotes" and Unicode 🔧'; + const sent = await f.run(["send", "Researcher", prompt, "--json"]); + assert.equal(JSON.parse(sent.stdout).result.ok, true); + assert.equal(sent.stderr, ""); + const [row] = f.rows(); + assert.equal(row.text, prompt); + assert.equal(row.role, "user"); + assert.equal(row.event, "send"); + assert.deepEqual(row.target, { id: target.id, name: target.name, kind: "bot" }); + assert.ok(Number.isFinite(Date.parse(row.recordedAt))); + assert.equal(readFileSync(f.path, "utf8").trim().split("\n").length, 1); + assert.doesNotMatch(readFileSync(f.path, "utf8"), /PRIVATE_/); + const count = f.calls.length; + const history = await f.run(["history", "researcher", "--search", "TIMEOUT", "--json"], {}, false); + assert.deepEqual(JSON.parse(history.stdout), [row]); + assert.equal(f.calls.length, count); + const grep = await exec("grep", ["-n", "timeout", f.path]); + assert.match(grep.stdout, /^1:/); + if (process.platform !== "win32") { + assert.equal(statSync(f.path).mode & 0o777, 0o600); + assert.equal(statSync(join(f.home, ".grok-bot-cli")).mode & 0o777, 0o700); + } +}); + +test("thread and chat preserve full replies, group and root metadata, and repeated observations", async (t) => { + const f = await fixture(t); + await f.run(["thread", "Researcher", "--limit", "80"]); + assert.deepEqual(f.calls.at(-1), { method: "/api/getAgentTranscriptTail", body: { id: target.id, limit: 80 } }); + await f.run(["chat", "Launch", "--root", "root-1", "--json"]); + assert.deepEqual(f.calls.at(-1), { method: "/api/getAgentThread", body: { id: group.id, rootId: "root-1" } }); + const rows = f.rows(); + assert.equal(rows.length, 2); + assert.equal(rows[0].text, reply); + assert.equal(rows[0].messageId, "reply-1"); + assert.equal(rows[0].timestamp, "2026-09-05T10:00:00Z"); + assert.equal(rows[1].rootId, "root-1"); + assert.equal(rows[1].target.kind, "group"); + assert.doesNotMatch(readFileSync(f.path, "utf8"), /PRIVATE_/); + const found = await f.run(["history", "group-1", "--search", "needle", "--json"], {}, false); + assert.deepEqual(JSON.parse(found.stdout), [rows[1]]); + await f.run(["thread", "Researcher"]); + assert.equal(f.rows().length, 3); +}); + +test("supports all transcript envelopes and text fields already displayed by the CLI", async (t) => { + const f = await fixture(t); + for (const [key, entry] of [ + ["messages", { messageId: "m1", sender: "assistant", message: "message text" }], + ["items", { kind: "user", prompt: "prompt text" }], + [null, { type: "assistant", content: "content text" }], + ]) { + f.state.payload = key ? { [key]: [entry] } : [entry]; + await f.run(["thread", "Researcher"]); + } + assert.deepEqual(f.rows().map((r) => r.text), ["message text", "prompt text", "content text"]); +}); + +test("recording opt-outs do not create storage or disable access to existing history", async (t) => { + const f = await fixture(t); + await f.run(["--no-history", "send", "Researcher", "private prompt"]); + for (const value of ["off", "FALSE", "0"]) { + await f.run(["thread", "Researcher"], { GROK_BOT_HISTORY: value }); + } + assert.equal(existsSync(f.path), false); + await f.run(["send", "Researcher", "saved"]); + await f.run(["--no-history", "thread", "Researcher"]); + const found = await f.run(["history", "--json"], { GROK_BOT_HISTORY: "off" }, false); + assert.equal(JSON.parse(found.stdout).length, 1); + assert.equal(f.rows().length, 1); +}); + +test("history path is offline and side-effect free; flag directory overrides environment", async (t) => { + const f = await fixture(t); + const defaultPath = await f.run(["history", "--path"], {}, false); + assert.equal(defaultPath.stdout.trim(), f.path); + assert.equal(existsSync(f.path), false); + const dir = join(f.home, "custom history"); + const extra = { GROK_BOT_HISTORY_DIR: join(f.home, "env-history") }; + await f.run(["send", "Researcher", "env"], extra); + await f.run(["--history-dir", dir, "send", "Researcher", "flag"], extra); + const found = await f.run(["--history-dir", dir, "history", "--json"], extra, false); + assert.equal(JSON.parse(found.stdout)[0].text, "flag"); + const envHistory = await f.run(["history", "--json"], extra, false); + assert.equal(JSON.parse(envHistory.stdout)[0].text, "env"); + const path = await f.run(["--history-dir", dir, "history", "--path", "--json"], extra, false); + assert.deepEqual(JSON.parse(path.stdout), { path: join(dir, "history.jsonl") }); +}); + +test("offline history handles missing files, filters before limiting, and validates options", async (t) => { + const f = await fixture(t); + assert.equal((await f.run(["history"], {}, false)).stdout.trim(), "No local history."); + assert.equal(existsSync(f.path), false); + for (const text of ["match one", "match two", "unrelated"]) await f.run(["send", "Researcher", text]); + const result = await f.run(["history", "--search", "match", "--limit", "1", "--json"], {}, false); + assert.equal(JSON.parse(result.stdout)[0].text, "match two"); + assert.deepEqual(JSON.parse((await f.run(["history", "unknown", "--json"], {}, false)).stdout), []); + for (const args of [["--limit", "0"], ["--limit", "1.5"], ["--limit", "NaN"], ["--path", "Researcher"], ["--unknown"]]) { + await assert.rejects(f.run(["history", ...args], {}, false), (err) => err.code === 1); + } +}); + +test("failed sends and empty threads leave no history; disk failure does not fail a successful send", async (t) => { + const f = await fixture(t); + f.state.failSend = true; + await assert.rejects(f.run(["send", "Researcher", "rejected"])); + assert.equal(existsSync(f.path), false); + f.state.payload = { entries: [] }; + await f.run(["thread", "Researcher"]); + assert.equal(existsSync(f.path), false); + f.state.failSend = false; + const blocked = join(f.home, "not-a-directory"); + writeFileSync(blocked, "occupied"); + const result = await f.run(["--history-dir", blocked, "send", "Researcher", "sent once", "--json"]); + assert.equal(JSON.parse(result.stdout).result.ok, true); + assert.match(result.stderr, /Warning: could not save local history/); + assert.equal(f.calls.filter((c) => c.method === "/api/sendPrompt" && c.body.prompt === "sent once").length, 1); +}); + +test("history skips malformed records and separates interrupted writes on the next append", async (t) => { + const f = await fixture(t); + await f.run(["send", "Researcher", "first"]); + appendFileSync(f.path, 'null\n{"version":1}\n{"text":"interrupted'); + await f.run(["send", "Researcher", "second"]); + const found = await f.run(["history", "--json"], {}, false); + assert.deepEqual(JSON.parse(found.stdout).map((r) => r.text), ["first", "second"]); + assert.match(found.stderr, /skipped 3 malformed/); +}); + +test("concurrent CLI processes append complete records", async (t) => { + const f = await fixture(t); + await Promise.all(Array.from({ length: 8 }, (_, i) => f.run(["send", "Researcher", "parallel " + i]))); + assert.equal(f.rows().length, 8); + assert.equal(new Set(f.rows().map((r) => r.text)).size, 8); +}); + +test("refuses to append through a history-file symlink", { skip: process.platform === "win32" }, async (t) => { + const f = await fixture(t); + const destination = join(f.home, "unrelated-file"); + writeFileSync(destination, "keep me"); + symlinkSync(destination, join(f.home, "history.jsonl")); + const result = await f.run(["--history-dir", f.home, "send", "Researcher", "hello"]); + assert.match(result.stderr, /could not save local history/); + assert.equal(readFileSync(destination, "utf8"), "keep me"); +});