diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index c6994f74a2..b52bc64a2a 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -23,6 +23,7 @@ runs helper features around provider requests. | `corsAllowOrigins?` | `string[]` | `[]` | Additional exact origins allowed by CORS. Loopback origins are always allowed. Authority-based browser extension origins such as `chrome-extension://` are supported; `*` is not a wildcard. Firefox and Safari regenerate the extension UUID (per install / per browser launch), so update the entry when the origin changes. | | `apiKeys?` | `OcxApiKey[]` | `[]` | Generated `ocx_…` credentials accepted by management and data-plane auth on non-loopback binds. Dashboard-managed. | | `storageCleanupPolicy?` | `StorageCleanupPolicy` | disabled | Opt-in archived-session cleanup policy. Never enabled implicitly. | +| `usageLedgerRetention?` | `{ enabled?: boolean; maxBytes?: number }` | disabled | Opt-in cap for `$OPENCODEX_HOME/usage.jsonl`. Never enabled implicitly. When enabled, older JSONL rows are dropped permanently so the file stays within `maxBytes` (default 512 MiB, floor 1 MiB). The derived `routing-history.sqlite` index is deleted after a rewrite and rebuilt on the next open. | | `appOwnedMemoryBudgetMb?` | `number` | `256` | Cap in MiB for evictable app-owned logs, caches, blobs, and continuation payloads. Range 64–4096; not an RSS cap. | | `codexAutoStart?` | `boolean` | `true` | Let the Codex shim run `ocx ensure` before launching Codex. False makes ensure a no-op. | | `codexShimAutoRestore?` | `boolean` | `true` | Restore an installed shim after a completed external Codex update replaces it. Environment opt-out: `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`. | @@ -171,6 +172,18 @@ either `target.reduceToBytes` or `target.removeOldestPercent`. `mode` defaults t Configure it on the Storage page or with `GET`/`PUT /api/storage/cleanup-policy`; trigger a manual run with `POST /api/storage/cleanup-policy/run`. +`usageLedgerRetention` is a separate opt-in for OpenCodex's own request ledger (`usage.jsonl`), not +Codex session archives. Default off. Enable it in `config.json`: + +```json +{ + "usageLedgerRetention": { "enabled": true, "maxBytes": 536870912 } +} +``` + +The ceiling is enforced at process start (before `/api/logs` hydration) and after each append once +the file exceeds `maxBytes`. Older rows are dropped permanently; there is no quarantine copy. + ## Quota-reset notifications (`quotaResetNotify`) Off by default. When the section is absent, no detection runs, no timer starts, and no state diff --git a/src/config.ts b/src/config.ts index 0e2a4d9ba0..4116a4a222 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1169,6 +1169,12 @@ const configSchema = z.object({ enabled: z.boolean().optional(), leadTimeMinutes: z.number().int().min(1).max(60).optional(), }).optional().catch(undefined), + // Opt-in usage.jsonl byte ceiling. A malformed hand edit disables only this + // circuit so it cannot trip the backup-and-defaults repair path. + usageLedgerRetention: z.object({ + enabled: z.boolean().optional(), + maxBytes: z.number().int().min(1024 * 1024).optional(), + }).optional().catch(undefined), // Model ids excluded from the Grok Build managed block (dashboard switches). grokExcludedModels: z.array(z.string()).optional(), // Invalid values degrade to undefined ("auto") instead of failing the whole diff --git a/src/server/index.ts b/src/server/index.ts index aedd6bf236..7d7ea6825e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -120,6 +120,7 @@ import { type RequestLogEntry, } from "./request-log"; import { sessionLaneIdFromRequest } from "./request-log-conversation"; +import { enforceUsageLedgerRetention } from "../usage/ledger-retention"; export { addFinalRequestLog, filterRequestLogs, @@ -740,6 +741,9 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server; + const enabled = o.enabled === true; + let maxBytes = base.maxBytes; + if (typeof o.maxBytes === "number" && Number.isFinite(o.maxBytes) && Math.floor(o.maxBytes) === o.maxBytes) { + maxBytes = Math.max(MIN_USAGE_LEDGER_MAX_BYTES, o.maxBytes); + } + return { enabled, maxBytes }; +} + +export function usageLedgerPath(configDir?: string): string { + const dir = (configDir ?? getConfigDir()).replace(/[\\/]+$/, ""); + return `${dir}/${USAGE_LEDGER_FILENAME}`; +} + +export function discardHistoryIndex(configDir?: string): void { + const db = historyIndexPath(configDir ?? getConfigDir()); + for (const path of [db, `${db}-wal`, `${db}-shm`]) { + try { + unlinkSync(path); + } catch { + /* absent is the success case */ + } + } +} + +/** + * Keep the newest complete JSONL rows whose bytes fit in `maxBytes`. + * No-op when the file is missing or already within the ceiling. + */ +export function compactUsageLedgerToMaxBytes( + path: string, + maxBytes: number, +): UsageLedgerCompactResult { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) { + throw new RangeError("usage ledger maxBytes must be a positive integer"); + } + if (!existsSync(path)) { + return { skipped: "missing", beforeBytes: 0, afterBytes: 0, droppedBytes: 0 }; + } + const beforeBytes = statSync(path).size; + if (beforeBytes <= maxBytes) { + return { skipped: "under_limit", beforeBytes, afterBytes: beforeBytes, droppedBytes: 0 }; + } + + const fd = openSync(path, "r"); + try { + let start = beforeBytes - maxBytes; + if (start > 0) { + const probeLen = Math.min(NEWLINE_PROBE_BYTES, beforeBytes - start); + const probe = Buffer.alloc(probeLen); + const n = readSync(fd, probe, 0, probeLen, start); + const nl = probe.subarray(0, n).indexOf(0x0a); + // Drop the possibly-partial first row. If this window has no newline, + // keep the raw tail rather than deleting the whole ledger. + if (nl >= 0) start = start + nl + 1; + } else { + start = 0; + } + + const tmp = `${path}.tmp-retention`; + const out = openSync(tmp, "w", 0o600); + try { + const buf = Buffer.alloc(COPY_CHUNK_BYTES); + let pos = start; + while (pos < beforeBytes) { + const n = readSync(fd, buf, 0, buf.length, pos); + if (n <= 0) break; + writeSync(out, buf, 0, n); + pos += n; + } + fsyncSync(out); + } finally { + closeSync(out); + } + renameSync(tmp, path); + try { chmodSync(path, 0o600); } catch { /* best-effort */ } + } finally { + closeSync(fd); + } + + const afterBytes = existsSync(path) ? statSync(path).size : 0; + return { + beforeBytes, + afterBytes, + droppedBytes: Math.max(0, beforeBytes - afterBytes), + }; +} + +let cachedPolicy: UsageLedgerRetention | null = null; + +export function resetUsageLedgerRetentionCacheForTests(): void { + cachedPolicy = null; +} + +function currentPolicy(): UsageLedgerRetention { + if (cachedPolicy) return cachedPolicy; + cachedPolicy = normalizeUsageLedgerRetention(loadConfig().usageLedgerRetention); + return cachedPolicy; +} + +/** Startup / post-append hook. Never throws to the request path. */ +export function enforceUsageLedgerRetention(configDir?: string): UsageLedgerCompactResult { + const dir = configDir ?? getConfigDir(); + const policy = currentPolicy(); + const path = usageLedgerPath(dir); + if (!policy.enabled) { + return { skipped: "disabled", beforeBytes: 0, afterBytes: 0, droppedBytes: 0 }; + } + const result = compactUsageLedgerToMaxBytes(path, policy.maxBytes); + if (!result.skipped) discardHistoryIndex(dir); + return result; +} diff --git a/src/usage/log.ts b/src/usage/log.ts index acbe8a00db..00b550d31a 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -6,6 +6,7 @@ import { enforceAppOwnedMemoryBudget } from "../lib/app-owned-memory"; import { recordOwnedConfigPath } from "../lib/config-ownership"; import { sanitizeLogMetadataString } from "../lib/redact"; import { usageDisplayTotalTokens } from "./totals"; +import { enforceUsageLedgerRetention } from "./ledger-retention"; import type { AttemptTierOutcome, OcxUsage } from "../types"; import { normalizeRouteDecisionTrace, type RouteDecisionTraceV1 } from "../routing/trace"; import { ACCOUNT_LOG_LABEL_RE, CODEX_ACCOUNT_LOG_LABEL_RE } from "../codex/account-label"; @@ -569,6 +570,7 @@ export function appendUsageEntry(entry: PersistedUsageEntry): void { const path = usageLogPath(); appendFileSync(path, `${JSON.stringify(normalizeUsageEntry(entry))}\n`, { encoding: "utf-8", mode: 0o600 }); try { chmodSync(path, 0o600); } catch { /* best-effort on platforms that ignore chmod */ } + try { enforceUsageLedgerRetention(); } catch { /* retention must not fail the request */ } } export type UsageLogRevision = { diff --git a/tests/usage-ledger-retention.test.ts b/tests/usage-ledger-retention.test.ts new file mode 100644 index 0000000000..2d6c861c08 --- /dev/null +++ b/tests/usage-ledger-retention.test.ts @@ -0,0 +1,90 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, readFileSync, writeFileSync, existsSync, statSync } from "node:fs"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { removeTreeWithRetry } from "./helpers/remove-tree"; +import { + compactUsageLedgerToMaxBytes, + discardHistoryIndex, + MIN_USAGE_LEDGER_MAX_BYTES, + normalizeUsageLedgerRetention, + usageLedgerPath, +} from "../src/usage/ledger-retention"; +import { HISTORY_DB_FILENAME } from "../src/routing/history/schema"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-ledger-ret-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) removeTreeWithRetry(testDir); +}); + +function writeLines(path: string, lines: string[]): void { + writeFileSync(path, lines.map((line) => `${line}\n`).join(""), { encoding: "utf-8", mode: 0o600 }); +} + +describe("normalizeUsageLedgerRetention", () => { + test("stays disabled unless enabled is exactly true", () => { + expect(normalizeUsageLedgerRetention(undefined).enabled).toBe(false); + expect(normalizeUsageLedgerRetention({ enabled: 1 }).enabled).toBe(false); + expect(normalizeUsageLedgerRetention({ enabled: "true" }).enabled).toBe(false); + expect(normalizeUsageLedgerRetention({ enabled: true }).enabled).toBe(true); + }); + + test("clamps maxBytes to the 1 MiB floor", () => { + expect(normalizeUsageLedgerRetention({ enabled: true, maxBytes: 12 }).maxBytes).toBe(MIN_USAGE_LEDGER_MAX_BYTES); + expect(normalizeUsageLedgerRetention({ enabled: true, maxBytes: 8 * 1024 * 1024 }).maxBytes).toBe(8 * 1024 * 1024); + }); +}); + +describe("compactUsageLedgerToMaxBytes", () => { + test("no-ops when the ledger is missing or already under the ceiling", () => { + const path = usageLedgerPath(testDir); + expect(compactUsageLedgerToMaxBytes(path, 1024).skipped).toBe("missing"); + writeLines(path, ['{"requestId":"a"}']); + const under = compactUsageLedgerToMaxBytes(path, 1024); + expect(under.skipped).toBe("under_limit"); + expect(readFileSync(path, "utf-8")).toContain('"a"'); + }); + + test("keeps the newest complete JSONL rows", () => { + const path = usageLedgerPath(testDir); + const old = `{"id":"old","pad":"${"x".repeat(200)}"}`; + const mid = `{"id":"mid","pad":"${"y".repeat(200)}"}`; + const newest = `{"id":"new","pad":"${"z".repeat(200)}"}`; + writeLines(path, [old, mid, newest]); + const before = statSync(path).size; + const twoNewest = Buffer.byteLength(`${mid}\n${newest}\n`, "utf-8"); + // Land the cut inside the oldest row so the first kept newline is the row boundary. + const result = compactUsageLedgerToMaxBytes(path, twoNewest + 10); + expect(result.skipped).toBeUndefined(); + expect(result.beforeBytes).toBe(before); + expect(result.afterBytes).toBeLessThan(before); + const kept = readFileSync(path, "utf-8"); + expect(kept).toContain('"id":"new"'); + expect(kept).not.toContain('"id":"old"'); + }); +}); + +describe("discardHistoryIndex", () => { + test("deletes the sqlite projection and wal companions", () => { + mkdirSync(testDir, { recursive: true }); + const db = join(testDir, HISTORY_DB_FILENAME); + writeFileSync(db, "sqlite"); + writeFileSync(`${db}-wal`, "wal"); + writeFileSync(`${db}-shm`, "shm"); + discardHistoryIndex(testDir); + expect(existsSync(db)).toBe(false); + expect(existsSync(`${db}-wal`)).toBe(false); + expect(existsSync(`${db}-shm`)).toBe(false); + }); +});