diff --git a/docs/RUNBOOK_CLOUD_PROD.md b/docs/RUNBOOK_CLOUD_PROD.md index c83002f7..f086c46a 100644 --- a/docs/RUNBOOK_CLOUD_PROD.md +++ b/docs/RUNBOOK_CLOUD_PROD.md @@ -97,19 +97,76 @@ gcloud scheduler jobs create http lisa-autonomy-sweep --project $PROJECT \ ## 6. 监控与告警 +**先建通知渠道**——没绑渠道的告警只会安静地待在控制台里,等于没建: + +```bash +CHANNEL=$(gcloud alpha monitoring channels create --project $PROJECT \ + --display-name "lisa-ops email" --type email \ + --channel-labels email_address= --format 'value(name)') +``` + +Uptime check 打 `/health`(专用 liveness 端点:无凭据、恒 200、零依赖。 +比 `/api/auth/config` 合适——后者还要 JSON 组装,不是纯活性信号): + ```bash -# Uptime check 打公开的 auth 配置端点(无需凭据、恒 200) -gcloud monitoring uptime create lisa-cloud-auth \ +gcloud monitoring uptime create lisa-cloud-health \ --resource-type uptime-url --resource-labels host=cloud.meetlisa.ai \ - --path /api/auth/config --project $PROJECT -# 预算告警(月 $200 起步,超 50/90/100% 邮件) + --path /health --project $PROJECT +``` + +日志告警一:异常消费(meter.ts,>$10/天/用户)。实际输出是小写的 +`[billing] ⚠ anomaly: …`(console.error → stderr → textPayload,severity ERROR) +——**不是**大写 `ANOMALY`,过滤串照抄下面这条,别凭记忆改: + +```bash +cat > /tmp/policy-anomaly.json <<'EOF' +{ + "displayName": "lisa-cloud billing anomaly (>$10/day/user)", + "combiner": "OR", + "conditions": [{ + "displayName": "billing anomaly logged", + "conditionMatchedLog": { + "filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"lisa-cloud\" AND textPayload:\"[billing] ⚠ anomaly\"" + } + }], + "alertStrategy": {"notificationRateLimit": {"period": "3600s"}, "autoClose": "86400s"} +} +EOF +gcloud alpha monitoring policies create --project $PROJECT \ + --policy-from-file /tmp/policy-anomaly.json --notification-channels "$CHANNEL" +``` + +日志告警二:5xx。服务端在 Cloud Run 上输出结构化 JSON(src/log.ts, +`K_SERVICE` 触发),真正的失败才是 severity=ERROR——直接盯请求 5xx 更稳: + +```bash +cat > /tmp/policy-5xx.json <<'EOF' +{ + "displayName": "lisa-cloud 5xx", + "combiner": "OR", + "conditions": [{ + "displayName": "request finished with 5xx", + "conditionMatchedLog": { + "filter": "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"lisa-cloud\" AND log_name:\"run.googleapis.com%2Frequests\" AND httpRequest.status>=500" + } + }], + "alertStrategy": {"notificationRateLimit": {"period": "3600s"}, "autoClose": "86400s"} +} +EOF +gcloud alpha monitoring policies create --project $PROJECT \ + --policy-from-file /tmp/policy-5xx.json --notification-channels "$CHANNEL" +``` + +预算告警(月 $200 起步,超 50/90/100% 邮件): + +```bash gcloud billing budgets create --billing-account \ --display-name lisa-cloud --budget-amount 200USD \ --threshold-rule=percent=0.5 --threshold-rule=percent=0.9 --threshold-rule=percent=1.0 ``` -日志侧已内建:异常消费告警(meter.ts,>$10/天/用户打 `[billing] ANOMALY`)、 -sweep 报告行(`[sweep] scanned…`)。建一条 log-based alert 盯 `ANOMALY` 即可。 +其余日志侧已内建:sweep 报告行 `[sweep] scanned…`(severity INFO, +jsonPayload.message);Scheduler 首跳失败直接看 Cloud Scheduler 的执行历史。 ## 7. 急停开关(记住这三个) diff --git a/packaging/gcp-relay/index.mjs b/packaging/gcp-relay/index.mjs index 7e2fa271..a9f3f64f 100644 --- a/packaging/gcp-relay/index.mjs +++ b/packaging/gcp-relay/index.mjs @@ -21,6 +21,15 @@ const RELAY_TOKEN = process.env.RELAY_TOKEN || ""; const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || ""; const PORT = Number(process.env.PORT) || 8080; +// Structured logs (Cloud Logging lifts `severity` and `httpRequest` from JSON +// lines). NEVER log headers or bodies here — requests carry the relay token +// inbound and the real Anthropic key outbound, and bodies are conversation +// content. Method/path/status/latency only. +function log(severity, message, extra = {}) { + const stream = severity === "INFO" ? process.stdout : process.stderr; + stream.write(JSON.stringify({ severity, message, ...extra }) + "\n"); +} + // Headers we must not copy from client→upstream (hop-by-hop or auth we replace). const STRIP_REQ = new Set(["host", "content-length", "connection", "x-api-key", "authorization"]); // Headers we must not copy from upstream→client (let Node re-frame the body). @@ -32,6 +41,21 @@ const presentedToken = (req) => ""; const server = http.createServer(async (req, res) => { + // One request log line per proxied call (health probes stay quiet). The + // path is query-free by construction of the Anthropic API, but strip anyway. + const startedMs = Date.now(); + const pathOnly = (req.url || "/").split("?")[0]; + res.on("finish", () => { + if (pathOnly === "/" || pathOnly === "/health" || pathOnly === "/healthz") return; + log(res.statusCode >= 500 ? "ERROR" : "INFO", `${req.method} ${pathOnly} → ${res.statusCode}`, { + httpRequest: { + requestMethod: req.method, + requestUrl: pathOnly, + status: res.statusCode, + latency: `${((Date.now() - startedMs) / 1000).toFixed(3)}s`, + }, + }); + }); try { if (req.url === "/" || req.url === "/health" || req.url === "/healthz") { res.writeHead(200, { "content-type": "text/plain" }); @@ -78,6 +102,7 @@ const server = http.createServer(async (req, res) => { body: req.method === "GET" || req.method === "HEAD" ? undefined : body, }); } catch (e) { + log("ERROR", `upstream fetch failed: ${String(e && e.message || e)}`); res.writeHead(502, { "content-type": "application/json" }); res.end(JSON.stringify({ type: "error", error: { type: "relay_upstream_error", message: String(e && e.message || e) } })); return; @@ -97,6 +122,7 @@ const server = http.createServer(async (req, res) => { } res.end(); } catch (e) { + log("ERROR", `relay error: ${String(e && e.stack || e)}`); if (!res.headersSent) res.writeHead(500, { "content-type": "text/plain" }); res.end("relay error"); } @@ -110,4 +136,4 @@ function safeEqual(a, b) { server.headersTimeout = 0; // long streaming turns server.requestTimeout = 0; -server.listen(PORT, () => console.log(`anthropic relay listening on :${PORT} → ${UPSTREAM}`)); +server.listen(PORT, () => log("INFO", `anthropic relay listening on :${PORT} → ${UPSTREAM}`)); diff --git a/src/log.test.ts b/src/log.test.ts new file mode 100644 index 00000000..ab95b528 --- /dev/null +++ b/src/log.test.ts @@ -0,0 +1,23 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { formatStructured, redactEmail, redactId } from "./log.js"; + +test("formatStructured emits one-line JSON with the severity Cloud Logging lifts", () => { + const line = formatStructured("INFO", "[web] resuming session abc"); + assert.equal(line.includes("\n"), false); + const parsed = JSON.parse(line) as { severity: string; message: string }; + assert.equal(parsed.severity, "INFO"); + assert.equal(parsed.message, "[web] resuming session abc"); +}); + +test("redactId keeps a prefix+suffix for correlation, never the middle", () => { + assert.equal(redactId("550e8400-e29b-41d4-a716-446655440000"), "550e…0000"); + assert.equal(redactId("short"), "sh…"); + assert.equal(redactId(""), ""); +}); + +test("redactEmail keeps first char + domain only", () => { + assert.equal(redactEmail("alice@example.com"), "a***@example.com"); + assert.equal(redactEmail("not-an-address"), "…"); + assert.equal(redactEmail("@nouser.com"), "…"); +}); diff --git a/src/log.ts b/src/log.ts new file mode 100644 index 00000000..31fc6373 --- /dev/null +++ b/src/log.ts @@ -0,0 +1,68 @@ +/** + * Operational logging with real severities. + * + * The Mac edition historically logs everything through console.error so the + * CLI's stdout stays free for the REPL — fine locally, but on Cloud Run every + * stderr line is ingested as severity=ERROR, which makes "resuming session" + * indistinguishable from an actual failure and poisons any log-based alerting + * (the whole service reads as a wall of errors). + * + * On Cloud Run (K_SERVICE is set by the platform) — or when LISA_LOG_FORMAT=json + * is forced — each line is emitted as one-line structured JSON. Cloud Logging + * lifts the `severity` field, so INFO is INFO and alerts can key on ERROR. + * Everywhere else the text goes to stderr exactly as before, so local behavior + * is unchanged. LISA_LOG_FORMAT=text forces the legacy mode even on Cloud Run. + */ + +export type LogSeverity = "INFO" | "WARNING" | "ERROR"; + +function structuredMode(env: NodeJS.ProcessEnv = process.env): boolean { + if (env.LISA_LOG_FORMAT === "json") return true; + if (env.LISA_LOG_FORMAT === "text") return false; + return !!env.K_SERVICE; +} + +/** One structured log line (exported for tests). */ +export function formatStructured(severity: LogSeverity, message: string): string { + return JSON.stringify({ severity, message }); +} + +function emit(severity: LogSeverity, message: string): void { + if (structuredMode()) { + const stream = severity === "INFO" ? process.stdout : process.stderr; + stream.write(formatStructured(severity, message) + "\n"); + } else { + console.error(message); + } +} + +export function logInfo(message: string): void { + emit("INFO", message); +} + +export function logWarn(message: string): void { + emit("WARNING", message); +} + +export function logError(message: string): void { + emit("ERROR", message); +} + +/** + * Redaction for log lines. Logs are operational telemetry, not an audit trail — + * the full identifiers live in the billing ledger / account store. Keeping a + * short prefix+suffix is enough to correlate a log line with a ledger row + * without making the log stream itself a directory of uids / transaction ids. + */ +export function redactId(id: string): string { + if (!id) return ""; + if (id.length <= 8) return id.slice(0, 2) + "…"; + return `${id.slice(0, 4)}…${id.slice(-4)}`; +} + +/** `alice@example.com` → `a***@example.com`; a non-address becomes `…`. */ +export function redactEmail(email: string): string { + const at = email.indexOf("@"); + if (at <= 0) return "…"; + return `${email[0]}***@${email.slice(at + 1)}`; +} diff --git a/src/web/server.ts b/src/web/server.ts index e6e1569b..67fe62b6 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -6,6 +6,7 @@ import crypto from "node:crypto"; import { fileURLToPath } from "node:url"; import { listRoomMusic, toPublicTrack } from "./room-music.js"; import { runAgent } from "../agent.js"; +import { logInfo, logWarn, logError, redactId, redactEmail } from "../log.js"; import { fireHooks } from "../hooks/runner.js"; import type { HookSpec } from "../plugins/types.js"; import { saveConfigEnv } from "../env.js"; @@ -374,10 +375,10 @@ async function resumeOrCreateWebSession(model: string): Promise { if (lastId) { try { const s = await SessionStore.open(lastId); - console.error(`[web] resuming session ${lastId} (from pointer)`); + logInfo(`[web] resuming session ${lastId} (from pointer)`); return s; } catch (err) { - console.error( + logWarn( `[web] pointer ${lastId} unreadable (${(err as Error).message}) — falling back to most recent session`, ); } @@ -393,16 +394,16 @@ async function resumeOrCreateWebSession(model: string): Promise { ); if (candidate) { const s = await SessionStore.open(candidate.id); - console.error( + logInfo( `[web] resuming session ${candidate.id} (most recent in ${cwd}, ${candidate.messageCount} msgs)`, ); return s; } } catch (err) { - console.error(`[web] could not scan sessions: ${(err as Error).message}`); + logWarn(`[web] could not scan sessions: ${(err as Error).message}`); } const s = await SessionStore.create({ cwd: process.cwd(), model }); - console.error(`[web] starting fresh session ${s.id}`); + logInfo(`[web] starting fresh session ${s.id}`); return s; } @@ -439,9 +440,9 @@ export async function startWebServer(opts: WebServerOptions): Promise runBirth(uid, emit)).promise; const runtime = tenantRuntimes.peek(lisaHome()); if (runtime) runtime.prompt = undefined; // pick the newborn soul up next turn - console.error(`[accounts] soul born for ${uid}`); + logInfo(`[accounts] soul born for ${redactId(uid)}`); } catch (e) { - console.error(`[accounts] birth failed for ${uid}: ${(e as Error).message}`); + logError(`[accounts] birth failed for ${redactId(uid)}: ${(e as Error).message}`); } })(); }; @@ -681,11 +682,11 @@ export async function startWebServer(opts: WebServerOptions): Promise console.error(msg), + log: (msg) => logInfo(msg), }); // Operational push: notify subscribed phones on agent done/error/permission + // Reve idle messages. Opt-in, ntfy by default (apns is a stub). See push.ts. - const pushBridge = new PushBridge({ log: (m) => console.error(m) }); + const pushBridge = new PushBridge({ log: (m) => logInfo(m) }); // Billing anomalies reach the operator's phone through the same channel as // agent errors (B8d) — pref "error", throttled inside the bridge. setAnomalySink((text) => pushBridge.onBillingAnomaly(text)); @@ -750,9 +751,9 @@ export async function startWebServer(opts: WebServerOptions): Promise 0) { broadcast({ type: "idle_message", text: formatDigestText(digest), at: new Date().toISOString(), source: "mail" }); } - console.error(`[mail] digest ${digest.date}: ${digest.total} mail · ${digest.needsYou.length} need-you`); + logInfo(`[mail] digest ${digest.date}: ${digest.total} mail · ${digest.needsYou.length} need-you`); return digest; } catch (err) { - console.error(`[mail] digest sweep failed: ${(err as Error).message}`); + logError(`[mail] digest sweep failed: ${(err as Error).message}`); return null; } finally { mailSweepRunning = false; @@ -818,10 +819,10 @@ export async function startWebServer(opts: WebServerOptions): Promise { } } catch (err) { if (!(err instanceof BillingStateError || err instanceof AccountStoreError)) throw err; - console.error(`[billing] chat admission unavailable: ${err.message}`); + logError(`[billing] chat admission unavailable: ${err.message}`); res.writeHead(503, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "billing_state_unavailable" })); return; @@ -3959,7 +3974,7 @@ self.addEventListener('fetch', (event) => { latestReflection: chat.reflectionSummary, }); if (modelContext.omittedMessages > 0) { - console.error( + logInfo( `[context] omitted ${modelContext.omittedMessages} older message(s); ` + `sending ~${modelContext.estimatedTokens} history tokens`, ); @@ -4124,7 +4139,7 @@ self.addEventListener('fetch', (event) => { } } catch (err) { if (!(err instanceof BillingStateError || err instanceof AccountStoreError)) throw err; - console.error(`[billing] reflection admission unavailable: ${err.message}`); + logError(`[billing] reflection admission unavailable: ${err.message}`); res.writeHead(503, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "billing_state_unavailable" })); return; @@ -4151,7 +4166,7 @@ self.addEventListener('fetch', (event) => { res.end(JSON.stringify(r)); } catch (err) { if (err instanceof BillingStateError || err instanceof AccountStoreError) { - console.error(`[billing] reflection settlement unavailable: ${err.message}`); + logError(`[billing] reflection settlement unavailable: ${err.message}`); res.writeHead(503, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "billing_state_unavailable" })); } else {