diff --git a/AGENTS.md b/AGENTS.md index ebb777be..e07b723b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,7 +180,7 @@ Each file in `aai_cli/commands/` is a Typer sub-app (`transcribe`, `stream`, `ag - **`code_gen/`** — backs `--show-code` on `transcribe`/`stream`/`agent`: builds a ready-to-run Python SDK script from exactly the flags passed (no API key needed; generated code reads `ASSEMBLYAI_API_KEY`). - **`auth/`** — browser-assisted `assembly login` via AMS + **Stytch B2B OAuth discovery** (`discovery.py`, `flow.py`, `loopback.py`, `ams.py`). Not Stytch Connected Apps. - **`init/`** — scaffolds a self-contained FastAPI + HTML starter (`audio-transcription`/`live-captions`/`voice-agent` templates), optionally installs deps and opens the browser; writes the key to a git-ignored `.env`. -- **`telemetry.py`** — anonymous, opt-out usage telemetry (Supabase-CLI model): `context.run_command` wraps each command body in `telemetry.track(ctx.command_path)`, which dispatches one allow-listed event (command path, outcome/exit code, duration, version/OS — never args, paths, or account data) to the Datadog logs intake via a **detached flusher subprocess** (the hidden `assembly telemetry flush`), so commands never wait on telemetry. `SHIPPED_CLIENT_TOKEN` is a committed write-only Datadog *client* token (`pub…`, embeddable by design — never an API key; `AAI_TELEMETRY_CLIENT_TOKEN` overrides). The test suite blanks it via an autouse conftest fixture so no test ever spawns a real flusher. Opt-out: `AAI_TELEMETRY_DISABLED=1` / `DO_NOT_TRACK=1` / `assembly telemetry disable` (persisted as `telemetry_enabled` in config.toml, alongside the random `device_id`). Send-side failures are swallowed (`OSError`/`CLIError`) — telemetry must never break a command. +- **`telemetry.py`** — anonymous, opt-out usage telemetry (Supabase-CLI model): `context.run_command` wraps each command body in `telemetry.track(ctx.command_path)`, which dispatches one allow-listed event (command path, outcome/exit code, duration, version/OS, and on failure the error message capped at 500 chars — never args or account data) to the Datadog logs intake via a **detached flusher subprocess** (the hidden `assembly telemetry flush`), so commands never wait on telemetry. `SHIPPED_CLIENT_TOKEN` is a committed write-only Datadog *client* token (`pub…`, embeddable by design — never an API key; `AAI_TELEMETRY_CLIENT_TOKEN` overrides). The test suite blanks it via an autouse conftest fixture so no test ever spawns a real flusher. Opt-out: `AAI_TELEMETRY_DISABLED=1` / `DO_NOT_TRACK=1` / `assembly telemetry disable` (persisted as `telemetry_enabled` in config.toml, alongside the random `device_id`). Send-side failures are swallowed (`OSError`/`CLIError`) — telemetry must never break a command. - **`commands/setup.py`** — `assembly setup install/status/remove` wires a coding agent up to AssemblyAI by installing three artifacts: the `assemblyai-docs` docs MCP (via `claude mcp add`), the AssemblyAI skill (via `npx skills add`), and the bundled `aai-cli` skill (copied out of the wheel, no network). Missing `claude`/`npx` is reported and skipped, not an error. The presence probes (docs MCP registered, skills on disk) live in `aai_cli/coding_agent.py` so `assembly doctor`'s coding-agent check can share them — command modules are import-linter-independent, so neither command may import the other. ## Conventions diff --git a/aai_cli/telemetry.py b/aai_cli/telemetry.py index 5b1d6f63..85360175 100644 --- a/aai_cli/telemetry.py +++ b/aai_cli/telemetry.py @@ -1,7 +1,8 @@ """Anonymous usage telemetry, modeled on the Supabase CLI's design. One allow-listed event per command run (command path, outcome, duration — never -arguments, file paths, ids, or account data) is shipped to the Datadog logs +arguments, ids, or account data; a failure also carries the error message, +capped at 500 chars) is shipped to the Datadog logs intake using a write-only *client* token (``pub…``), the credential class Datadog designs to be embedded in client apps. ``SHIPPED_CLIENT_TOKEN`` carries it (it is public by design — never put an API key there); @@ -47,6 +48,10 @@ _SEND_TIMEOUT_SECONDS = 5.0 +# Cap on the error message shipped with a failure event: enough for any CLI +# error line, while bounding the payload if an upstream message embeds a body. +_ERROR_MESSAGE_MAX_CHARS = 500 + def client_token() -> str: """The write-only intake token: env override first, then the shipped one.""" @@ -75,7 +80,12 @@ def is_enabled() -> bool: def build_event( - command: str, *, outcome: str, exit_code: int, duration_ms: int + command: str, + *, + outcome: str, + exit_code: int, + duration_ms: int, + error_message: str | None = None, ) -> dict[str, object]: """One invocation event, shaped for the Datadog logs intake. @@ -85,10 +95,11 @@ def build_event( hostname ever rides along. A failure additionally sets ``status: error`` and the reserved - ``error.kind`` so the event feeds Datadog **Error Tracking** (issue - grouping), not just log search. ``error.kind`` reuses the anonymous - ``outcome`` (the ``CLIError.error_type``) — the error *message* and stack - trace are deliberately omitted, so no free text or PII ever rides along. + ``error.kind``/``error.message`` so the event feeds Datadog **Error + Tracking** (issue grouping), not just log search. ``error.kind`` reuses the + anonymous ``outcome`` (the ``CLIError.error_type``); ``error.message`` is + the one-line message the user saw (capped at ``_ERROR_MESSAGE_MAX_CHARS``). + Stack traces are still deliberately omitted. """ succeeded = outcome == "success" event: dict[str, object] = { @@ -108,7 +119,10 @@ def build_event( "device_id": config.get_device_id(), } if not succeeded: - event["error"] = {"kind": outcome} + error: dict[str, object] = {"kind": outcome} + if error_message: + error["message"] = error_message[:_ERROR_MESSAGE_MAX_CHARS] + event["error"] = error return event @@ -154,11 +168,24 @@ def flush_payload(raw: str) -> None: ) -def _safe_dispatch(command: str, started: float, *, outcome: str, exit_code: int) -> None: +def _safe_dispatch( + command: str, + started: float, + *, + outcome: str, + exit_code: int, + error_message: str | None = None, +) -> None: duration_ms = int((time.monotonic() - started) * 1000) try: dispatch( - build_event(command, outcome=outcome, exit_code=exit_code, duration_ms=duration_ms) + build_event( + command, + outcome=outcome, + exit_code=exit_code, + duration_ms=duration_ms, + error_message=error_message, + ) ) except (OSError, CLIError): # Best-effort by contract: a config/spawn failure while *recording* a command @@ -182,14 +209,22 @@ def track(command: str) -> Generator[None]: try: yield except CLIError as err: - _safe_dispatch(command, started, outcome=err.error_type, exit_code=err.exit_code) + _safe_dispatch( + command, + started, + outcome=err.error_type, + exit_code=err.exit_code, + error_message=err.message, + ) raise except typer.Exit as exc: code = exc.exit_code outcome = "success" if code == 0 else "error" _safe_dispatch(command, started, outcome=outcome, exit_code=code) raise - except BaseException: - _safe_dispatch(command, started, outcome="internal_error", exit_code=1) + except BaseException as exc: + _safe_dispatch( + command, started, outcome="internal_error", exit_code=1, error_message=str(exc) + ) raise _safe_dispatch(command, started, outcome="success", exit_code=0) diff --git a/docs/datadog/cli-usage-dashboard.json b/docs/datadog/cli-usage-dashboard.json index 4056057a..cabfb4b9 100644 --- a/docs/datadog/cli-usage-dashboard.json +++ b/docs/datadog/cli-usage-dashboard.json @@ -9,7 +9,7 @@ { "definition": { "type": "note", - "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`. Failures ship `status:error` + the reserved `@error.kind` (= the anonymous error type, **no message or stack trace**), so they also feed **Error Tracking** — the error tiles below filter on `status:error`. Events take ~1–2 min to index.", + "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`, `@error.message`. Failures ship `status:error` + the reserved `@error.kind` (= the anonymous error type) and `@error.message` (the one-line message the user saw, capped at 500 chars — **no stack trace**), so they also feed **Error Tracking** — the error tiles below filter on `status:error`. Events take ~1–2 min to index.", "background_color": "purple", "font_size": "14", "text_align": "left", @@ -802,6 +802,105 @@ "width": 12, "height": 5 } + }, + { + "definition": { + "title": "Top error messages", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "errors", + "data_source": "logs", + "search": { + "query": "source:aai-cli status:error" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@error.message", + "limit": 15, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "errors" + } + ] + } + ] + }, + "layout": { + "x": 0, + "y": 13, + "width": 5, + "height": 4 + } + }, + { + "definition": { + "title": "Recent errors", + "type": "list_stream", + "requests": [ + { + "response_format": "event_list", + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "timestamp", + "width": "auto" + }, + { + "field": "@command", + "width": "auto" + }, + { + "field": "@error.kind", + "width": "auto" + }, + { + "field": "@error.message", + "width": "full" + }, + { + "field": "@cli_version", + "width": "auto" + } + ], + "query": { + "data_source": "logs_stream", + "query_string": "source:aai-cli status:error", + "indexes": [], + "sort": { + "column": "timestamp", + "order": "desc" + } + } + } + ] + }, + "layout": { + "x": 5, + "y": 13, + "width": 7, + "height": 4 + } } ] }, @@ -809,7 +908,7 @@ "x": 0, "y": 12, "width": 12, - "height": 14 + "height": 18 } }, { @@ -959,7 +1058,7 @@ }, "layout": { "x": 0, - "y": 26, + "y": 30, "width": 12, "height": 5 } @@ -1170,7 +1269,7 @@ }, "layout": { "x": 0, - "y": 31, + "y": 35, "width": 12, "height": 5 } diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index f27c4d23..0bbcfbfb 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -142,11 +142,45 @@ def test_build_event_failure_feeds_error_tracking(monkeypatch): assert event["outcome"] == "api_error" assert event["exit_code"] == 1 # status:error + the reserved error.kind are what promote it into Error Tracking; - # error.kind mirrors the anonymous outcome, and no message/stack ever rides along. + # error.kind mirrors the anonymous outcome. No message was provided, so none rides along. assert event["status"] == "error" assert event["error"] == {"kind": "api_error"} +def test_build_event_failure_carries_error_message(): + event = telemetry.build_event( + "aai transcribe", + outcome="api_error", + exit_code=1, + duration_ms=5, + error_message="Audio file not found: clip.wav", + ) + # error.message is the reserved attribute Error Tracking groups/displays on. + assert event["error"] == {"kind": "api_error", "message": "Audio file not found: clip.wav"} + + +def test_build_event_error_message_capped_at_500_chars(): + # Exactly at the cap: untouched. + exact = "y" * 500 + event = telemetry.build_event( + "aai stream", outcome="api_error", exit_code=1, duration_ms=5, error_message=exact + ) + assert event["error"] == {"kind": "api_error", "message": exact} + # One over: truncated to exactly the cap. + event = telemetry.build_event( + "aai stream", outcome="api_error", exit_code=1, duration_ms=5, error_message="x" * 501 + ) + assert event["error"] == {"kind": "api_error", "message": "x" * 500} + + +def test_build_event_blank_error_message_is_omitted(): + # str(exc) can be "" (e.g. RuntimeError()); don't ship an empty message field. + event = telemetry.build_event( + "aai stream", outcome="internal_error", exit_code=1, duration_ms=5, error_message="" + ) + assert event["error"] == {"kind": "internal_error"} + + # --- dispatch (detached flusher handoff) ------------------------------------ @@ -279,6 +313,8 @@ def test_track_cli_error_keeps_error_type_and_reraises(events): (event,) = events assert event["outcome"] == "usage_error" assert event["exit_code"] == 2 + # The clean CLIError message the user saw rides along for Error Tracking. + assert event["error"] == {"kind": "usage_error", "message": "bad flag"} @pytest.mark.parametrize( @@ -290,6 +326,8 @@ def test_track_typer_exit_maps_code(events, code, outcome): (event,) = events assert event["outcome"] == outcome assert event["exit_code"] == code + # A bare typer.Exit carries no message, so the failure event has only the kind. + assert event.get("error") == ({"kind": "error"} if code else None) def test_track_unexpected_exception_is_internal_error(events): @@ -298,6 +336,7 @@ def test_track_unexpected_exception_is_internal_error(events): (event,) = events assert event["outcome"] == "internal_error" assert event["exit_code"] == 1 + assert event["error"] == {"kind": "internal_error", "message": "boom"} @pytest.mark.parametrize("exc", [OSError("spawn failed"), CLIError("corrupt config")])