diff --git a/aai_cli/telemetry.py b/aai_cli/telemetry.py index 809a7d29..5b1d6f63 100644 --- a/aai_cli/telemetry.py +++ b/aai_cli/telemetry.py @@ -83,12 +83,20 @@ def build_event( arguments or options), the outcome class, and coarse machine facts. The device id is a random UUID minted locally — no account id, email, or hostname ever rides along. + + A failure additionally sets ``status: error`` and the reserved + ``error.kind`` so the event feeds Datadog **Error Tracking** (issue + grouping), not just log search. ``error.kind`` reuses the anonymous + ``outcome`` (the ``CLIError.error_type``) — the error *message* and stack + trace are deliberately omitted, so no free text or PII ever rides along. """ - return { + succeeded = outcome == "success" + event: dict[str, object] = { "ddsource": "aai-cli", "service": "aai-cli", "ddtags": f"version:{__version__}", "message": f"{command} {outcome}", + "status": "info" if succeeded else "error", "command": command, "outcome": outcome, "exit_code": exit_code, @@ -99,6 +107,9 @@ def build_event( "ci": bool(os.environ.get("CI")), "device_id": config.get_device_id(), } + if not succeeded: + event["error"] = {"kind": outcome} + return event def dispatch(event: Mapping[str, object]) -> None: diff --git a/docs/datadog/README.md b/docs/datadog/README.md index 0becd13b..2a4ed721 100644 --- a/docs/datadog/README.md +++ b/docs/datadog/README.md @@ -18,6 +18,16 @@ dashboard shows data, add these in **Logs → Configuration → Facets** (or exp `@device_id` (unique-device counts) and `@duration_ms` (latency percentiles) are the two that matter most. +## Error Tracking + +Failures ship with `status:error` and the reserved `@error.kind` attribute (set to the +anonymous error type — the same value as `@outcome`, e.g. `api_error`, `not_authenticated`), +so they feed Datadog **Error Tracking** (issue grouping, first-/last-seen, regression +detection, alerting), not just log search. `error.kind` is enough to enable tracking on +its own. The error message and stack trace are **deliberately omitted** — no free text or +PII ever leaves the machine — so issues group by error *type × command*, not by stack. +The error tiles on the dashboard filter on `status:error`. + ## Import UI: **Dashboards → New Dashboard → ⚙️ → Import dashboard JSON**, paste the file. @@ -33,9 +43,9 @@ curl -X POST "https://api.datadoghq.com/api/v1/dashboard" \ ## Notes -- "Error rate" counts any `@outcome != success` — that includes expected user errors - (e.g. `auth_failure`), not just crashes. Filter to `@outcome:internal_error` for a - crash-only view. +- "Error rate" counts any `status:error` (every non-success outcome) — that includes + expected user errors (e.g. `not_authenticated`), not just crashes. Filter to + `@error.kind:internal_error` for a crash-only view. - `@device_id` is a per-machine random UUID, so it approximates installs, not people. - Events take ~1–2 minutes to index; a fresh `source:aai-cli` search can look empty for a minute even when delivery succeeded. diff --git a/docs/datadog/cli-usage-dashboard.json b/docs/datadog/cli-usage-dashboard.json index bdc20f4b..4056057a 100644 --- a/docs/datadog/cli-usage-dashboard.json +++ b/docs/datadog/cli-usage-dashboard.json @@ -9,7 +9,7 @@ { "definition": { "type": "note", - "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`. Events take ~1–2 min to index.", + "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`. Failures ship `status:error` + the reserved `@error.kind` (= the anonymous error type, **no message or stack trace**), so they also feed **Error Tracking** — the error tiles below filter on `status:error`. Events take ~1–2 min to index.", "background_color": "purple", "font_size": "14", "text_align": "left", @@ -439,7 +439,7 @@ "name": "errors", "data_source": "logs", "search": { - "query": "source:aai-cli -@outcome:success" + "query": "source:aai-cli status:error" }, "indexes": [ "*" @@ -511,7 +511,7 @@ "name": "errors", "data_source": "logs", "search": { - "query": "source:aai-cli -@outcome:success" + "query": "source:aai-cli status:error" }, "indexes": [ "*" @@ -606,7 +606,7 @@ "name": "errors", "data_source": "logs", "search": { - "query": "source:aai-cli -@outcome:success" + "query": "source:aai-cli status:error" }, "indexes": [ "*" @@ -653,7 +653,7 @@ "name": "errors", "data_source": "logs", "search": { - "query": "source:aai-cli -@outcome:success" + "query": "source:aai-cli status:error" }, "indexes": [ "*" @@ -734,6 +734,74 @@ "width": 4, "height": 4 } + }, + { + "definition": { + "title": "All errors by frequency (type × command)", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "errors", + "data_source": "logs", + "search": { + "query": "source:aai-cli status:error" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@outcome", + "limit": 100, + "sort": { + "aggregation": "count", + "order": "desc" + } + }, + { + "facet": "@command", + "limit": 100, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "errors", + "alias": "Occurrences", + "cell_display_mode": "bar" + } + ], + "sort": { + "count": 100, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 5 + } } ] }, @@ -741,7 +809,7 @@ "x": 0, "y": 12, "width": 12, - "height": 9 + "height": 14 } }, { @@ -891,7 +959,7 @@ }, "layout": { "x": 0, - "y": 21, + "y": 26, "width": 12, "height": 5 } @@ -1102,7 +1170,7 @@ }, "layout": { "x": 0, - "y": 26, + "y": 31, "width": 12, "height": 5 } diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 2b590cd3..f27c4d23 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -114,6 +114,7 @@ def test_build_event_is_allowlisted_and_exact(monkeypatch): "service": "aai-cli", "ddtags": f"version:{__version__}", "message": "aai transcribe success", + "status": "info", "command": "aai transcribe", "outcome": "success", "exit_code": 0, @@ -126,12 +127,24 @@ def test_build_event_is_allowlisted_and_exact(monkeypatch): } -def test_build_event_marks_ci(monkeypatch): +def test_build_event_success_is_info_with_no_error_attribute(): + # A success stays an info log and carries no `error` namespace, so it never + # lands in Datadog Error Tracking. + event = telemetry.build_event("aai transcribe", outcome="success", exit_code=0, duration_ms=1) + assert event["status"] == "info" + assert "error" not in event + + +def test_build_event_failure_feeds_error_tracking(monkeypatch): monkeypatch.setenv("CI", "true") event = telemetry.build_event("aai stream", outcome="api_error", exit_code=1, duration_ms=5) assert event["ci"] is True assert event["outcome"] == "api_error" assert event["exit_code"] == 1 + # status:error + the reserved error.kind are what promote it into Error Tracking; + # error.kind mirrors the anonymous outcome, and no message/stack ever rides along. + assert event["status"] == "error" + assert event["error"] == {"kind": "api_error"} # --- dispatch (detached flusher handoff) ------------------------------------