Feed CLI errors into Datadog Error Tracking

aai_cli/telemetry.py

@@ -83,12 +83,20 @@ def build_event(
     arguments or options), the outcome class, and coarse machine facts. The
     device id is a random UUID minted locally — no account id, email, or
     hostname ever rides along.
+
+    A failure additionally sets ``status: error`` and the reserved
+    ``error.kind`` so the event feeds Datadog **Error Tracking** (issue
+    grouping), not just log search. ``error.kind`` reuses the anonymous
+    ``outcome`` (the ``CLIError.error_type``) — the error *message* and stack
+    trace are deliberately omitted, so no free text or PII ever rides along.
     """
-    return {
+    succeeded = outcome == "success"
+    event: dict[str, object] = {
         "ddsource": "aai-cli",
         "service": "aai-cli",
         "ddtags": f"version:{__version__}",
         "message": f"{command} {outcome}",
+        "status": "info" if succeeded else "error",
         "command": command,
         "outcome": outcome,
         "exit_code": exit_code,
@@ -99,6 +107,9 @@ def build_event(
         "ci": bool(os.environ.get("CI")),
         "device_id": config.get_device_id(),
     }
+    if not succeeded:
+        event["error"] = {"kind": outcome}
+    return event
 
 
 def dispatch(event: Mapping[str, object]) -> None:

docs/datadog/README.md

@@ -18,6 +18,16 @@ dashboard shows data, add these in **Logs → Configuration → Facets** (or exp
 `@device_id` (unique-device counts) and `@duration_ms` (latency percentiles) are the
 two that matter most.
 
+## Error Tracking
+
+Failures ship with `status:error` and the reserved `@error.kind` attribute (set to the
+anonymous error type — the same value as `@outcome`, e.g. `api_error`, `not_authenticated`),
+so they feed Datadog **Error Tracking** (issue grouping, first-/last-seen, regression
+detection, alerting), not just log search. `error.kind` is enough to enable tracking on
+its own. The error message and stack trace are **deliberately omitted** — no free text or
+PII ever leaves the machine — so issues group by error *type × command*, not by stack.
+The error tiles on the dashboard filter on `status:error`.
+
 ## Import
 
 UI: **Dashboards → New Dashboard → ⚙️ → Import dashboard JSON**, paste the file.
@@ -33,9 +43,9 @@ curl -X POST "https://api.datadoghq.com/api/v1/dashboard" \
 
 ## Notes
 
-- "Error rate" counts any `@outcome != success` — that includes expected user errors
-  (e.g. `auth_failure`), not just crashes. Filter to `@outcome:internal_error` for a
-  crash-only view.
+- "Error rate" counts any `status:error` (every non-success outcome) — that includes
+  expected user errors (e.g. `not_authenticated`), not just crashes. Filter to
+  `@error.kind:internal_error` for a crash-only view.
 - `@device_id` is a per-machine random UUID, so it approximates installs, not people.
 - Events take ~1–2 minutes to index; a fresh `source:aai-cli` search can look empty
   for a minute even when delivery succeeded.

docs/datadog/cli-usage-dashboard.json

@@ -9,7 +9,7 @@
     {
       "definition": {
         "type": "note",
-        "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`. Events take ~1–2 min to index.",
+        "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`. Failures ship `status:error` + the reserved `@error.kind` (= the anonymous error type, **no message or stack trace**), so they also feed **Error Tracking** — the error tiles below filter on `status:error`. Events take ~1–2 min to index.",
         "background_color": "purple",
         "font_size": "14",
         "text_align": "left",
@@ -439,7 +439,7 @@
                       "name": "errors",
                       "data_source": "logs",
                       "search": {
-                        "query": "source:aai-cli -@outcome:success"
+                        "query": "source:aai-cli status:error"
                       },
                       "indexes": [
                         "*"
@@ -511,7 +511,7 @@
                       "name": "errors",
                       "data_source": "logs",
                       "search": {
-                        "query": "source:aai-cli -@outcome:success"
+                        "query": "source:aai-cli status:error"
                       },
                       "indexes": [
                         "*"
@@ -606,7 +606,7 @@
                       "name": "errors",
                       "data_source": "logs",
                       "search": {
-                        "query": "source:aai-cli -@outcome:success"
+                        "query": "source:aai-cli status:error"
                       },
                       "indexes": [
                         "*"
@@ -653,7 +653,7 @@
                       "name": "errors",
                       "data_source": "logs",
                       "search": {
-                        "query": "source:aai-cli -@outcome:success"
+                        "query": "source:aai-cli status:error"
                       },
                       "indexes": [
                         "*"
@@ -734,14 +734,82 @@
               "width": 4,
               "height": 4
             }
+          },
+          {
+            "definition": {
+              "title": "All errors by frequency (type × command)",
+              "type": "query_table",
+              "requests": [
+                {
+                  "response_format": "scalar",
+                  "queries": [
+                    {
+                      "name": "errors",
+                      "data_source": "logs",
+                      "search": {
+                        "query": "source:aai-cli status:error"
+                      },
+                      "indexes": [
+                        "*"
+                      ],
+                      "compute": {
+                        "aggregation": "count"
+                      },
+                      "group_by": [
+                        {
+                          "facet": "@outcome",
+                          "limit": 100,
+                          "sort": {
+                            "aggregation": "count",
+                            "order": "desc"
+                          }
+                        },
+                        {
+                          "facet": "@command",
+                          "limit": 100,
+                          "sort": {
+                            "aggregation": "count",
+                            "order": "desc"
+                          }
+                        }
+                      ]
+                    }
+                  ],
+                  "formulas": [
+                    {
+                      "formula": "errors",
+                      "alias": "Occurrences",
+                      "cell_display_mode": "bar"
+                    }
+                  ],
+                  "sort": {
+                    "count": 100,
+                    "order_by": [
+                      {
+                        "type": "formula",
+                        "index": 0,
+                        "order": "desc"
+                      }
+                    ]
+                  }
+                }
+              ],
+              "has_search_bar": "auto"
+            },
+            "layout": {
+              "x": 0,
+              "y": 8,
+              "width": 12,
+              "height": 5
+            }
           }
         ]
       },
       "layout": {
         "x": 0,
         "y": 12,
         "width": 12,
-        "height": 9
+        "height": 14
       }
     },
     {
@@ -891,7 +959,7 @@
       },
       "layout": {
         "x": 0,
-        "y": 21,
+        "y": 26,
         "width": 12,
         "height": 5
       }
@@ -1102,7 +1170,7 @@
       },
       "layout": {
         "x": 0,
-        "y": 26,
+        "y": 31,
         "width": 12,
         "height": 5
       }

tests/test_telemetry.py

@@ -114,6 +114,7 @@ def test_build_event_is_allowlisted_and_exact(monkeypatch):
         "service": "aai-cli",
         "ddtags": f"version:{__version__}",
         "message": "aai transcribe success",
+        "status": "info",
         "command": "aai transcribe",
         "outcome": "success",
         "exit_code": 0,
@@ -126,12 +127,24 @@ def test_build_event_is_allowlisted_and_exact(monkeypatch):
     }
 
 
-def test_build_event_marks_ci(monkeypatch):
+def test_build_event_success_is_info_with_no_error_attribute():
+    # A success stays an info log and carries no `error` namespace, so it never
+    # lands in Datadog Error Tracking.
+    event = telemetry.build_event("aai transcribe", outcome="success", exit_code=0, duration_ms=1)
+    assert event["status"] == "info"
+    assert "error" not in event
+
+
+def test_build_event_failure_feeds_error_tracking(monkeypatch):
     monkeypatch.setenv("CI", "true")
     event = telemetry.build_event("aai stream", outcome="api_error", exit_code=1, duration_ms=5)
     assert event["ci"] is True
     assert event["outcome"] == "api_error"
     assert event["exit_code"] == 1
+    # status:error + the reserved error.kind are what promote it into Error Tracking;
+    # error.kind mirrors the anonymous outcome, and no message/stack ever rides along.
+    assert event["status"] == "error"
+    assert event["error"] == {"kind": "api_error"}
 
 
 # --- dispatch (detached flusher handoff) ------------------------------------