Merge origin/main (telemetry error events, sandbox-conflict warning) into the QA-fix branch

claude · claude · commit 4a5594de8863 · 2026-06-12T02:24:38.000Z
Main's #90 fixed two of the same QA findings differently: adopt its --sandbox/--env conflict warning (--env wins, surfaced as {"warning": …} in JSON mode) over this branch's hard usage error, and keep this branch's fuller speak sandbox-hint suggestion which subsumes main's. https://claude.ai/code/session_01LkB3yQbPDG7Ex4mNL3Wtb6
diff --git a/AGENTS.md b/AGENTS.md
@@ -180,7 +180,7 @@ Each file in `aai_cli/commands/` is a Typer sub-app (`transcribe`, `stream`, `ag
 - **`code_gen/`** — backs `--show-code` on `transcribe`/`stream`/`agent`: builds a ready-to-run Python SDK script from exactly the flags passed (no API key needed; generated code reads `ASSEMBLYAI_API_KEY`).
 - **`auth/`** — browser-assisted `assembly login` via AMS + **Stytch B2B OAuth discovery** (`discovery.py`, `flow.py`, `loopback.py`, `ams.py`). Not Stytch Connected Apps.
 - **`init/`** — scaffolds a self-contained FastAPI + HTML starter (`audio-transcription`/`live-captions`/`voice-agent` templates), optionally installs deps and opens the browser; writes the key to a git-ignored `.env`.
-- **`telemetry.py`** — anonymous, opt-out usage telemetry (Supabase-CLI model): `context.run_command` wraps each command body in `telemetry.track(ctx.command_path)`, which dispatches one allow-listed event (command path, outcome/exit code, duration, version/OS — never args, paths, or account data) to the Datadog logs intake via a **detached flusher subprocess** (the hidden `assembly telemetry flush`), so commands never wait on telemetry. `SHIPPED_CLIENT_TOKEN` is a committed write-only Datadog *client* token (`pub…`, embeddable by design — never an API key; `AAI_TELEMETRY_CLIENT_TOKEN` overrides). The test suite blanks it via an autouse conftest fixture so no test ever spawns a real flusher. Opt-out: `AAI_TELEMETRY_DISABLED=1` / `DO_NOT_TRACK=1` / `assembly telemetry disable` (persisted as `telemetry_enabled` in config.toml, alongside the random `device_id`). Send-side failures are swallowed (`OSError`/`CLIError`) — telemetry must never break a command.
+- **`telemetry.py`** — anonymous, opt-out usage telemetry (Supabase-CLI model): `context.run_command` wraps each command body in `telemetry.track(ctx.command_path)`, which dispatches one allow-listed event (command path, outcome/exit code, duration, version/OS, and on failure the error message capped at 500 chars — never args or account data) to the Datadog logs intake via a **detached flusher subprocess** (the hidden `assembly telemetry flush`), so commands never wait on telemetry. `SHIPPED_CLIENT_TOKEN` is a committed write-only Datadog *client* token (`pub…`, embeddable by design — never an API key; `AAI_TELEMETRY_CLIENT_TOKEN` overrides). The test suite blanks it via an autouse conftest fixture so no test ever spawns a real flusher. Opt-out: `AAI_TELEMETRY_DISABLED=1` / `DO_NOT_TRACK=1` / `assembly telemetry disable` (persisted as `telemetry_enabled` in config.toml, alongside the random `device_id`). Send-side failures are swallowed (`OSError`/`CLIError`) — telemetry must never break a command.
 - **`commands/setup.py`** — `assembly setup install/status/remove` wires a coding agent up to AssemblyAI by installing three artifacts: the `assemblyai-docs` docs MCP (via `claude mcp add`), the AssemblyAI skill (via `npx skills add`), and the bundled `aai-cli` skill (copied out of the wheel, no network). Missing `claude`/`npx` is reported and skipped, not an error. The presence probes (docs MCP registered, skills on disk) live in `aai_cli/coding_agent.py` so `assembly doctor`'s coding-agent check can share them — command modules are import-linter-independent, so neither command may import the other.
 
 ## Conventions
diff --git a/aai_cli/commands/speak.py b/aai_cli/commands/speak.py
@@ -217,7 +217,8 @@ def speak(
     --out. Speaker-labeled input (from 'assembly transcribe
     --speaker-labels') is detected automatically: the labels are stripped
     and each speaker gets a different voice. This feature only exists in
-    the sandbox today — run it as 'assembly --sandbox speak'.
+    the sandbox today — run it as 'assembly --sandbox speak' (--sandbox
+    goes before the subcommand).
     """
 
     def body(state: AppState, json_mode: bool) -> None:
diff --git a/aai_cli/main.py b/aai_cli/main.py
@@ -297,6 +297,17 @@ def _command_line_requests_json(raw_args: list[str]) -> bool:
     return False
 
 
+def _sandbox_conflict_warning(sandbox: bool, env: str | None) -> str | None:
+    """A warning when ``--sandbox`` and a contradictory ``--env`` are both passed.
+
+    Credentials are environment-bound, so the conflict must not be resolved silently:
+    ``--env`` wins, and the warning names the loser so the user can drop a flag.
+    """
+    if sandbox and env is not None and env != "sandbox000":
+        return f"--sandbox ignored: --env {env} takes precedence."
+    return None
+
+
 def _offer_or_help(ctx: typer.Context, state: AppState) -> None:
     """No subcommand given: offer guided setup to a credential-less, interactive user;
     otherwise print help. Never prompts in a non-interactive session, and never on
@@ -353,21 +364,12 @@ def main(
     ),
 ) -> None:
     # The command's own --json flag isn't parsed yet, so sniff the pending command line:
-    # a root-callback failure (a bad --env, a --sandbox/--env conflict) still emits the
-    # JSON error shape when the invocation opted into JSON, and renders human text
-    # on stderr otherwise.
+    # a root-callback failure (e.g. bad --env) still emits the JSON error shape when the
+    # invocation opted into JSON, and renders human text on stderr otherwise.
     raw_args: list[str] = ctx.meta.get(_RAW_ARGS_META_KEY, [])
     json_mode = output.resolve_json(explicit=_command_line_requests_json(raw_args))
-    if sandbox and env is not None and env != "sandbox000":
-        # Resolving the disagreement silently (to either side) would send credentials
-        # to an environment the user didn't expect, so refuse the contradiction.
-        conflict = UsageError(
-            f"--sandbox conflicts with --env {env}: --sandbox is shorthand for --env sandbox000.",
-            suggestion="Drop --sandbox, or pass --env sandbox000.",
-        )
-        output.emit_error(conflict, json_mode=json_mode)
-        raise typer.Exit(code=conflict.exit_code)
-    if sandbox:
+    conflict_warning = _sandbox_conflict_warning(sandbox, env)
+    if sandbox and env is None:
         env = "sandbox000"
     state = AppState(profile=profile, env=env, quiet=quiet)
     ctx.obj = state
@@ -376,11 +378,11 @@ def main(
     except CLIError as err:
         output.emit_error(err, json_mode=json_mode)
         raise typer.Exit(code=err.exit_code) from None
-    warning = env_override_warning(state)
-    if warning and not quiet:
-        # Surfaced in JSON mode too (as {"warning": …}), so a `--json` pipeline gets a
-        # machine-readable hint instead of an unexplained downstream auth failure.
-        output.emit_warning(warning, json_mode=json_mode)
+    for warning in (conflict_warning, env_override_warning(state)):
+        if warning and not quiet:
+            # Surfaced in JSON mode too (as {"warning": …}), so a `--json` pipeline gets
+            # a machine-readable hint instead of an unexplained downstream auth failure.
+            output.emit_warning(warning, json_mode=json_mode)
     if ctx.invoked_subcommand is None:
         _offer_or_help(ctx, state)
 
diff --git a/aai_cli/telemetry.py b/aai_cli/telemetry.py
@@ -1,7 +1,8 @@
 """Anonymous usage telemetry, modeled on the Supabase CLI's design.
 
 One allow-listed event per command run (command path, outcome, duration — never
-arguments, file paths, ids, or account data) is shipped to the Datadog logs
+arguments, ids, or account data; a failure also carries the error message,
+capped at 500 chars) is shipped to the Datadog logs
 intake using a write-only *client* token (``pub…``), the credential class
 Datadog designs to be embedded in client apps. ``SHIPPED_CLIENT_TOKEN`` carries
 it (it is public by design — never put an API key there);
@@ -47,6 +48,10 @@
 
 _SEND_TIMEOUT_SECONDS = 5.0
 
+# Cap on the error message shipped with a failure event: enough for any CLI
+# error line, while bounding the payload if an upstream message embeds a body.
+_ERROR_MESSAGE_MAX_CHARS = 500
+
 
 def client_token() -> str:
     """The write-only intake token: env override first, then the shipped one."""
@@ -131,7 +136,12 @@ def _maybe_emit_first_run_notice() -> None:
 
 
 def build_event(
-    command: str, *, outcome: str, exit_code: int, duration_ms: int
+    command: str,
+    *,
+    outcome: str,
+    exit_code: int,
+    duration_ms: int,
+    error_message: str | None = None,
 ) -> dict[str, object]:
     """One invocation event, shaped for the Datadog logs intake.
 
@@ -141,10 +151,11 @@ def build_event(
     hostname ever rides along.
 
     A failure additionally sets ``status: error`` and the reserved
-    ``error.kind`` so the event feeds Datadog **Error Tracking** (issue
-    grouping), not just log search. ``error.kind`` reuses the anonymous
-    ``outcome`` (the ``CLIError.error_type``) — the error *message* and stack
-    trace are deliberately omitted, so no free text or PII ever rides along.
+    ``error.kind``/``error.message`` so the event feeds Datadog **Error
+    Tracking** (issue grouping), not just log search. ``error.kind`` reuses the
+    anonymous ``outcome`` (the ``CLIError.error_type``); ``error.message`` is
+    the one-line message the user saw (capped at ``_ERROR_MESSAGE_MAX_CHARS``).
+    Stack traces are still deliberately omitted.
     """
     succeeded = outcome == "success"
     event: dict[str, object] = {
@@ -164,7 +175,10 @@ def build_event(
         "device_id": config.get_device_id(),
     }
     if not succeeded:
-        event["error"] = {"kind": outcome}
+        error: dict[str, object] = {"kind": outcome}
+        if error_message:
+            error["message"] = error_message[:_ERROR_MESSAGE_MAX_CHARS]
+        event["error"] = error
     return event
 
 
@@ -210,11 +224,24 @@ def flush_payload(raw: str) -> None:
         )
 
 
-def _safe_dispatch(command: str, started: float, *, outcome: str, exit_code: int) -> None:
+def _safe_dispatch(
+    command: str,
+    started: float,
+    *,
+    outcome: str,
+    exit_code: int,
+    error_message: str | None = None,
+) -> None:
     duration_ms = int((time.monotonic() - started) * 1000)
     try:
         dispatch(
-            build_event(command, outcome=outcome, exit_code=exit_code, duration_ms=duration_ms)
+            build_event(
+                command,
+                outcome=outcome,
+                exit_code=exit_code,
+                duration_ms=duration_ms,
+                error_message=error_message,
+            )
         )
     except (OSError, CLIError):
         # Best-effort by contract: a config/spawn failure while *recording* a command
@@ -239,14 +266,22 @@ def track(command: str) -> Generator[None]:
     try:
         yield
     except CLIError as err:
-        _safe_dispatch(command, started, outcome=err.error_type, exit_code=err.exit_code)
+        _safe_dispatch(
+            command,
+            started,
+            outcome=err.error_type,
+            exit_code=err.exit_code,
+            error_message=err.message,
+        )
         raise
     except typer.Exit as exc:
         code = exc.exit_code
         outcome = "success" if code == 0 else "error"
         _safe_dispatch(command, started, outcome=outcome, exit_code=code)
         raise
-    except BaseException:
-        _safe_dispatch(command, started, outcome="internal_error", exit_code=1)
+    except BaseException as exc:
+        _safe_dispatch(
+            command, started, outcome="internal_error", exit_code=1, error_message=str(exc)
+        )
         raise
     _safe_dispatch(command, started, outcome="success", exit_code=0)
diff --git a/aai_cli/transcribe_batch.py b/aai_cli/transcribe_batch.py
@@ -73,7 +73,11 @@ def expand_sources(source: str | None, *, from_stdin: bool, sample: bool) -> lis
     """
     if from_stdin:
         return _stdin_sources(source, sample=sample)
-    if source is None or sample or source == "-" or source.startswith(_URL_PREFIXES):
+    # `not source` (rather than `is None`) also catches the empty string — e.g. an
+    # unset shell variable in `assembly transcribe "$FILE"`. `Path("")` is `Path(".")`,
+    # so it would otherwise fall into the directory branch and batch-transcribe the
+    # whole working directory; instead it stays single-source and fails validation.
+    if not source or sample or source == "-" or source.startswith(_URL_PREFIXES):
         return None
     path = Path(source)
     if path.is_dir():
diff --git a/docs/datadog/cli-usage-dashboard.json b/docs/datadog/cli-usage-dashboard.json
@@ -9,7 +9,7 @@
     {
       "definition": {
         "type": "note",
-        "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`. Failures ship `status:error` + the reserved `@error.kind` (= the anonymous error type, **no message or stack trace**), so they also feed **Error Tracking** — the error tiles below filter on `status:error`. Events take ~1–2 min to index.",
+        "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`, `@error.message`. Failures ship `status:error` + the reserved `@error.kind` (= the anonymous error type) and `@error.message` (the one-line message the user saw, capped at 500 chars — **no stack trace**), so they also feed **Error Tracking** — the error tiles below filter on `status:error`. Events take ~1–2 min to index.",
         "background_color": "purple",
         "font_size": "14",
         "text_align": "left",
@@ -802,14 +802,113 @@
               "width": 12,
               "height": 5
             }
+          },
+          {
+            "definition": {
+              "title": "Top error messages",
+              "type": "toplist",
+              "requests": [
+                {
+                  "response_format": "scalar",
+                  "queries": [
+                    {
+                      "name": "errors",
+                      "data_source": "logs",
+                      "search": {
+                        "query": "source:aai-cli status:error"
+                      },
+                      "indexes": [
+                        "*"
+                      ],
+                      "compute": {
+                        "aggregation": "count"
+                      },
+                      "group_by": [
+                        {
+                          "facet": "@error.message",
+                          "limit": 15,
+                          "sort": {
+                            "aggregation": "count",
+                            "order": "desc"
+                          }
+                        }
+                      ]
+                    }
+                  ],
+                  "formulas": [
+                    {
+                      "formula": "errors"
+                    }
+                  ]
+                }
+              ]
+            },
+            "layout": {
+              "x": 0,
+              "y": 13,
+              "width": 5,
+              "height": 4
+            }
+          },
+          {
+            "definition": {
+              "title": "Recent errors",
+              "type": "list_stream",
+              "requests": [
+                {
+                  "response_format": "event_list",
+                  "columns": [
+                    {
+                      "field": "status_line",
+                      "width": "auto"
+                    },
+                    {
+                      "field": "timestamp",
+                      "width": "auto"
+                    },
+                    {
+                      "field": "@command",
+                      "width": "auto"
+                    },
+                    {
+                      "field": "@error.kind",
+                      "width": "auto"
+                    },
+                    {
+                      "field": "@error.message",
+                      "width": "full"
+                    },
+                    {
+                      "field": "@cli_version",
+                      "width": "auto"
+                    }
+                  ],
+                  "query": {
+                    "data_source": "logs_stream",
+                    "query_string": "source:aai-cli status:error",
+                    "indexes": [],
+                    "sort": {
+                      "column": "timestamp",
+                      "order": "desc"
+                    }
+                  }
+                }
+              ]
+            },
+            "layout": {
+              "x": 5,
+              "y": 13,
+              "width": 7,
+              "height": 4
+            }
           }
         ]
       },
       "layout": {
         "x": 0,
         "y": 12,
         "width": 12,
-        "height": 14
+        "height": 18
       }
     },
     {
@@ -959,7 +1058,7 @@
       },
       "layout": {
         "x": 0,
-        "y": 26,
+        "y": 30,
         "width": 12,
         "height": 5
       }
@@ -1170,7 +1269,7 @@
       },
       "layout": {
         "x": 0,
-        "y": 31,
+        "y": 35,
         "width": 12,
         "height": 5
       }
diff --git a/tests/__snapshots__/test_cli_output_snapshots.ambr b/tests/__snapshots__/test_cli_output_snapshots.ambr
@@ -776,7 +776,8 @@
    --out. Speaker-labeled input (from 'assembly transcribe
    --speaker-labels') is detected automatically: the labels are stripped
    and each speaker gets a different voice. This feature only exists in
-   the sandbox today — run it as 'assembly --sandbox speak'.
+   the sandbox today — run it as 'assembly --sandbox speak' (--sandbox
+   goes before the subcommand).
   
   ╭─ Arguments ──────────────────────────────────────────────────────────────────╮
   │   text      [TEXT]  Text to speak. Omit to read from stdin.                  │
diff --git a/tests/test_smoke.py b/tests/test_smoke.py
diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py
diff --git a/tests/test_transcribe_batch_sources.py b/tests/test_transcribe_batch_sources.py
