From 21999ffec0030fa2ad3e6016dce3cf6a22ead675 Mon Sep 17 00:00:00 2001 From: Aaryan Divate Date: Sun, 7 Jun 2026 23:04:06 -0700 Subject: [PATCH 1/4] Org and Project Configuration --- README.md | 14 +- scripts/generate_cli.py | 142 ++- src/judgment_cli/context.py | 59 ++ src/judgment_cli/context_entities.py | 122 +++ src/judgment_cli/context_resolver.py | 304 ++++++ src/judgment_cli/generated_commands.py | 1267 ++++++++++++++++++++---- src/judgment_cli/judges.py | 41 +- src/judgment_cli/main.py | 201 +++- src/judgment_cli/ui.py | 172 +++- tests/test_context.py | 333 +++++++ tests/test_ui.py | 36 + 11 files changed, 2505 insertions(+), 186 deletions(-) create mode 100644 src/judgment_cli/context.py create mode 100644 src/judgment_cli/context_entities.py create mode 100644 src/judgment_cli/context_resolver.py create mode 100644 tests/test_context.py create mode 100644 tests/test_ui.py diff --git a/README.md b/README.md index b890c4f..d6201c9 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Credentials are written atomically with `0600` permissions to a platform-appropr | Priority | Method | Example | |----------|-------------|----------------------------------------------------------| -| 1 | Env vars | `JUDGMENT_API_KEY`, `JUDGMENT_ORG_ID`, `JUDGMENT_BASE_URL`, `JUDGMENT_AUTH_URL` | +| 1 | Env vars | `JUDGMENT_API_KEY`, `JUDGMENT_ORG_ID`, `JUDGMENT_PROJECT_ID`, `JUDGMENT_BASE_URL`, `JUDGMENT_AUTH_URL` | | 2 | Config file | `judgment login` | ```bash @@ -87,11 +87,23 @@ judgment completion fish > ~/.config/fish/completions/judgment.fish Run `judgment --help` for the full command list, and `judgment --help` for the flags on a specific command. +Select a default organization and project from the CLI: + +```bash +judgment context set # arrow-key selector; type to filter orgs/projects +judgment context show +``` + +Once context is set, project-scoped commands can omit IDs. You can still pass +`--organization-id`, `--project-id`, `--organization`, `--project`, or the old +positional IDs for scripts. + ```bash # Projects judgment projects list # Traces +judgment traces search --pagination '{"limit":25,"cursorSortValue":null,"cursorItemId":null}' judgment traces search --pagination '{"limit":25,"cursorSortValue":null,"cursorItemId":null}' judgment traces get judgment traces spans diff --git a/scripts/generate_cli.py b/scripts/generate_cli.py index 1bd37c8..679e308 100644 --- a/scripts/generate_cli.py +++ b/scripts/generate_cli.py @@ -41,6 +41,7 @@ # Operations whose CLI command is hand-written in judgment_cli/judges.py # (or another extension module) and must not be auto-generated. MANUAL_COMMANDS = {"judges.upload"} +CONTEXT_FIELDS = {"organization_id", "project_id"} # --------------------------------------------------------------------------- @@ -173,6 +174,10 @@ def py_var_name(name: str) -> str: return s +def _is_context_field(name: str) -> bool: + return name in CONTEXT_FIELDS + + def _schema_type(schema: dict[str, Any]) -> str | None: if "type" in schema: return schema["type"] @@ -322,16 +327,82 @@ def generate_command_code( body_props = extract_json_body_properties(operation) is_table = cmd_name in ("list", "search") + context_names: set[str] = set() + context_names.update(pp for pp in path_params if _is_context_field(pp)) + context_names.update( + qp["name"] + for qp in query_params + if qp["required"] and _is_context_field(qp["name"]) + ) + if method != "GET": + context_names.update( + prop["name"] + for prop in body_props + if prop["required"] and _is_context_field(prop["name"]) + ) + + has_context = bool(context_names) + needs_organization_id = "organization_id" in context_names + needs_project_id = "project_id" in context_names + add_organization_options = needs_organization_id or needs_project_id + + def query_is_positional(qp: dict[str, Any]) -> bool: + return bool(qp["required"] and _is_positional_scalar(qp["schema"])) + + def body_is_context(prop: dict[str, Any]) -> bool: + return bool(prop["required"] and _is_context_field(prop["name"])) + + positional_names: list[str] = [] + if has_context: + positional_names.extend(pp for pp in path_params if not _is_context_field(pp)) + positional_names.extend( + qp["name"] + for qp in query_params + if query_is_positional(qp) and not _is_context_field(qp["name"]) + ) + if method != "GET": + positional_names.extend( + prop["name"] + for prop in body_props + if prop["positional"] and not body_is_context(prop) + ) lines: list[str] = [f'@{group_name}_group.command("{cmd_name}")'] - for pp in path_params: - lines.append(f'@click.argument("{pp}")') + if has_context: + if add_organization_options: + lines.append( + '@click.option("--organization-id", "--org-id", ' + '"organization_id_option", default=None, ' + 'help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.")' + ) + lines.append( + '@click.option("--organization", "--org", ' + '"organization_name_option", default=None, ' + 'help="Organization name to resolve.")' + ) + if needs_project_id: + lines.append( + '@click.option("--project-id", "project_id_option", default=None, ' + 'help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.")' + ) + lines.append( + '@click.option("--project", "project_name_option", default=None, ' + 'help="Project name to resolve.")' + ) + lines.append('@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]")') + else: + for pp in path_params: + lines.append(f'@click.argument("{pp}")') for qp in query_params: opt = cli_option_name(qp["name"]) var = py_var_name(qp["name"]) - if qp["required"] and _is_positional_scalar(qp["schema"]): + if has_context and qp["required"] and _is_context_field(qp["name"]): + continue + if has_context and query_is_positional(qp): + continue + if query_is_positional(qp): type_arg = click_param_args(qp["schema"]) lines.append(f'@click.argument("{qp["name"]}"{type_arg})') elif qp["required"]: @@ -349,6 +420,10 @@ def generate_command_code( for prop in body_props: opt = cli_option_name(prop["name"]) var = py_var_name(prop["name"]) + if has_context and body_is_context(prop): + continue + if has_context and prop["positional"]: + continue if prop["positional"]: type_arg = click_param_args(prop["schema"]) lines.append(f'@click.argument("{prop["name"]}"{type_arg})') @@ -391,13 +466,68 @@ def generate_command_code( lines.append("@click.pass_context") - sig_parts = ["ctx"] + path_params + [py_var_name(q["name"]) for q in query_params] + sig_parts = ["ctx"] + if has_context: + sig_parts.append("_args") + else: + sig_parts += path_params + sig_parts += [py_var_name(q["name"]) for q in query_params] sig_parts.append("output_format") - if method != "GET": + if has_context: + if add_organization_options: + sig_parts += ["organization_id_option", "organization_name_option"] + if needs_project_id: + sig_parts += ["project_id_option", "project_name_option"] + sig_parts += [ + py_var_name(q["name"]) + for q in query_params + if not (q["required"] and _is_context_field(q["name"])) + and not query_is_positional(q) + ] + if method != "GET": + sig_parts += [ + py_var_name(prop["name"]) + for prop in body_props + if not body_is_context(prop) and not prop["positional"] + ] + elif method != "GET": sig_parts += [py_var_name(prop["name"]) for prop in body_props] lines.append(f"def {func_name}({', '.join(sig_parts)}):") lines.append(_emit_docstring(description)) + if has_context: + organization_option = ( + "organization_id_option" if add_organization_options else "None" + ) + organization_name_option = ( + "organization_name_option" if add_organization_options else "None" + ) + project_option = "project_id_option" if needs_project_id else "None" + project_name_option = "project_name_option" if needs_project_id else "None" + lines.append(" _parsed = _parse_contextual_positionals(") + lines.append(" _args,") + lines.append(f" positional_names={positional_names!r},") + lines.append(f" needs_organization_id={needs_organization_id!r},") + lines.append(f" needs_project_id={needs_project_id!r},") + lines.append(f" organization_id={organization_option},") + lines.append(f" project_id={project_option},") + lines.append(" )") + lines.append(" _context = _resolve_context(") + lines.append(' ctx.obj["client"],') + lines.append(" organization_id=_parsed.organization_id,") + lines.append(f" organization_name={organization_name_option},") + lines.append(" project_id=_parsed.project_id,") + lines.append(f" project_name={project_name_option},") + lines.append(f" require_project={needs_project_id!r},") + lines.append(" )") + if needs_organization_id: + lines.append(" organization_id = _context.organization_id") + if needs_project_id: + lines.append(" project_id = _context.project_id") + for name in positional_names: + var = py_var_name(name) + lines.append(f' {var} = _parsed.values["{name}"]') + if path_params: lines.append(f' url = f"{path}"') else: @@ -481,6 +611,8 @@ def generate_all(spec: dict) -> str: import click + from judgment_cli.context_resolver import parse_contextual_positionals as _parse_contextual_positionals + from judgment_cli.context_resolver import resolve_context as _resolve_context from judgment_cli.ui import table_output as _table_output, yaml_output as _yaml_output """) diff --git a/src/judgment_cli/context.py b/src/judgment_cli/context.py new file mode 100644 index 0000000..ea348ef --- /dev/null +++ b/src/judgment_cli/context.py @@ -0,0 +1,59 @@ +"""Persistent default organization/project context for the CLI.""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from judgment_cli import config + + +@dataclass(frozen=True) +class ActiveContext: + organization_id: str + project_id: str | None = None + organization_name: str | None = None + project_name: str | None = None + + +def context_path() -> Path: + return config.credentials_path().with_name("context.json") + + +def load_context() -> dict[str, Any]: + path = context_path() + if not path.exists(): + return {} + try: + data = json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + return {} + return data if isinstance(data, dict) else {} + + +def save_context(context: ActiveContext) -> Path: + data = { + "organization_id": context.organization_id, + "organization_name": context.organization_name, + "project_id": context.project_id, + "project_name": context.project_name, + } + data = {key: value for key, value in data.items() if value} + + path = context_path() + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(json.dumps(data, indent=2) + "\n") + return path + + +def clear_context() -> bool: + path = context_path() + if path.exists(): + path.unlink() + return True + return False diff --git a/src/judgment_cli/context_entities.py b/src/judgment_cli/context_entities.py new file mode 100644 index 0000000..a700b9e --- /dev/null +++ b/src/judgment_cli/context_entities.py @@ -0,0 +1,122 @@ +"""Normalize organization and project API records for CLI context flows.""" + +from __future__ import annotations + +from typing import Any, Iterable + +import click + +from judgment_cli.context import ActiveContext + + +def extract_items(response: object, preferred_keys: tuple[str, ...]) -> list[dict[str, Any]]: + if isinstance(response, list): + return [item for item in response if isinstance(item, dict)] + if not isinstance(response, dict): + return [] + for key in preferred_keys: + value = response.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + for value in response.values(): + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + return [] + + +def sort_projects_by_usage(projects: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + return sorted(projects, key=_project_usage_sort_key, reverse=True) + + +def organization_label(organization: dict[str, Any]) -> str: + name = organization_name(organization) or "(unnamed organization)" + oid = organization_id(organization) or "-" + return f"{name} {oid}" + + +def project_label(project: dict[str, Any]) -> str: + name = project_name(project) or "(unnamed project)" + pid = project_id(project) or "-" + traces = trace_count(project) + suffix = f" {traces:,} traces" if traces is not None else "" + return f"{name}{suffix} {pid}" + + +def active_context_from_items( + organization: dict[str, Any], + project: dict[str, Any] | None = None, +) -> ActiveContext: + oid = organization_id(organization) + if not oid: + raise click.ClickException("Selected organization is missing an ID.") + return ActiveContext( + organization_id=oid, + organization_name=organization_name(organization), + project_id=project_id(project) if project else None, + project_name=project_name(project) if project else None, + ) + + +def display_name(item: dict[str, Any] | None) -> str | None: + return organization_name(item) or project_name(item) + + +def organization_id(item: dict[str, Any] | None) -> str | None: + return field(item, ("organization_id", "org_id", "id")) + + +def organization_name(item: dict[str, Any] | None) -> str | None: + return field( + item, + ( + "organization_name", + "org_name", + "name", + "display_name", + "detail.name", + ), + ) + + +def project_id(item: dict[str, Any] | None) -> str | None: + return field(item, ("project_id", "id")) + + +def project_name(item: dict[str, Any] | None) -> str | None: + return field(item, ("project_name", "name", "display_name")) + + +def trace_count(project: dict[str, Any]) -> int | None: + for key in ("total_traces", "trace_count", "traces_count", "num_traces"): + value = project.get(key) + if isinstance(value, int): + return value + if isinstance(value, str) and value.isdigit(): + return int(value) + return None + + +def field(item: dict[str, Any] | None, names: tuple[str, ...]) -> str | None: + if not item: + return None + for name in names: + value: object = item + for part in name.split("."): + if not isinstance(value, dict): + value = None + break + value = value.get(part) + if isinstance(value, str) and value: + return value + return None + + +def _project_usage_sort_key(project: dict[str, Any]) -> tuple[int, int, str]: + favorite = bool( + project.get("favorite") + or project.get("is_favorite") + or project.get("is_favorited") + ) + traces = trace_count(project) or 0 + name = project_name(project) or "" + return (int(favorite), traces, name.casefold()) diff --git a/src/judgment_cli/context_resolver.py b/src/judgment_cli/context_resolver.py new file mode 100644 index 0000000..22c4948 --- /dev/null +++ b/src/judgment_cli/context_resolver.py @@ -0,0 +1,304 @@ +"""Resolve organization/project context for generated CLI commands.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import click + +from judgment_cli import context as context_store +from judgment_cli.context_entities import ( + display_name, + extract_items, + organization_id as get_organization_id, + organization_label, + organization_name as get_organization_name, + project_id as get_project_id, + project_label, + project_name as get_project_name, + sort_projects_by_usage, +) +from judgment_cli.env import optional_env_var + + +ORG_ENV_VARS = ("JUDGMENT_ORG_ID", "JUDGMENT_ORGANIZATION_ID") +PROJECT_ENV_VAR = "JUDGMENT_PROJECT_ID" + + +@dataclass(frozen=True) +class ParsedContextualArgs: + organization_id: str | None + project_id: str | None + values: dict[str, str] + + +def fetch_organizations(client: Any) -> list[dict[str, Any]]: + response = client.request("GET", "/organizations") + return extract_items(response, ("organizations",)) + + +def fetch_projects(client: Any, organization_id: str) -> list[dict[str, Any]]: + response = client.request( + "GET", + "/projects", + params={"organization_id": organization_id}, + ) + return sort_projects_by_usage(extract_items(response, ("projects",))) + + +def parse_contextual_positionals( + args: tuple[str, ...], + *, + positional_names: list[str], + needs_organization_id: bool, + needs_project_id: bool, + organization_id: str | None = None, + project_id: str | None = None, +) -> ParsedContextualArgs: + """Split optional leading org/project IDs from command-specific args.""" + values = list(args) + expected = len(positional_names) + extra = len(values) - expected + max_extra = int(needs_organization_id) + int(needs_project_id) + + if extra < 0: + missing = positional_names[len(values) :] + raise click.UsageError(f"Missing argument(s): {', '.join(missing)}") + if extra > max_extra: + raise click.UsageError( + f"Got {len(values)} positional arguments, expected at most " + f"{expected + max_extra}." + ) + + positional_organization_id: str | None = None + positional_project_id: str | None = None + if needs_organization_id and needs_project_id: + if extra == 2: + positional_organization_id = values[0] + positional_project_id = values[1] + elif extra == 1: + positional_project_id = values[0] + elif needs_organization_id and extra == 1: + positional_organization_id = values[0] + elif needs_project_id and extra == 1: + positional_project_id = values[0] + + if ( + organization_id + and positional_organization_id + and organization_id != positional_organization_id + ): + raise click.UsageError( + "Pass organization ID either positionally or with --organization-id, not both." + ) + if project_id and positional_project_id and project_id != positional_project_id: + raise click.UsageError( + "Pass project ID either positionally or with --project-id, not both." + ) + + command_values = values[extra:] + return ParsedContextualArgs( + organization_id=organization_id or positional_organization_id, + project_id=project_id or positional_project_id, + values=dict(zip(positional_names, command_values)), + ) + + +def resolve_context( + client: Any, + *, + organization_id: str | None = None, + organization_name: str | None = None, + project_id: str | None = None, + project_name: str | None = None, + require_project: bool = False, +) -> context_store.ActiveContext: + """Resolve org/project IDs from CLI values, env vars, context, or API data.""" + if organization_id and organization_name: + raise click.ClickException( + "Pass only one of --organization-id or --organization." + ) + if project_id and project_name: + raise click.ClickException("Pass only one of --project-id or --project.") + + saved = context_store.load_context() + env_org_id = _first_env(ORG_ENV_VARS) + env_project_id = optional_env_var(PROJECT_ENV_VAR) + + org: dict[str, Any] | None = None + project: dict[str, Any] | None = None + + if organization_name: + org = _find_one_by_name( + fetch_organizations(client), + organization_name, + "organization", + ) + organization_id = get_organization_id(org) + elif not organization_id: + organization_id = env_org_id or _str_or_none(saved.get("organization_id")) + + if project_name: + if organization_id: + projects = fetch_projects(client, organization_id) + project = _find_one_by_name(projects, project_name, "project") + project_id = get_project_id(project) + else: + org, project = _find_project_across_organizations( + client, + project_name=project_name, + ) + organization_id = get_organization_id(org) + project_id = get_project_id(project) + elif not project_id: + project_id = env_project_id + saved_project_id = _str_or_none(saved.get("project_id")) + saved_organization_id = _str_or_none(saved.get("organization_id")) + if ( + not project_id + and saved_project_id + and (not organization_id or saved_organization_id == organization_id) + ): + project_id = saved_project_id + + if not organization_id and project_id: + org, project = _find_project_across_organizations( + client, + project_id=project_id, + ) + organization_id = get_organization_id(org) + project_id = get_project_id(project) + + if not organization_id: + organizations = fetch_organizations(client) + if len(organizations) == 1: + org = organizations[0] + organization_id = get_organization_id(org) + else: + raise click.ClickException(_organization_help(organizations)) + + if not require_project: + return context_store.ActiveContext( + organization_id=organization_id, + organization_name=get_organization_name(org) + or _saved_name(saved, "organization", organization_id), + ) + + if not project_id: + saved_project_id = _str_or_none(saved.get("project_id")) + if ( + saved_project_id + and saved.get("organization_id") == organization_id + ): + project_id = saved_project_id + else: + projects = fetch_projects(client, organization_id) + if len(projects) == 1: + project = projects[0] + project_id = get_project_id(project) + else: + raise click.ClickException(_project_help(projects, organization_id)) + + return context_store.ActiveContext( + organization_id=organization_id, + project_id=project_id, + organization_name=get_organization_name(org) + or _saved_name(saved, "organization", organization_id), + project_name=get_project_name(project) + or _saved_name(saved, "project", project_id), + ) + + +def _find_one_by_name( + items: list[dict[str, Any]], + name: str, + entity_name: str, +) -> dict[str, Any]: + exact = [ + item + for item in items + if (display_name(item) or "").casefold() == name.casefold() + ] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + raise click.ClickException( + f"Multiple {entity_name}s named {name!r}; pass the ID instead." + ) + raise click.ClickException(f"No {entity_name} named {name!r} was found.") + + +def _find_project_across_organizations( + client: Any, + *, + project_id: str | None = None, + project_name: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + matches: list[tuple[dict[str, Any], dict[str, Any]]] = [] + for organization in fetch_organizations(client): + oid = get_organization_id(organization) + if not oid: + continue + for project in fetch_projects(client, oid): + if project_id and get_project_id(project) == project_id: + matches.append((organization, project)) + elif ( + project_name + and (get_project_name(project) or "").casefold() + == project_name.casefold() + ): + matches.append((organization, project)) + + label = project_id or project_name or "project" + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + raise click.ClickException( + f"Multiple projects matched {label!r}; pass --organization-id too." + ) + raise click.ClickException(f"No project matched {label!r}.") + + +def _organization_help(organizations: list[dict[str, Any]]) -> str: + if not organizations: + return "No organizations were found for this account." + choices = "\n".join(f" {organization_label(org)}" for org in organizations[:10]) + return ( + "No organization selected. Run `judgment context set`, pass " + "--organization-id, or set JUDGMENT_ORG_ID.\n\nOrganizations:\n" + f"{choices}" + ) + + +def _project_help(projects: list[dict[str, Any]], organization_id: str) -> str: + if not projects: + return f"No projects were found for organization {organization_id}." + choices = "\n".join(f" {project_label(project)}" for project in projects[:10]) + return ( + "No project selected. Run `judgment context set`, pass --project-id, " + "or set JUDGMENT_PROJECT_ID.\n\nMost-used projects:\n" + f"{choices}" + ) + + +def _first_env(names: tuple[str, ...]) -> str | None: + for name in names: + value = optional_env_var(name) + if value: + return value + return None + + +def _saved_name( + saved: dict[str, Any], + prefix: str, + entity_id: str | None, +) -> str | None: + if saved.get(f"{prefix}_id") != entity_id: + return None + return _str_or_none(saved.get(f"{prefix}_name")) + + +def _str_or_none(value: object) -> str | None: + return value if isinstance(value, str) and value else None diff --git a/src/judgment_cli/generated_commands.py b/src/judgment_cli/generated_commands.py index 6cfa98a..d39367c 100644 --- a/src/judgment_cli/generated_commands.py +++ b/src/judgment_cli/generated_commands.py @@ -7,6 +7,8 @@ import click +from judgment_cli.context_resolver import parse_contextual_positionals as _parse_contextual_positionals +from judgment_cli.context_resolver import resolve_context as _resolve_context from judgment_cli.ui import table_output as _table_output, yaml_output as _yaml_output @@ -21,13 +23,34 @@ def agent_threads_group() -> None: @agent_threads_group.command("get") -@click.argument("organization_id") -@click.argument("project_id") -@click.argument("thread_id") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") @click.pass_context -def agent_threads_get(ctx, output_format, organization_id, project_id, thread_id): +def agent_threads_get(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): 'Get an agent thread.\n\n\x08\nGet one agent thread conversation, including its transcript, metadata, active run status, and timestamps.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['thread_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + thread_id = _parsed.values["thread_id"] url = "/agent-threads/get" body = {} body["organization_id"] = organization_id @@ -38,26 +61,50 @@ def agent_threads_get(ctx, output_format, organization_id, project_id, thread_id @agent_threads_group.command("list") -@click.argument("organization_id") -@click.argument("project_id") -@click.option("--agent-type", "agent_type", required=True, type=click.Choice(['global_copilot', 'custom_agent'])) -@click.argument("agent_name") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") +@click.option("--agent-type", "agent_type", required=True, help='JSON value for agent_type.') +@click.option("--agent-name", "agent_name", required=True, help='JSON value for agent_name.') @click.option("--judge-id", "judge_id", default=None, help='Restrict to threads associated with this judge.') +@click.option("--all-users", "all_users", default=None, type=bool, help='When true and the caller is a Judgment admin, return threads from all users instead of only the caller.') @click.option("--limit", "limit", default=None, type=float, help='Maximum number of threads to return (1–100).') @click.option("--cursor-updated-at", "cursor_updated_at", default=None, help='Pagination cursor: `updated_at` value from a previous `next_cursor`.') @click.option("--cursor-thread-id", "cursor_thread_id", default=None, help='Pagination cursor: `thread_id` value from a previous `next_cursor`.') @click.option("-o", "--output", "output_format", type=click.Choice(["table", "yaml", "json"]), default="table", help="Output format.") @click.pass_context -def agent_threads_list(ctx, output_format, organization_id, project_id, agent_type, agent_name, judge_id, limit, cursor_updated_at, cursor_thread_id): +def agent_threads_list(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, agent_type, agent_name, judge_id, all_users, limit, cursor_updated_at, cursor_thread_id): "List agent thread conversations.\n\n\x08\nList the authenticated user's agent thread conversations in a project (global_copilot or custom_agent). Returns each thread's title, type, message count, active run status, and timestamps." + _parsed = _parse_contextual_positionals( + _args, + positional_names=[], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id url = "/agent-threads/list" body = {} body["organization_id"] = organization_id body["project_id"] = project_id - body["agent_type"] = agent_type - body["agent_name"] = agent_name + body["agent_type"] = json.loads(agent_type) + body["agent_name"] = json.loads(agent_name) if judge_id is not None: body["judge_id"] = judge_id + if all_users is not None: + body["all_users"] = all_users if limit is not None: body["limit"] = limit if cursor_updated_at is not None: @@ -79,9 +126,11 @@ def automations_group() -> None: @automations_group.command("create") -@click.argument("organization_id") -@click.argument("project_id") -@click.argument("name") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("--description", "description", default=None, help='Human-readable description shown in the UI.') @click.option("--conditions", "conditions", required=True, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on `combine_type` (`all` vs `any`).\n\n**Condition shape:**\n```\n{\n "metric": {\n "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n "name": "",\n "threshold": ?\n },\n "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n}\n```\n\n**Common scorer_type values:**\n- `behavior` — Judge-scored behavior (name = behavior name, e.g. "Relevance")\n- `static` — Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n- `prompt`/`custom` — Prompt or custom scorer by name\n- `span_attribute` — Arbitrary span attribute key (name = attribute key)\n- `error` — Span error condition') @click.option("--combine-type", "combine_type", required=True, type=click.Choice(['all', 'any'])) @@ -90,8 +139,27 @@ def automations_group() -> None: @click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON object describing the rate-limit window. Omit to leave unset; if provided, all three fields are required.\n\n**Shape:** `{ "count": , "period": , "period_unit": "seconds" | "minutes" | "hours" | "days" }`\n\nExample: `{ "count": 5, "period": 1, "period_unit": "hours" }` (max 5 triggers per 1 hour)') @click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") @click.pass_context -def automations_create(ctx, output_format, organization_id, project_id, name, description, conditions, combine_type, actions, cooldown_period, trigger_frequency): +def automations_create(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, description, conditions, combine_type, actions, cooldown_period, trigger_frequency): 'Create an automation.\n\n\x08\nCreate an automation (rule) in a project. An automation watches behavior/latency/cost metrics and fires actions when its conditions match. Requires the developer role.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['name'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + name = _parsed.values["name"] url = "/automations/create" body = {} body["organization_id"] = organization_id @@ -112,13 +180,34 @@ def automations_create(ctx, output_format, organization_id, project_id, name, de @automations_group.command("delete") -@click.argument("organization_id") -@click.argument("project_id") -@click.argument("rule_id") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") @click.pass_context -def automations_delete(ctx, output_format, organization_id, project_id, rule_id): +def automations_delete(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): 'Delete an automation.\n\n\x08\nDelete an automation. Requires the admin role.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['rule_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + rule_id = _parsed.values["rule_id"] url = "/automations/delete" body = {} body["organization_id"] = organization_id @@ -129,13 +218,34 @@ def automations_delete(ctx, output_format, organization_id, project_id, rule_id) @automations_group.command("get") -@click.argument("organization_id") -@click.argument("project_id") -@click.argument("rule_id") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") @click.pass_context -def automations_get(ctx, output_format, organization_id, project_id, rule_id): +def automations_get(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): """Get an automation by ID.""" + _parsed = _parse_contextual_positionals( + _args, + positional_names=['rule_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + rule_id = _parsed.values["rule_id"] url = "/automations/detail" body = {} body["organization_id"] = organization_id @@ -146,12 +256,33 @@ def automations_get(ctx, output_format, organization_id, project_id, rule_id): @automations_group.command("list") -@click.argument("organization_id") -@click.argument("project_id") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("-o", "--output", "output_format", type=click.Choice(["table", "yaml", "json"]), default="table", help="Output format.") @click.pass_context -def automations_list(ctx, output_format, organization_id, project_id): +def automations_list(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): """List automations.""" + _parsed = _parse_contextual_positionals( + _args, + positional_names=[], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id url = "/automations/list" body = {} body["organization_id"] = organization_id @@ -161,9 +292,11 @@ def automations_list(ctx, output_format, organization_id, project_id): @automations_group.command("update") -@click.argument("organization_id") -@click.argument("project_id") -@click.argument("rule_id") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("--name", "name", default=None, help='New name for the automation.') @click.option("--description", "description", default=None, help='New description for the automation.') @click.option("--conditions", "conditions", default=None, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on `combine_type` (`all` vs `any`).\n\n**Condition shape:**\n```\n{\n "metric": {\n "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n "name": "",\n "threshold": ?\n },\n "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n}\n```\n\n**Common scorer_type values:**\n- `behavior` — Judge-scored behavior (name = behavior name, e.g. "Relevance")\n- `static` — Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n- `prompt`/`custom` — Prompt or custom scorer by name\n- `span_attribute` — Arbitrary span attribute key (name = attribute key)\n- `error` — Span error condition') @@ -174,8 +307,27 @@ def automations_list(ctx, output_format, organization_id, project_id): @click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON 3-tuple `[count, period, unit]` describing the rate-limit window. Omit to leave unchanged.\n\n**Shape:** `[, , ]`\n\nExample: `[5, 1, "hours"]` (max 5 triggers per 1 hour)') @click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") @click.pass_context -def automations_update(ctx, output_format, organization_id, project_id, rule_id, name, description, conditions, combine_type, actions, active, cooldown_period, trigger_frequency): +def automations_update(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, name, description, conditions, combine_type, actions, active, cooldown_period, trigger_frequency): 'Update an automation.\n\n\x08\nUpdate an existing automation. All fields other than the IDs are optional — only supplied fields are applied. Use `active: true/false` to enable or disable without changing other fields. Requires the developer role.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['rule_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + rule_id = _parsed.values["rule_id"] url = "/automations/update" body = {} body["organization_id"] = organization_id @@ -212,19 +364,40 @@ def behaviors_group() -> None: @behaviors_group.command("create-binary") -@click.argument("organization_id") -@click.argument("project_id") -@click.argument("name") -@click.argument("prompt") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("--description", "description", default=None, help='Human-readable description shown in the UI.') -@click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.') +@click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.3-codex" when omitted.') @click.option("--category-ids", "category_ids", multiple=True, help='UUIDs of categories to attach the behavior to. Pass an array of category UUIDs.') @click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this is supplied.\n\n**Shape:**\n```\n{\n "online_evaluation_mode": "continuous" | "on_demand",\n "online_sampling_rate": ,\n "online_span_triggers": [\n {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"","key":""?}\n ],\n "online_session_scoring": \n}\n```\n\n`continuous` runs the judge automatically on qualifying spans; `on_demand` requires a manual `judgment traces evaluate` call. `online_sampling_rate` is a percent (0–100) of matching spans to score.') @click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be `score_type=binary` and have no existing behaviors.') @click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") @click.pass_context -def behaviors_create_binary(ctx, output_format, organization_id, project_id, name, prompt, description, model, category_ids, advanced_settings, judge_id): +def behaviors_create_binary(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, description, model, category_ids, advanced_settings, judge_id): 'Create a binary (yes/no) behavior.\n\n\x08\nCreate a binary behavior. The judge LLM uses your prompt to decide true/false on each qualifying span.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['name', 'prompt'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + name = _parsed.values["name"] + prompt = _parsed.values["prompt"] url = "/behaviors/create-binary" body = {} body["organization_id"] = organization_id @@ -246,19 +419,40 @@ def behaviors_create_binary(ctx, output_format, organization_id, project_id, nam @behaviors_group.command("create-classifier") -@click.argument("organization_id") -@click.argument("project_id") -@click.argument("name") -@click.argument("prompt") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar="[ID_OR_ARG]") @click.option("--options", "options", required=True, help='JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.\n\n**Shape:**\n```\n[\n {"name":"