diff --git a/README.md b/README.md index b890c4f..74ed28d 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Credentials are written atomically with `0600` permissions to a platform-appropr | Priority | Method | Example | |----------|-------------|----------------------------------------------------------| -| 1 | Env vars | `JUDGMENT_API_KEY`, `JUDGMENT_ORG_ID`, `JUDGMENT_BASE_URL`, `JUDGMENT_AUTH_URL` | +| 1 | Env vars | `JUDGMENT_API_KEY`, `JUDGMENT_ORG_ID`, `JUDGMENT_PROJECT_ID`, `JUDGMENT_BASE_URL`, `JUDGMENT_AUTH_URL` | | 2 | Config file | `judgment login` | ```bash @@ -87,11 +87,23 @@ judgment completion fish > ~/.config/fish/completions/judgment.fish Run `judgment --help` for the full command list, and `judgment --help` for the flags on a specific command. +Select a default organization and project from the CLI: + +```bash +judgment context set # arrow-key selector; type to filter orgs/projects +judgment context show +``` + +Once context is set, project-scoped commands can omit IDs. You can still pass +`--organization-id`, `--project-id`, `--organization`, `--project`, or the old +positional IDs for scripts. + ```bash # Projects judgment projects list # Traces +judgment traces search --pagination '{"limit":25,"cursorSortValue":null,"cursorItemId":null}' judgment traces search --pagination '{"limit":25,"cursorSortValue":null,"cursorItemId":null}' judgment traces get judgment traces spans @@ -131,4 +143,4 @@ uv sync --extra dev uv run python scripts/generate_cli.py ``` -`generate_cli.py` rewrites `src/judgment_cli/generated_commands.py` from the OpenAPI spec. Pass `--spec ` to point at a different spec; `--help` for the full usage. +`generate_cli.py` rewrites `src/judgment_cli/generated/` from the OpenAPI spec. Pass `--spec ` to point at a different spec; `--help` for the full usage. diff --git a/scripts/generate_cli.py b/scripts/generate_cli.py index 1bd37c8..e291c34 100644 --- a/scripts/generate_cli.py +++ b/scripts/generate_cli.py @@ -2,7 +2,7 @@ """Auto-generate Click CLI commands from the Judgment OpenAPI spec. This script consumes ``cli-server``'s OpenAPI document and emits -``src/judgment_cli/generated_commands.py``. The CLI server is the single +``src/judgment_cli/generated/``. The CLI server is the single source of truth for command names, descriptions, option help, and group structure — this generator is a thin renderer. @@ -17,9 +17,10 @@ * schema-level ``description`` on each request-body / query property — Click ``--option`` help. -Routes whose ``operationId`` is in :data:`MANUAL_COMMANDS` are skipped so -that hand-written commands (e.g. ``judgment judges upload``) own those -slots. +Routes whose ``operationId`` is in :data:`MANUAL_COMMANDS` are skipped by +the Click command generator so that hand-written commands (e.g. ``judgment +judges upload``) own those slots. The API client generator still emits +wrappers and types for those routes. Run ``python scripts/generate_cli.py --help`` for usage. """ @@ -32,15 +33,20 @@ import re import sys import textwrap +from pathlib import Path from typing import Any import httpx +from openapi_type_emitter import OpenApiTypeEmitter + DEFAULT_SPEC = "https://cli.judgmentlabs.ai/openapi/json" +GENERATED_PACKAGE_DIR = Path("src/judgment_cli/generated") # Operations whose CLI command is hand-written in judgment_cli/judges.py # (or another extension module) and must not be auto-generated. MANUAL_COMMANDS = {"judges.upload"} +CONTEXT_FIELDS = {"organization_id", "project_id"} # --------------------------------------------------------------------------- @@ -88,7 +94,11 @@ def derive_group_and_command( ) -def collect_operations(spec: dict) -> list[dict[str, Any]]: +def collect_operations( + spec: dict, + *, + include_manual: bool = False, +) -> list[dict[str, Any]]: operations: list[dict[str, Any]] = [] seen: set[tuple[str, str]] = set() for path, path_item in spec.get("paths", {}).items(): @@ -100,7 +110,7 @@ def collect_operations(spec: dict) -> list[dict[str, Any]]: if not isinstance(operation, dict): continue op_id = operation.get("operationId") - if op_id in MANUAL_COMMANDS: + if op_id in MANUAL_COMMANDS and not include_manual: continue group, command = derive_group_and_command( operation, path, method.upper() @@ -164,6 +174,11 @@ def cli_option_name(name: str) -> str: return s.lower().replace("_", "-") +def cli_arg_metavar(name: str) -> str: + """Render a schema field name as a Click usage placeholder.""" + return cli_option_name(name).replace("-", "_").upper() + + def py_var_name(name: str) -> str: """Coerce *name* into a valid (non-reserved) Python identifier.""" s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name) @@ -173,12 +188,21 @@ def py_var_name(name: str) -> str: return s +def _is_context_field(name: str) -> bool: + return name in CONTEXT_FIELDS + + def _schema_type(schema: dict[str, Any]) -> str | None: if "type" in schema: return schema["type"] - for option in schema.get("anyOf", []): - if option.get("type") and option["type"] != "null": - return option["type"] + types = [] + for option in schema.get("anyOf") or schema.get("allOf") or []: + option_type = _schema_type(option) if isinstance(option, dict) else None + if option_type and option_type != "null": + types.append(option_type) + unique_types = list(dict.fromkeys(types)) + if len(unique_types) == 1: + return unique_types[0] return None @@ -207,7 +231,11 @@ def _is_positional_scalar(schema: dict[str, Any]) -> bool: instead, where a name like ``--combine-type all`` reads better than an anonymous ``{all|any}`` slot in the signature. """ - return _schema_type(schema) == "string" and not schema.get("enum") + return ( + schema.get("type") == "string" + and not schema.get("enum") + and "const" not in schema + ) def click_type_expr(schema: dict[str, Any]) -> str | None: @@ -222,21 +250,31 @@ def click_type_expr(schema: dict[str, Any]) -> str | None: def click_choice_expr(schema: dict[str, Any]) -> str | None: - values = schema.get("enum") + values = _schema_choice_values(schema) if not values: return None quoted = ", ".join(repr(v) for v in values) return f"click.Choice([{quoted}])" +def _schema_choice_values(schema: dict[str, Any]) -> list[Any]: + values = schema.get("enum") or ([schema["const"]] if "const" in schema else []) + for option in schema.get("anyOf") or schema.get("allOf") or []: + if isinstance(option, dict): + values.extend(_schema_choice_values(option)) + return list(dict.fromkeys(values)) + + def _schema_description(schema: dict[str, Any]) -> str | None: - """Extract a human description from a schema (or any of its anyOf branches).""" + """Extract a human description from a schema or composed schema branch.""" desc = schema.get("description") if desc: return desc - for option in schema.get("anyOf", []): - if isinstance(option, dict) and option.get("description"): - return option["description"] + for option in schema.get("anyOf") or schema.get("allOf") or []: + if isinstance(option, dict): + desc = _schema_description(option) + if desc: + return desc return None @@ -244,6 +282,15 @@ def _quote(text: str) -> str: return repr(text) +def _pascal_case(name: str) -> str: + parts = re.split(r"[^a-zA-Z0-9]+", name) + return "".join(part.capitalize() for part in parts if part) + + +def api_func_name(operation_id: str) -> str: + return py_var_name(operation_id.replace(".", "_").replace("-", "_")) + + def _emit_docstring(description: str) -> str: """Render a Click command docstring, preserving paragraph breaks.""" short, _, long = description.partition("\n\n") @@ -275,6 +322,33 @@ def click_param_args( return f", {', '.join(args)}" if args else "" +def contextual_args_metavar( + positional_names: list[str], + *, + needs_organization_id: bool, + needs_project_id: bool, +) -> str: + """Render the contextual catch-all argument's usage string. + + Generated commands use one ``nargs=-1`` argument so users can optionally + pass leading context IDs before the command's own positional values. The + parser accepts: + + * ``[ORG_ID]`` for org-scoped commands + * ``[[ORG_ID] PROJECT_ID]`` for project-scoped commands + * the OpenAPI-derived command positional fields after those context IDs + """ + parts: list[str] = [] + if needs_organization_id and needs_project_id: + parts.append("[[ORG_ID] PROJECT_ID]") + elif needs_project_id: + parts.append("[PROJECT_ID]") + elif needs_organization_id: + parts.append("[ORG_ID]") + parts.extend(cli_arg_metavar(name) for name in positional_names) + return " ".join(parts) + + def extract_json_body_properties(operation: dict) -> list[dict[str, Any]]: request_body = operation.get("requestBody") or {} json_content = (request_body.get("content") or {}).get( @@ -320,18 +394,92 @@ def generate_command_code( path_params = extract_path_params(path) query_params = extract_query_params(operation) body_props = extract_json_body_properties(operation) + api_call = api_func_name(operation["operationId"]) is_table = cmd_name in ("list", "search") + context_names: set[str] = set() + context_names.update(pp for pp in path_params if _is_context_field(pp)) + context_names.update( + qp["name"] + for qp in query_params + if qp["required"] and _is_context_field(qp["name"]) + ) + if method != "GET": + context_names.update( + prop["name"] + for prop in body_props + if prop["required"] and _is_context_field(prop["name"]) + ) + + has_context = bool(context_names) + needs_organization_id = "organization_id" in context_names + needs_project_id = "project_id" in context_names + add_organization_options = needs_organization_id or needs_project_id + + def query_is_positional(qp: dict[str, Any]) -> bool: + return bool(qp["required"] and _is_positional_scalar(qp["schema"])) + + def body_is_context(prop: dict[str, Any]) -> bool: + return bool(prop["required"] and _is_context_field(prop["name"])) + + positional_names: list[str] = [] + if has_context: + positional_names.extend(pp for pp in path_params if not _is_context_field(pp)) + positional_names.extend( + qp["name"] + for qp in query_params + if query_is_positional(qp) and not _is_context_field(qp["name"]) + ) + if method != "GET": + positional_names.extend( + prop["name"] + for prop in body_props + if prop["positional"] and not body_is_context(prop) + ) lines: list[str] = [f'@{group_name}_group.command("{cmd_name}")'] - for pp in path_params: - lines.append(f'@click.argument("{pp}")') + if has_context: + if add_organization_options: + lines.append( + '@click.option("--organization-id", "--org-id", ' + '"organization_id_option", default=None, ' + 'help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.")' + ) + lines.append( + '@click.option("--organization", "--org", ' + '"organization_name_option", default=None, ' + 'help="Organization name to resolve.")' + ) + if needs_project_id: + lines.append( + '@click.option("--project-id", "project_id_option", default=None, ' + 'help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.")' + ) + lines.append( + '@click.option("--project", "project_name_option", default=None, ' + 'help="Project name to resolve.")' + ) + metavar = contextual_args_metavar( + positional_names, + needs_organization_id=needs_organization_id, + needs_project_id=needs_project_id, + ) + lines.append( + f'@click.argument("_args", nargs=-1, metavar={metavar!r})' + ) + else: + for pp in path_params: + lines.append(f'@click.argument("{pp}")') for qp in query_params: opt = cli_option_name(qp["name"]) var = py_var_name(qp["name"]) - if qp["required"] and _is_positional_scalar(qp["schema"]): + if has_context and qp["required"] and _is_context_field(qp["name"]): + continue + if has_context and query_is_positional(qp): + continue + if query_is_positional(qp): type_arg = click_param_args(qp["schema"]) lines.append(f'@click.argument("{qp["name"]}"{type_arg})') elif qp["required"]: @@ -349,6 +497,10 @@ def generate_command_code( for prop in body_props: opt = cli_option_name(prop["name"]) var = py_var_name(prop["name"]) + if has_context and body_is_context(prop): + continue + if has_context and prop["positional"]: + continue if prop["positional"]: type_arg = click_param_args(prop["schema"]) lines.append(f'@click.argument("{prop["name"]}"{type_arg})') @@ -391,35 +543,73 @@ def generate_command_code( lines.append("@click.pass_context") - sig_parts = ["ctx"] + path_params + [py_var_name(q["name"]) for q in query_params] + sig_parts = ["ctx"] + if has_context: + sig_parts.append("_args") + else: + sig_parts += path_params + sig_parts += [py_var_name(q["name"]) for q in query_params] sig_parts.append("output_format") - if method != "GET": + if has_context: + if add_organization_options: + sig_parts += ["organization_id_option", "organization_name_option"] + if needs_project_id: + sig_parts += ["project_id_option", "project_name_option"] + sig_parts += [ + py_var_name(q["name"]) + for q in query_params + if not (q["required"] and _is_context_field(q["name"])) + and not query_is_positional(q) + ] + if method != "GET": + sig_parts += [ + py_var_name(prop["name"]) + for prop in body_props + if not body_is_context(prop) and not prop["positional"] + ] + elif method != "GET": sig_parts += [py_var_name(prop["name"]) for prop in body_props] lines.append(f"def {func_name}({', '.join(sig_parts)}):") lines.append(_emit_docstring(description)) - if path_params: - lines.append(f' url = f"{path}"') - else: - lines.append(f' url = "{path}"') - - if query_params: - lines.append(" params = {}") - for qp in query_params: - var = py_var_name(qp["name"]) - if qp["required"]: - lines.append(f' params["{qp["name"]}"] = {var}') - else: - lines.append(f" if {var} is not None:") - lines.append(f' params["{qp["name"]}"] = {var}') + if has_context: + organization_option = ( + "organization_id_option" if add_organization_options else "None" + ) + organization_name_option = ( + "organization_name_option" if add_organization_options else "None" + ) + project_option = "project_id_option" if needs_project_id else "None" + project_name_option = "project_name_option" if needs_project_id else "None" + lines.append(" _parsed = _parse_contextual_positionals(") + lines.append(" _args,") + lines.append(f" positional_names={positional_names!r},") + lines.append(f" needs_organization_id={needs_organization_id!r},") + lines.append(f" needs_project_id={needs_project_id!r},") + lines.append(f" organization_id={organization_option},") + lines.append(f" project_id={project_option},") + lines.append(" )") + lines.append(" _context = _resolve_context(") + lines.append(' ctx.obj["client"],') + lines.append(" organization_id=_parsed.organization_id,") + lines.append(f" organization_name={organization_name_option},") + lines.append(" project_id=_parsed.project_id,") + lines.append(f" project_name={project_name_option},") + lines.append(f" require_project={needs_project_id!r},") + lines.append(" )") + if needs_organization_id: + lines.append(" organization_id = _context.organization_id") + if needs_project_id: + lines.append(" project_id = _context.project_id") + for name in positional_names: + var = py_var_name(name) + lines.append(f' {var} = _parsed.values["{name}"]') if method == "GET": - call_args = [f'"{method}"', "url"] - if query_params: - call_args.append("params=params") - lines.append( - f' result = ctx.obj["client"].request({", ".join(call_args)})' - ) + call_args = ['ctx.obj["client"]'] + call_args.extend(path_params) + call_args.extend(py_var_name(qp["name"]) for qp in query_params) + lines.append(f' result = _api.{api_call}({", ".join(call_args)})') if is_table: lines.append(" _table_output(result, output_format=output_format)") else: @@ -445,9 +635,7 @@ def generate_command_code( lines.append(f" if {var} is not None:") lines.append(f' body["{prop["name"]}"] = json.loads({var})') - lines.append( - f' result = ctx.obj["client"].request("{method}", url, json_body=body)' - ) + lines.append(f' result = _api.{api_call}(ctx.obj["client"], body)') if is_table: lines.append(" _table_output(result, output_format=output_format)") else: @@ -481,6 +669,9 @@ def generate_all(spec: dict) -> str: import click + from judgment_cli.generated import api as _api + from judgment_cli.context_resolver import parse_contextual_positionals as _parse_contextual_positionals + from judgment_cli.context_resolver import resolve_context as _resolve_context from judgment_cli.ui import table_output as _table_output, yaml_output as _yaml_output """) @@ -533,6 +724,168 @@ def generate_all(spec: dict) -> str: return out +def _response_schema(operation: dict[str, Any]) -> dict[str, Any]: + return ( + ((operation.get("responses") or {}).get("200") or {}) + .get("content", {}) + .get("application/json", {}) + .get("schema", {}) + ) + + +def _request_body(operation: dict[str, Any]) -> tuple[str, dict[str, Any]] | None: + content = (operation.get("requestBody") or {}).get("content", {}) + for content_type in ("application/json", "multipart/form-data"): + schema = (content.get(content_type) or {}).get("schema") + if isinstance(schema, dict): + return content_type, schema + return None + + +def _response_type_name(operation_id: str) -> str: + return f"{_pascal_case(operation_id)}Response" + + +def _request_type_name(operation_id: str) -> str: + return f"{_pascal_case(operation_id)}Body" + + +def _multipart_request_setup( + schema: dict[str, Any], + *, + required_names: list[str], +) -> list[str]: + lines = [ + " data: dict[str, str] = {}", + " files: dict[str, tuple[str, bytes, str]] = {}", + ] + properties = schema.get("properties") or {} + required = set(required_names) + for prop_name, prop_schema in properties.items(): + if not isinstance(prop_schema, dict): + continue + var_name = py_var_name(prop_name) + is_required = prop_name in required + if is_required: + lines.append(f" {var_name} = body[{prop_name!r}]") + indent = " " + else: + lines.append(f" {var_name} = body.get({prop_name!r})") + lines.append(f" if {var_name} is not None:") + indent = " " + + if prop_schema.get("type") == "string" and prop_schema.get("format") == "binary": + lines.append(f"{indent}files[{prop_name!r}] = {var_name}") + elif prop_schema.get("type") == "object": + lines.append(f"{indent}data[{prop_name!r}] = json.dumps({var_name})") + else: + lines.append(f"{indent}data[{prop_name!r}] = str({var_name})") + return lines + + +def generate_api_client(spec: dict) -> tuple[str, str]: + type_emitter = OpenApiTypeEmitter() + functions: list[str] = [] + operations = collect_operations(spec, include_manual=True) + + for entry in operations: + operation_id = entry["operation"]["operationId"] + response_schema = _response_schema(entry["operation"]) + if not response_schema: + raise SystemExit( + f"Operation {operation_id} is missing a 200 JSON response schema." + ) + response_type = type_emitter.type_for( + response_schema, + _response_type_name(operation_id), + ) + request_body = _request_body(entry["operation"]) + request_schema = request_body[1] if request_body else None + content_type = request_body[0] if request_body else None + request_type = ( + type_emitter.type_for( + request_schema, + _request_type_name(operation_id), + ) + if request_schema + else None + ) + func_name = api_func_name(operation_id) + query_params = extract_query_params(entry["operation"]) + signature = [f"client: JudgmentClient"] + signature.extend(f"{name}: str" for name in extract_path_params(entry["path"])) + for param in query_params: + param_type = type_emitter.type_for( + param.get("schema") or {}, + f"{_pascal_case(operation_id)}{_pascal_case(param['name'])}", + ) + default = "" if param.get("required") else " | None = None" + signature.append(f"{py_var_name(param['name'])}: {param_type}{default}") + if request_type: + signature.append(f"body: {request_type}") + functions.append( + f"def {func_name}({', '.join(signature)}) -> {response_type}:" + ) + if content_type == "multipart/form-data" and request_schema: + functions.extend( + _multipart_request_setup(request_schema, required_names=request_schema.get("required") or []) + ) + functions.append(" return cast(") + functions.append(f" {response_type},") + request_method = ( + "client.multipart" + if content_type == "multipart/form-data" + else "client.request" + ) + functions.append(f" {request_method}(") + functions.append(f" {entry['method']!r},") + path_expr = f'f"{entry["path"]}"' if extract_path_params(entry["path"]) else repr(entry["path"]) + functions.append(f" {path_expr},") + if query_params and content_type != "multipart/form-data": + params = ", ".join( + f"{param['name']!r}: {py_var_name(param['name'])}" + for param in query_params + ) + functions.append(f" params={{{params}}},") + if content_type == "multipart/form-data": + functions.append(" data=data,") + functions.append(" files=files,") + elif request_type: + functions.append(" json_body=body,") + functions.append(" ),") + functions.append(" )") + functions.extend(["", ""]) + + types_out = textwrap.dedent("""\ + # Auto-generated by scripts/generate_cli.py + # DO NOT EDIT MANUALLY + + from __future__ import annotations + + from typing import Any, Literal, TypedDict + + """) + for lines in type_emitter.definitions.values(): + types_out += "\n".join(lines) + "\n\n\n" + + api_out = textwrap.dedent("""\ + # Auto-generated by scripts/generate_cli.py + # DO NOT EDIT MANUALLY + + from __future__ import annotations + + import json + + from typing import cast + + from judgment_cli.client import JudgmentClient + from judgment_cli.generated.types import * # noqa: F403 + + """) + api_out += "\n".join(functions).rstrip() + "\n" + return types_out, api_out + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -554,12 +907,25 @@ def main() -> None: spec = load_spec(args.spec) print(f"Found {len(spec.get('paths', {}))} paths", file=sys.stderr) - code = generate_all(spec) + commands_code = generate_all(spec) + types_code, api_code = generate_api_client(spec) + + GENERATED_PACKAGE_DIR.mkdir(parents=True, exist_ok=True) - out_path = "src/judgment_cli/generated_commands.py" - with open(out_path, "w") as f: - f.write(code) - print(f"Wrote {out_path}", file=sys.stderr) + init_path = GENERATED_PACKAGE_DIR / "__init__.py" + init_path.write_text( + "# Auto-generated package for OpenAPI-derived CLI code.\n", + ) + print(f"Wrote {init_path}", file=sys.stderr) + + files = { + GENERATED_PACKAGE_DIR / "commands.py": commands_code, + GENERATED_PACKAGE_DIR / "api.py": api_code, + GENERATED_PACKAGE_DIR / "types.py": types_code, + } + for path, code in files.items(): + path.write_text(code) + print(f"Wrote {path}", file=sys.stderr) if __name__ == "__main__": diff --git a/scripts/generate_reference.py b/scripts/generate_reference.py index f23c14b..1f4b581 100644 --- a/scripts/generate_reference.py +++ b/scripts/generate_reference.py @@ -7,7 +7,7 @@ ``content/docs/cli-reference/`` tree. Source of truth lives in ``src/judgment_cli/main.py`` and -``src/judgment_cli/generated_commands.py``; this script is a thin +``src/judgment_cli/generated/commands.py``; this script is a thin renderer that introspects ``Click.Command``/``Click.Group`` objects. Usage:: diff --git a/scripts/openapi_type_emitter.py b/scripts/openapi_type_emitter.py new file mode 100644 index 0000000..c8c21c7 --- /dev/null +++ b/scripts/openapi_type_emitter.py @@ -0,0 +1,255 @@ +"""OpenAPI-to-Python type emitter for the generated CLI API client.""" + +from __future__ import annotations + +import json +import keyword +import re +from typing import Any + + +def _pascal_case(name: str) -> str: + parts = re.split(r"[^a-zA-Z0-9]+", name) + return "".join(part.capitalize() for part in parts if part) + + +def _nested_type_name(parent_name: str, property_name: str) -> str: + return f"{parent_name}{_pascal_case(property_name)}" + + +class OpenApiTypeEmitter: + """Emit Python type hints for the OpenAPI subset produced by cli-server. + + Supported deliberately: + - primitive JSON types, binary strings, arrays, fixed tuples, and maps + - enum/const values as ``Literal`` + - ``anyOf``/``oneOf`` as unions + - ``allOf`` only when branches collapse to one type or merge objects + - object ``required`` fields as required ``TypedDict`` keys, with + optional fields emitted through ``total=False`` base classes + + Unsupported constructs fail generation instead of silently becoming + inaccurate ``Any``. + """ + + def __init__(self) -> None: + self.definitions: dict[str, list[str]] = {} + self._fingerprints: dict[str, str] = {} + + def type_for(self, schema: dict[str, Any], name: str) -> str: + if "$ref" in schema: + self._unsupported(name, "$ref schemas must be resolved before generation") + schema, nullable = self._without_null(schema) + type_expr = self._type_for_non_null(schema, name) + if type_expr in {"Any", "None"} or "None" in type_expr.split(" | "): + return type_expr + return f"{type_expr} | None" if nullable else type_expr + + def _type_for_non_null(self, schema: dict[str, Any], name: str) -> str: + if not schema: + return "Any" + for key in ("oneOf", "anyOf"): + if key in schema: + return self._union_type(schema[key], name, key) + if "allOf" in schema: + return self._all_of_type(schema["allOf"], name) + + values = schema.get("enum") or ( + [schema["const"]] if "const" in schema else [] + ) + if values: + return "Literal[" + ", ".join(repr(value) for value in values) + "]" + + schema_type = schema.get("type") + if schema_type == "string" and schema.get("format") == "binary": + return "tuple[str, bytes, str]" + primitives = { + "string": "str", + "integer": "int", + "number": "float", + "boolean": "bool", + } + if schema_type in primitives: + return primitives[schema_type] + if schema_type == "array": + return self._array_type(schema, name) + if schema_type == "object" or "properties" in schema: + return self._object_type(schema, name) + self._unsupported(name, f"unsupported schema keys: {sorted(schema)}") + + def _without_null( + self, + schema: dict[str, Any], + ) -> tuple[dict[str, Any], bool]: + nullable = bool(schema.get("nullable")) + clean = {key: value for key, value in schema.items() if key != "nullable"} + for key in ("anyOf", "oneOf", "allOf"): + options = clean.get(key) + if not isinstance(options, list): + continue + clean[key] = [ + option + for option in options + if not self._is_null_option(option, key) + ] + nullable = nullable or len(clean[key]) != len(options) + return clean, nullable + + def _is_null_option(self, option: Any, key: str) -> bool: + if not isinstance(option, dict): + self._unsupported("schema", f"{key} contains a non-object option") + return option.get("type") == "null" + + def _union_type(self, options: list[Any], name: str, key: str) -> str: + if not options: + return "None" + option_types = [ + self.type_for(option, f"{name}Option{index}") + for index, option in enumerate(options, start=1) + ] + unique_types = list(dict.fromkeys(option_types)) + return "Any" if "Any" in unique_types else " | ".join(unique_types) + + def _all_of_type(self, options: list[Any], name: str) -> str: + if not options: + return "Any" + if all(isinstance(option, dict) and self._is_object_schema(option) for option in options): + return self._object_type(self._merge_object_options(options, name), name) + + unique_types = list(dict.fromkeys( + self.type_for(option, f"{name}AllOf{index}") + for index, option in enumerate(options, start=1) + )) + if len(unique_types) == 1: + return unique_types[0] + self._unsupported(name, f"allOf produced incompatible types: {unique_types}") + + def _array_type(self, schema: dict[str, Any], name: str) -> str: + items = schema.get("items") + if isinstance(items, list): + if schema.get("additionalItems") not in (False, None): + self._unsupported( + name, + "tuple arrays with additionalItems are unsupported", + ) + item_types = [ + self.type_for(item, f"{name}Item{index}") + for index, item in enumerate(items, start=1) + ] + return f"tuple[{', '.join(item_types)}]" + if isinstance(items, dict): + return f"list[{self.type_for(items, f'{name}Item')}]" + self._unsupported(name, "array schema is missing an object items schema") + + def _object_type(self, schema: dict[str, Any], name: str) -> str: + properties = schema.get("properties") or {} + if properties: + self._emit_typeddict(name, schema, properties) + return name + + map_schema = self._map_value_schema(schema, name) + if map_schema is None: + return "dict[str, Any]" + return f"dict[str, {self.type_for(map_schema, f'{name}Value')}]" + + def _emit_typeddict( + self, + name: str, + schema: dict[str, Any], + properties: dict[str, Any], + ) -> None: + fingerprint = json.dumps(schema, sort_keys=True, separators=(",", ":")) + existing = self._fingerprints.get(name) + if existing is not None: + if existing != fingerprint: + self._unsupported(name, "multiple schemas generated the same type name") + return + + required = set(schema.get("required") or []) + if required - set(properties): + self._unsupported( + name, + f"required keys not present in properties: {sorted(required - set(properties))}", + ) + + required_lines: list[str] = [] + optional_lines: list[str] = [] + for prop_name, prop_schema in properties.items(): + if not isinstance(prop_schema, dict): + self._unsupported(name, f"property {prop_name!r} is not a schema") + if not prop_name.isidentifier() or keyword.iskeyword(prop_name): + self._unsupported( + name, + f"property {prop_name!r} is not a Python identifier", + ) + prop_type = self.type_for( + prop_schema, + _nested_type_name(name, prop_name), + ) + line = f" {prop_name}: {prop_type}" + (required_lines if prop_name in required else optional_lines).append(line) + + self._fingerprints[name] = fingerprint + if required_lines and optional_lines: + optional_name = f"{name}Optional" + self.definitions[optional_name] = [ + f"class {optional_name}(TypedDict, total=False):", + *optional_lines, + ] + self.definitions[name] = [f"class {name}({optional_name}):", *required_lines] + elif optional_lines: + self.definitions[name] = [ + f"class {name}(TypedDict, total=False):", + *optional_lines, + ] + else: + self.definitions[name] = [ + f"class {name}(TypedDict):", + *(required_lines or [" pass"]), + ] + + def _map_value_schema( + self, + schema: dict[str, Any], + name: str, + ) -> dict[str, Any] | None: + pattern_properties = schema.get("patternProperties") + if isinstance(pattern_properties, dict): + if len(pattern_properties) != 1: + self._unsupported(name, "multiple patternProperties are unsupported") + return next(iter(pattern_properties.values())) or {} + + additional = schema.get("additionalProperties") + if additional is True: + return {} + if isinstance(additional, dict): + return additional + return None + + def _merge_object_options( + self, + options: list[Any], + name: str, + ) -> dict[str, Any]: + properties: dict[str, Any] = {} + required: list[str] = [] + for option in options: + if not isinstance(option, dict): + self._unsupported(name, "allOf option is not a schema object") + for prop_name, prop_schema in (option.get("properties") or {}).items(): + if prop_name in properties and properties[prop_name] != prop_schema: + self._unsupported( + name, + f"allOf property {prop_name!r} is defined differently", + ) + properties[prop_name] = prop_schema + for prop_name in option.get("required") or []: + if prop_name not in required: + required.append(prop_name) + return {"type": "object", "required": required, "properties": properties} + + def _is_object_schema(self, schema: dict[str, Any]) -> bool: + return schema.get("type") == "object" or "properties" in schema + + def _unsupported(self, name: str, reason: str) -> None: + raise SystemExit(f"Unsupported OpenAPI schema for {name}: {reason}") diff --git a/src/judgment_cli/context.py b/src/judgment_cli/context.py new file mode 100644 index 0000000..ea348ef --- /dev/null +++ b/src/judgment_cli/context.py @@ -0,0 +1,59 @@ +"""Persistent default organization/project context for the CLI.""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from judgment_cli import config + + +@dataclass(frozen=True) +class ActiveContext: + organization_id: str + project_id: str | None = None + organization_name: str | None = None + project_name: str | None = None + + +def context_path() -> Path: + return config.credentials_path().with_name("context.json") + + +def load_context() -> dict[str, Any]: + path = context_path() + if not path.exists(): + return {} + try: + data = json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + return {} + return data if isinstance(data, dict) else {} + + +def save_context(context: ActiveContext) -> Path: + data = { + "organization_id": context.organization_id, + "organization_name": context.organization_name, + "project_id": context.project_id, + "project_name": context.project_name, + } + data = {key: value for key, value in data.items() if value} + + path = context_path() + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(json.dumps(data, indent=2) + "\n") + return path + + +def clear_context() -> bool: + path = context_path() + if path.exists(): + path.unlink() + return True + return False diff --git a/src/judgment_cli/context_resolver.py b/src/judgment_cli/context_resolver.py new file mode 100644 index 0000000..2967c79 --- /dev/null +++ b/src/judgment_cli/context_resolver.py @@ -0,0 +1,286 @@ +"""Resolve organization/project context for generated CLI commands.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import click + +from judgment_cli import context as context_store +from judgment_cli.env import optional_env_var +from judgment_cli.generated.api import organizations_list, projects_list +from judgment_cli.generated.types import ( + OrganizationsListResponseOrganizationsItem as OrganizationRecord, +) +from judgment_cli.generated.types import ( + ProjectsListResponseProjectsItem as ProjectRecord, +) + + +ORG_ENV_VAR = "JUDGMENT_ORG_ID" +PROJECT_ENV_VAR = "JUDGMENT_PROJECT_ID" + + +@dataclass(frozen=True) +class ParsedContextualArgs: + organization_id: str | None + project_id: str | None + values: dict[str, str] + + +def parse_contextual_positionals( + args: tuple[str, ...], + *, + positional_names: list[str], + needs_organization_id: bool, + needs_project_id: bool, + organization_id: str | None = None, + project_id: str | None = None, +) -> ParsedContextualArgs: + """Split optional leading org/project IDs from command-specific args.""" + values = list(args) + expected = len(positional_names) + extra = len(values) - expected + max_extra = int(needs_organization_id) + int(needs_project_id) + + if extra < 0: + missing = positional_names[len(values) :] + raise click.UsageError(f"Missing argument(s): {', '.join(missing)}") + if extra > max_extra: + raise click.UsageError( + f"Got {len(values)} positional arguments, expected at most " + f"{expected + max_extra}." + ) + + positional_organization_id: str | None = None + positional_project_id: str | None = None + if needs_organization_id and needs_project_id: + if extra == 2: + positional_organization_id = values[0] + positional_project_id = values[1] + elif extra == 1: + positional_project_id = values[0] + elif needs_organization_id and extra == 1: + positional_organization_id = values[0] + elif needs_project_id and extra == 1: + positional_project_id = values[0] + + if ( + organization_id + and positional_organization_id + and organization_id != positional_organization_id + ): + raise click.UsageError( + "Pass organization ID either positionally or with --organization-id, not both." + ) + if project_id and positional_project_id and project_id != positional_project_id: + raise click.UsageError( + "Pass project ID either positionally or with --project-id, not both." + ) + + command_values = values[extra:] + return ParsedContextualArgs( + organization_id=organization_id or positional_organization_id, + project_id=project_id or positional_project_id, + values=dict(zip(positional_names, command_values)), + ) + + +def resolve_context( + client: Any, + *, + organization_id: str | None = None, + organization_name: str | None = None, + project_id: str | None = None, + project_name: str | None = None, + require_project: bool = False, +) -> context_store.ActiveContext: + """Resolve org/project IDs from CLI values, env vars, context, or API data.""" + if organization_id and organization_name: + raise click.ClickException( + "Pass only one of --organization-id or --organization." + ) + if project_id and project_name: + raise click.ClickException("Pass only one of --project-id or --project.") + + saved = context_store.load_context() + env_org_id = optional_env_var(ORG_ENV_VAR) + env_project_id = optional_env_var(PROJECT_ENV_VAR) + + org: OrganizationRecord | None = None + project: ProjectRecord | None = None + + if organization_name: + organization_matches = [ + organization + for organization in organizations_list(client)["organizations"] + if organization["detail"]["name"].casefold() + == organization_name.casefold() + ] + if len(organization_matches) > 1: + raise click.ClickException( + f"Multiple organizations named {organization_name!r}; pass the ID instead." + ) + if not organization_matches: + raise click.ClickException( + f"No organization named {organization_name!r} was found." + ) + org = organization_matches[0] + organization_id = org["organization_id"] + elif not organization_id: + organization_id = env_org_id or _str_or_none(saved.get("organization_id")) + + if project_name: + if organization_id: + projects = projects_list(client, organization_id)["projects"] + project_matches = [ + candidate + for candidate in projects + if candidate["project_name"].casefold() == project_name.casefold() + ] + if len(project_matches) > 1: + raise click.ClickException( + f"Multiple projects named {project_name!r}; pass the ID instead." + ) + if not project_matches: + raise click.ClickException( + f"No project named {project_name!r} was found." + ) + project = project_matches[0] + project_id = project["project_id"] + else: + org, project = _find_project_across_organizations( + client, + project_name=project_name, + ) + organization_id = org["organization_id"] + project_id = project["project_id"] + elif not project_id: + project_id = env_project_id + saved_project_id = _str_or_none(saved.get("project_id")) + saved_organization_id = _str_or_none(saved.get("organization_id")) + if ( + not project_id + and saved_project_id + and (not organization_id or saved_organization_id == organization_id) + ): + project_id = saved_project_id + + if not organization_id and project_id: + org, project = _find_project_across_organizations( + client, + project_id=project_id, + ) + organization_id = org["organization_id"] + project_id = project["project_id"] + + if not organization_id: + organizations = organizations_list(client)["organizations"] + if len(organizations) == 1: + org = organizations[0] + organization_id = org["organization_id"] + else: + raise click.ClickException(_organization_help(organizations)) + + if not require_project: + return context_store.ActiveContext( + organization_id=organization_id, + organization_name=(org["detail"]["name"] if org else None) + or _saved_name(saved, "organization", organization_id), + ) + + if not project_id: + saved_project_id = _str_or_none(saved.get("project_id")) + if ( + saved_project_id + and saved.get("organization_id") == organization_id + ): + project_id = saved_project_id + else: + projects = projects_list(client, organization_id)["projects"] + if len(projects) == 1: + project = projects[0] + project_id = project["project_id"] + else: + raise click.ClickException(_project_help(projects, organization_id)) + + return context_store.ActiveContext( + organization_id=organization_id, + project_id=project_id, + organization_name=(org["detail"]["name"] if org else None) + or _saved_name(saved, "organization", organization_id), + project_name=(project["project_name"] if project else None) + or _saved_name(saved, "project", project_id), + ) + + +def _find_project_across_organizations( + client: Any, + *, + project_id: str | None = None, + project_name: str | None = None, +) -> tuple[OrganizationRecord, ProjectRecord]: + matches: list[tuple[OrganizationRecord, ProjectRecord]] = [] + for organization in organizations_list(client)["organizations"]: + oid = organization["organization_id"] + for project in projects_list(client, oid)["projects"]: + if project_id and project["project_id"] == project_id: + matches.append((organization, project)) + elif ( + project_name + and project["project_name"].casefold() == project_name.casefold() + ): + matches.append((organization, project)) + + label = project_id or project_name or "project" + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + raise click.ClickException( + f"Multiple projects matched {label!r}; pass --organization-id too." + ) + raise click.ClickException(f"No project matched {label!r}.") + + +def _organization_help(organizations: list[OrganizationRecord]) -> str: + if not organizations: + return "No organizations were found for this account." + choices = "\n".join( + f" {organization['detail']['name']} {organization['organization_id']}" + for organization in organizations[:10] + ) + return ( + "No organization selected. Run `judgment context set`, pass " + "--organization-id, or set JUDGMENT_ORG_ID.\n\nOrganizations:\n" + f"{choices}" + ) + + +def _project_help(projects: list[ProjectRecord], organization_id: str) -> str: + if not projects: + return f"No projects were found for organization {organization_id}." + choices = [] + for project in projects[:10]: + traces = project["total_traces"] + suffix = f" {int(traces):,} traces" if traces is not None else "" + choices.append(f" {project['project_name']}{suffix} {project['project_id']}") + choices_text = "\n".join(choices) + return ( + "No project selected. Run `judgment context set`, pass --project-id, " + "or set JUDGMENT_PROJECT_ID.\n\nProjects:\n" + choices_text + ) + + +def _saved_name( + saved: dict[str, Any], + prefix: str, + entity_id: str | None, +) -> str | None: + if saved.get(f"{prefix}_id") != entity_id: + return None + return _str_or_none(saved.get(f"{prefix}_name")) + + +def _str_or_none(value: object) -> str | None: + return value if isinstance(value, str) and value else None diff --git a/src/judgment_cli/generated/__init__.py b/src/judgment_cli/generated/__init__.py new file mode 100644 index 0000000..3baac50 --- /dev/null +++ b/src/judgment_cli/generated/__init__.py @@ -0,0 +1 @@ +# Auto-generated package for OpenAPI-derived CLI code. diff --git a/src/judgment_cli/generated/api.py b/src/judgment_cli/generated/api.py new file mode 100644 index 0000000..15a7fa0 --- /dev/null +++ b/src/judgment_cli/generated/api.py @@ -0,0 +1,548 @@ +# Auto-generated by scripts/generate_cli.py +# DO NOT EDIT MANUALLY + +from __future__ import annotations + +import json + +from typing import cast + +from judgment_cli.client import JudgmentClient +from judgment_cli.generated.types import * # noqa: F403 + +def agent_threads_get(client: JudgmentClient, body: AgentThreadsGetBody) -> AgentThreadsGetResponse: + return cast( + AgentThreadsGetResponse, + client.request( + 'POST', + '/agent-threads/get', + json_body=body, + ), + ) + + +def agent_threads_list(client: JudgmentClient, body: AgentThreadsListBody) -> AgentThreadsListResponse: + return cast( + AgentThreadsListResponse, + client.request( + 'POST', + '/agent-threads/list', + json_body=body, + ), + ) + + +def automations_create(client: JudgmentClient, body: AutomationsCreateBody) -> AutomationsCreateResponse: + return cast( + AutomationsCreateResponse, + client.request( + 'POST', + '/automations/create', + json_body=body, + ), + ) + + +def automations_delete(client: JudgmentClient, body: AutomationsDeleteBody) -> AutomationsDeleteResponse: + return cast( + AutomationsDeleteResponse, + client.request( + 'POST', + '/automations/delete', + json_body=body, + ), + ) + + +def automations_get(client: JudgmentClient, body: AutomationsGetBody) -> AutomationsGetResponse: + return cast( + AutomationsGetResponse, + client.request( + 'POST', + '/automations/detail', + json_body=body, + ), + ) + + +def automations_list(client: JudgmentClient, body: AutomationsListBody) -> AutomationsListResponse: + return cast( + AutomationsListResponse, + client.request( + 'POST', + '/automations/list', + json_body=body, + ), + ) + + +def automations_update(client: JudgmentClient, body: AutomationsUpdateBody) -> AutomationsUpdateResponse: + return cast( + AutomationsUpdateResponse, + client.request( + 'POST', + '/automations/update', + json_body=body, + ), + ) + + +def behaviors_create_binary(client: JudgmentClient, body: BehaviorsCreateBinaryBody) -> BehaviorsCreateBinaryResponse: + return cast( + BehaviorsCreateBinaryResponse, + client.request( + 'POST', + '/behaviors/create-binary', + json_body=body, + ), + ) + + +def behaviors_create_classifier(client: JudgmentClient, body: BehaviorsCreateClassifierBody) -> BehaviorsCreateClassifierResponse: + return cast( + BehaviorsCreateClassifierResponse, + client.request( + 'POST', + '/behaviors/create-classifier', + json_body=body, + ), + ) + + +def behaviors_delete(client: JudgmentClient, body: BehaviorsDeleteBody) -> BehaviorsDeleteResponse: + return cast( + BehaviorsDeleteResponse, + client.request( + 'POST', + '/behaviors/delete', + json_body=body, + ), + ) + + +def behaviors_get(client: JudgmentClient, body: BehaviorsGetBody) -> BehaviorsGetResponse: + return cast( + BehaviorsGetResponse, + client.request( + 'POST', + '/behaviors/detail', + json_body=body, + ), + ) + + +def behaviors_list(client: JudgmentClient, body: BehaviorsListBody) -> BehaviorsListResponse: + return cast( + BehaviorsListResponse, + client.request( + 'POST', + '/behaviors/list', + json_body=body, + ), + ) + + +def behaviors_update(client: JudgmentClient, body: BehaviorsUpdateBody) -> BehaviorsUpdateResponse: + return cast( + BehaviorsUpdateResponse, + client.request( + 'POST', + '/behaviors/update', + json_body=body, + ), + ) + + +def docs_get_page(client: JudgmentClient, path: str) -> DocsGetPageResponse: + return cast( + DocsGetPageResponse, + client.request( + 'GET', + '/docs/page', + params={'path': path}, + ), + ) + + +def docs_search(client: JudgmentClient, body: DocsSearchBody) -> DocsSearchResponse: + return cast( + DocsSearchResponse, + client.request( + 'POST', + '/docs/search', + json_body=body, + ), + ) + + +def judges_create(client: JudgmentClient, body: JudgesCreateBody) -> JudgesCreateResponse: + return cast( + JudgesCreateResponse, + client.request( + 'POST', + '/judges/create', + json_body=body, + ), + ) + + +def judges_delete(client: JudgmentClient, body: JudgesDeleteBody) -> JudgesDeleteResponse: + return cast( + JudgesDeleteResponse, + client.request( + 'POST', + '/judges/delete', + json_body=body, + ), + ) + + +def judges_get(client: JudgmentClient, body: JudgesGetBody) -> JudgesGetResponse: + return cast( + JudgesGetResponse, + client.request( + 'POST', + '/judges/get', + json_body=body, + ), + ) + + +def judges_get_settings(client: JudgmentClient, body: JudgesGetSettingsBody) -> JudgesGetSettingsResponse: + return cast( + JudgesGetSettingsResponse, + client.request( + 'POST', + '/judges/settings', + json_body=body, + ), + ) + + +def judges_list(client: JudgmentClient, body: JudgesListBody) -> JudgesListResponse: + return cast( + JudgesListResponse, + client.request( + 'POST', + '/judges/list', + json_body=body, + ), + ) + + +def judges_models(client: JudgmentClient, organization_id: str) -> JudgesModelsResponse: + return cast( + JudgesModelsResponse, + client.request( + 'GET', + '/judges/models', + params={'organization_id': organization_id}, + ), + ) + + +def judges_set_tag(client: JudgmentClient, body: JudgesSetTagBody) -> JudgesSetTagResponse: + return cast( + JudgesSetTagResponse, + client.request( + 'POST', + '/judges/set-tag', + json_body=body, + ), + ) + + +def judges_update(client: JudgmentClient, body: JudgesUpdateBody) -> JudgesUpdateResponse: + return cast( + JudgesUpdateResponse, + client.request( + 'POST', + '/judges/update', + json_body=body, + ), + ) + + +def judges_update_settings(client: JudgmentClient, body: JudgesUpdateSettingsBody) -> JudgesUpdateSettingsResponse: + return cast( + JudgesUpdateSettingsResponse, + client.request( + 'POST', + '/judges/update-settings', + json_body=body, + ), + ) + + +def judges_upload(client: JudgmentClient, body: JudgesUploadBody) -> JudgesUploadResponse: + data: dict[str, str] = {} + files: dict[str, tuple[str, bytes, str]] = {} + organization_id = body['organization_id'] + data['organization_id'] = str(organization_id) + project_id = body['project_id'] + data['project_id'] = str(project_id) + metadata = body['metadata'] + data['metadata'] = json.dumps(metadata) + bundle = body['bundle'] + files['bundle'] = bundle + return cast( + JudgesUploadResponse, + client.multipart( + 'POST', + '/judges/upload', + data=data, + files=files, + ), + ) + + +def organizations_list(client: JudgmentClient) -> OrganizationsListResponse: + return cast( + OrganizationsListResponse, + client.request( + 'GET', + '/organizations', + ), + ) + + +def projects_add_favorite(client: JudgmentClient, body: ProjectsAddFavoriteBody) -> ProjectsAddFavoriteResponse: + return cast( + ProjectsAddFavoriteResponse, + client.request( + 'POST', + '/projects/add-favorite', + json_body=body, + ), + ) + + +def projects_create(client: JudgmentClient, body: ProjectsCreateBody) -> ProjectsCreateResponse: + return cast( + ProjectsCreateResponse, + client.request( + 'POST', + '/projects/create', + json_body=body, + ), + ) + + +def projects_list(client: JudgmentClient, organization_id: str) -> ProjectsListResponse: + return cast( + ProjectsListResponse, + client.request( + 'GET', + '/projects', + params={'organization_id': organization_id}, + ), + ) + + +def projects_remove_favorite(client: JudgmentClient, body: ProjectsRemoveFavoriteBody) -> ProjectsRemoveFavoriteResponse: + return cast( + ProjectsRemoveFavoriteResponse, + client.request( + 'POST', + '/projects/remove-favorite', + json_body=body, + ), + ) + + +def prompts_commit(client: JudgmentClient, body: PromptsCommitBody) -> PromptsCommitResponse: + return cast( + PromptsCommitResponse, + client.request( + 'POST', + '/prompts/commit', + json_body=body, + ), + ) + + +def prompts_get(client: JudgmentClient, body: PromptsGetBody) -> PromptsGetResponse: + return cast( + PromptsGetResponse, + client.request( + 'POST', + '/prompts/get', + json_body=body, + ), + ) + + +def prompts_list(client: JudgmentClient, body: PromptsListBody) -> PromptsListResponse: + return cast( + PromptsListResponse, + client.request( + 'POST', + '/prompts/list', + json_body=body, + ), + ) + + +def prompts_tag(client: JudgmentClient, body: PromptsTagBody) -> PromptsTagResponse: + return cast( + PromptsTagResponse, + client.request( + 'POST', + '/prompts/tag', + json_body=body, + ), + ) + + +def prompts_untag(client: JudgmentClient, body: PromptsUntagBody) -> PromptsUntagResponse: + return cast( + PromptsUntagResponse, + client.request( + 'POST', + '/prompts/untag', + json_body=body, + ), + ) + + +def prompts_versions(client: JudgmentClient, body: PromptsVersionsBody) -> PromptsVersionsResponse: + return cast( + PromptsVersionsResponse, + client.request( + 'POST', + '/prompts/versions', + json_body=body, + ), + ) + + +def sessions_get(client: JudgmentClient, body: SessionsGetBody) -> SessionsGetResponseOption1 | None: + return cast( + SessionsGetResponseOption1 | None, + client.request( + 'POST', + '/sessions/detail', + json_body=body, + ), + ) + + +def sessions_search(client: JudgmentClient, body: SessionsSearchBody) -> SessionsSearchResponse: + return cast( + SessionsSearchResponse, + client.request( + 'POST', + '/sessions/search', + json_body=body, + ), + ) + + +def sessions_trace_behaviors(client: JudgmentClient, body: SessionsTraceBehaviorsBody) -> dict[str, SessionsTraceBehaviorsResponseValue]: + return cast( + dict[str, SessionsTraceBehaviorsResponseValue], + client.request( + 'POST', + '/sessions/trace-behaviors', + json_body=body, + ), + ) + + +def sessions_trace_ids(client: JudgmentClient, body: SessionsTraceIdsBody) -> SessionsTraceIdsResponse: + return cast( + SessionsTraceIdsResponse, + client.request( + 'POST', + '/sessions/trace-ids', + json_body=body, + ), + ) + + +def traces_add_tags(client: JudgmentClient, body: TracesAddTagsBody) -> TracesAddTagsResponse: + return cast( + TracesAddTagsResponse, + client.request( + 'POST', + '/traces/add-tags', + json_body=body, + ), + ) + + +def traces_behaviors(client: JudgmentClient, body: TracesBehaviorsBody) -> list[TracesBehaviorsResponseItem]: + return cast( + list[TracesBehaviorsResponseItem], + client.request( + 'POST', + '/traces/behaviors', + json_body=body, + ), + ) + + +def traces_evaluate(client: JudgmentClient, body: TracesEvaluateBody) -> TracesEvaluateResponse: + return cast( + TracesEvaluateResponse, + client.request( + 'POST', + '/traces/evaluate', + json_body=body, + ), + ) + + +def traces_get(client: JudgmentClient, body: TracesGetBody) -> TracesGetResponseOption1 | None: + return cast( + TracesGetResponseOption1 | None, + client.request( + 'POST', + '/traces/detail', + json_body=body, + ), + ) + + +def traces_search(client: JudgmentClient, body: TracesSearchBody) -> TracesSearchResponse: + return cast( + TracesSearchResponse, + client.request( + 'POST', + '/traces/search', + json_body=body, + ), + ) + + +def traces_span(client: JudgmentClient, body: TracesSpanBody) -> list[TracesSpanResponseItemOption1 | None]: + return cast( + list[TracesSpanResponseItemOption1 | None], + client.request( + 'POST', + '/traces/span', + json_body=body, + ), + ) + + +def traces_spans(client: JudgmentClient, body: TracesSpansBody) -> list[TracesSpansResponseItem]: + return cast( + list[TracesSpansResponseItem], + client.request( + 'POST', + '/traces/spans', + json_body=body, + ), + ) + + +def traces_tags(client: JudgmentClient, body: TracesTagsBody) -> list[str]: + return cast( + list[str], + client.request( + 'POST', + '/traces/tags', + json_body=body, + ), + ) diff --git a/src/judgment_cli/generated/commands.py b/src/judgment_cli/generated/commands.py new file mode 100644 index 0000000..4d4e8f4 --- /dev/null +++ b/src/judgment_cli/generated/commands.py @@ -0,0 +1,2002 @@ +# Auto-generated by scripts/generate_cli.py +# DO NOT EDIT MANUALLY + +from __future__ import annotations + +import json + +import click + +from judgment_cli.generated import api as _api +from judgment_cli.context_resolver import parse_contextual_positionals as _parse_contextual_positionals +from judgment_cli.context_resolver import resolve_context as _resolve_context +from judgment_cli.ui import table_output as _table_output, yaml_output as _yaml_output + + +# ──────────────────────────────────────────────────────────────────── +# Group: agent-threads +# ──────────────────────────────────────────────────────────────────── + + +@click.group("agent-threads") +def agent_threads_group() -> None: + 'List and inspect agent thread conversations (global_copilot, custom_agent).' + + +@agent_threads_group.command("get") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID] THREAD_ID') +@click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") +@click.pass_context +def agent_threads_get(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): + 'Get an agent thread.\n\n\x08\nGet one agent thread conversation, including its transcript, metadata, active run status, and timestamps.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['thread_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + thread_id = _parsed.values["thread_id"] + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + body["thread_id"] = thread_id + result = _api.agent_threads_get(ctx.obj["client"], body) + _yaml_output(result, output_format=output_format) + + +@agent_threads_group.command("list") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID]') +@click.option("--agent-type", "agent_type", required=True, type=click.Choice(['global_copilot', 'custom_agent']), help='Active agent thread kinds available for new conversations: `global_copilot` or `custom_agent`.') +@click.option("--agent-name", "agent_name", required=True) +@click.option("--judge-id", "judge_id", default=None, help='Restrict to threads associated with this judge.') +@click.option("--agent-config-id", "agent_config_id", default=None, help='Restrict to threads for this exact agent config.') +@click.option("--scope", "scope", default=None, type=click.Choice(['owner', 'project'])) +@click.option("--owner-user-id", "owner_user_id", default=None, help='Restrict project history to a specific thread owner.') +@click.option("--all-users", "all_users", default=None, type=bool, help='Deprecated alias for scope=project. Prefer the `scope` query parameter.') +@click.option("--limit", "limit", default=None, type=float, help='Maximum number of threads to return (1–100).') +@click.option("--cursor-updated-at", "cursor_updated_at", default=None, help='Pagination cursor: `updated_at` value from a previous `next_cursor`.') +@click.option("--cursor-thread-id", "cursor_thread_id", default=None, help='Pagination cursor: `thread_id` value from a previous `next_cursor`.') +@click.option("-o", "--output", "output_format", type=click.Choice(["table", "yaml", "json"]), default="table", help="Output format.") +@click.pass_context +def agent_threads_list(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, agent_type, agent_name, judge_id, agent_config_id, scope, owner_user_id, all_users, limit, cursor_updated_at, cursor_thread_id): + "List agent thread conversations.\n\n\x08\nList the authenticated user's agent thread conversations in a project (global_copilot or custom_agent). Returns each thread's title, type, message count, active run status, and timestamps." + _parsed = _parse_contextual_positionals( + _args, + positional_names=[], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + body["agent_type"] = agent_type + body["agent_name"] = agent_name + if judge_id is not None: + body["judge_id"] = judge_id + if agent_config_id is not None: + body["agent_config_id"] = agent_config_id + if scope is not None: + body["scope"] = scope + if owner_user_id is not None: + body["owner_user_id"] = owner_user_id + if all_users is not None: + body["all_users"] = all_users + if limit is not None: + body["limit"] = limit + if cursor_updated_at is not None: + body["cursor_updated_at"] = cursor_updated_at + if cursor_thread_id is not None: + body["cursor_thread_id"] = cursor_thread_id + result = _api.agent_threads_list(ctx.obj["client"], body) + _table_output(result, output_format=output_format) + + +# ──────────────────────────────────────────────────────────────────── +# Group: automations +# ──────────────────────────────────────────────────────────────────── + + +@click.group("automations") +def automations_group() -> None: + 'Manage automations (rules) that fire actions when metrics match conditions.' + + +@automations_group.command("create") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID] NAME') +@click.option("--description", "description", default=None, help='Human-readable description shown in the UI.') +@click.option("--conditions", "conditions", required=True, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on `combine_type` (`all` vs `any`).\n\n**Condition shape:**\n```\n{\n "metric": {\n "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n "name": "",\n "threshold": ?\n },\n "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n}\n```\n\n**Common scorer_type values:**\n- `behavior` — Judge-scored behavior (name = behavior name, e.g. "Relevance")\n- `static` — Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n- `prompt`/`custom` — Prompt or custom scorer by name\n- `span_attribute` — Arbitrary span attribute key (name = attribute key)\n- `error` — Span error condition') +@click.option("--combine-type", "combine_type", required=True, type=click.Choice(['all', 'any'])) +@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n**Shape:**\n```\n{\n "notification": {\n "enabled": ?,\n "communication_methods": ["email" | "slack" | "pagerduty"],\n "email_addresses": ["", ...]?,\n "pagerduty_config": {"routing_key":"","severity":"critical"|"error"|"warning"|"info"}?\n }?,\n "dataset_addition": {\n "enabled": ?,\n "dataset_name": "",\n "metadata_fields": ?\n }?,\n "behavior_evaluation": {\n "enabled": ?,\n "behavior_judge_names": ["", ...]\n }?\n}\n```\n\nSlack notifications are configured per-organization in the Judgment UI; pass `"slack"` in `communication_methods` to use them.') +@click.option("--cooldown-period", "cooldown_period", default=None, help='JSON object describing the minimum wait between triggers. Omit to leave the cooldown unset; if provided, both `value` and `unit` are required.\n\n**Shape:** `{ "value": , "unit": "seconds" | "minutes" | "hours" | "days" }`\n\nExample: `{ "value": 15, "unit": "minutes" }` (at least 15 min between triggers)') +@click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON object describing the rate-limit window. Omit to leave unset; if provided, all three fields are required.\n\n**Shape:** `{ "count": , "period": , "period_unit": "seconds" | "minutes" | "hours" | "days" }`\n\nExample: `{ "count": 5, "period": 1, "period_unit": "hours" }` (max 5 triggers per 1 hour)') +@click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") +@click.pass_context +def automations_create(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, description, conditions, combine_type, actions, cooldown_period, trigger_frequency): + 'Create an automation.\n\n\x08\nCreate an automation (rule) in a project. An automation watches behavior/latency/cost metrics and fires actions when its conditions match. Requires the developer role.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['name'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + name = _parsed.values["name"] + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + body["name"] = name + if description is not None: + body["description"] = description + body["conditions"] = json.loads(conditions) + body["combine_type"] = combine_type + if actions is not None: + body["actions"] = json.loads(actions) + if cooldown_period is not None: + body["cooldown_period"] = json.loads(cooldown_period) + if trigger_frequency is not None: + body["trigger_frequency"] = json.loads(trigger_frequency) + result = _api.automations_create(ctx.obj["client"], body) + _yaml_output(result, output_format=output_format) + + +@automations_group.command("delete") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID] RULE_ID') +@click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") +@click.pass_context +def automations_delete(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): + 'Delete an automation.\n\n\x08\nDelete an automation. Requires the admin role.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['rule_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + rule_id = _parsed.values["rule_id"] + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + body["rule_id"] = rule_id + result = _api.automations_delete(ctx.obj["client"], body) + _yaml_output(result, output_format=output_format) + + +@automations_group.command("get") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID] RULE_ID') +@click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") +@click.pass_context +def automations_get(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): + """Get an automation by ID.""" + _parsed = _parse_contextual_positionals( + _args, + positional_names=['rule_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + rule_id = _parsed.values["rule_id"] + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + body["rule_id"] = rule_id + result = _api.automations_get(ctx.obj["client"], body) + _yaml_output(result, output_format=output_format) + + +@automations_group.command("list") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID]') +@click.option("-o", "--output", "output_format", type=click.Choice(["table", "yaml", "json"]), default="table", help="Output format.") +@click.pass_context +def automations_list(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option): + """List automations.""" + _parsed = _parse_contextual_positionals( + _args, + positional_names=[], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + result = _api.automations_list(ctx.obj["client"], body) + _table_output(result, output_format=output_format) + + +@automations_group.command("update") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID] RULE_ID') +@click.option("--name", "name", default=None, help='New name for the automation.') +@click.option("--description", "description", default=None, help='New description for the automation.') +@click.option("--conditions", "conditions", default=None, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on `combine_type` (`all` vs `any`).\n\n**Condition shape:**\n```\n{\n "metric": {\n "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n "name": "",\n "threshold": ?\n },\n "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n}\n```\n\n**Common scorer_type values:**\n- `behavior` — Judge-scored behavior (name = behavior name, e.g. "Relevance")\n- `static` — Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n- `prompt`/`custom` — Prompt or custom scorer by name\n- `span_attribute` — Arbitrary span attribute key (name = attribute key)\n- `error` — Span error condition') +@click.option("--combine-type", "combine_type", default=None, type=click.Choice(['all', 'any'])) +@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n**Shape:**\n```\n{\n "notification": {\n "enabled": ?,\n "communication_methods": ["email" | "slack" | "pagerduty"],\n "email_addresses": ["", ...]?,\n "pagerduty_config": {"routing_key":"","severity":"critical"|"error"|"warning"|"info"}?\n }?,\n "dataset_addition": {\n "enabled": ?,\n "dataset_name": "",\n "metadata_fields": ?\n }?,\n "behavior_evaluation": {\n "enabled": ?,\n "behavior_judge_names": ["", ...]\n }?\n}\n```\n\nSlack notifications are configured per-organization in the Judgment UI; pass `"slack"` in `communication_methods` to use them.') +@click.option("--active", "active", default=None, type=bool, help='Enable (true) or disable (false) the automation without modifying other fields.') +@click.option("--cooldown-period", "cooldown_period", default=None, help='JSON 2-tuple `[period, unit]` describing the minimum wait between triggers. Omit to leave unchanged.\n\n**Shape:** `[, ]`\n\nExample: `[15, "minutes"]` (at least 15 min between triggers)') +@click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON 3-tuple `[count, period, unit]` describing the rate-limit window. Omit to leave unchanged.\n\n**Shape:** `[, , ]`\n\nExample: `[5, 1, "hours"]` (max 5 triggers per 1 hour)') +@click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") +@click.pass_context +def automations_update(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, name, description, conditions, combine_type, actions, active, cooldown_period, trigger_frequency): + 'Update an automation.\n\n\x08\nUpdate an existing automation. All fields other than the IDs are optional — only supplied fields are applied. Use `active: true/false` to enable or disable without changing other fields. Requires the developer role.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['rule_id'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + rule_id = _parsed.values["rule_id"] + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + body["rule_id"] = rule_id + if name is not None: + body["name"] = name + if description is not None: + body["description"] = description + if conditions is not None: + body["conditions"] = json.loads(conditions) + if combine_type is not None: + body["combine_type"] = combine_type + if actions is not None: + body["actions"] = json.loads(actions) + if active is not None: + body["active"] = active + if cooldown_period is not None: + body["cooldown_period"] = json.loads(cooldown_period) + if trigger_frequency is not None: + body["trigger_frequency"] = json.loads(trigger_frequency) + result = _api.automations_update(ctx.obj["client"], body) + _yaml_output(result, output_format=output_format) + + +# ──────────────────────────────────────────────────────────────────── +# Group: behaviors +# ──────────────────────────────────────────────────────────────────── + + +@click.group("behaviors") +def behaviors_group() -> None: + 'View and manage behaviors (the binary or categorical labels judges assign).' + + +@behaviors_group.command("create-binary") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID] NAME PROMPT') +@click.option("--description", "description", default=None, help='Human-readable description shown in the UI.') +@click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.3-codex" when omitted.') +@click.option("--category-ids", "category_ids", multiple=True, help='UUIDs of categories to attach the behavior to. Pass an array of category UUIDs.') +@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this is supplied.\n\n**Shape:**\n```\n{\n "online_evaluation_mode": "continuous" | "on_demand",\n "online_sampling_rate": ,\n "online_span_triggers": [\n {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"","key":""?}\n ],\n "online_session_scoring": \n}\n```\n\n`continuous` runs the judge automatically on qualifying spans; `on_demand` requires a manual `judgment traces evaluate` call. `online_sampling_rate` is a percent (0–100) of matching spans to score.') +@click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be `score_type=binary` and have no existing behaviors.') +@click.option("-o", "--output", "output_format", type=click.Choice(["yaml", "json"]), default="yaml", help="Output format.") +@click.pass_context +def behaviors_create_binary(ctx, _args, output_format, organization_id_option, organization_name_option, project_id_option, project_name_option, description, model, category_ids, advanced_settings, judge_id): + 'Create a binary (yes/no) behavior.\n\n\x08\nCreate a binary behavior. The judge LLM uses your prompt to decide true/false on each qualifying span.' + _parsed = _parse_contextual_positionals( + _args, + positional_names=['name', 'prompt'], + needs_organization_id=True, + needs_project_id=True, + organization_id=organization_id_option, + project_id=project_id_option, + ) + _context = _resolve_context( + ctx.obj["client"], + organization_id=_parsed.organization_id, + organization_name=organization_name_option, + project_id=_parsed.project_id, + project_name=project_name_option, + require_project=True, + ) + organization_id = _context.organization_id + project_id = _context.project_id + name = _parsed.values["name"] + prompt = _parsed.values["prompt"] + body = {} + body["organization_id"] = organization_id + body["project_id"] = project_id + body["name"] = name + body["prompt"] = prompt + if description is not None: + body["description"] = description + if model is not None: + body["model"] = model + if category_ids: + body["category_ids"] = list(category_ids) + if advanced_settings is not None: + body["advanced_settings"] = json.loads(advanced_settings) + if judge_id is not None: + body["judge_id"] = judge_id + result = _api.behaviors_create_binary(ctx.obj["client"], body) + _yaml_output(result, output_format=output_format) + + +@behaviors_group.command("create-classifier") +@click.option("--organization-id", "--org-id", "organization_id_option", default=None, help="Organization ID. Defaults to JUDGMENT_ORG_ID or saved context.") +@click.option("--organization", "--org", "organization_name_option", default=None, help="Organization name to resolve.") +@click.option("--project-id", "project_id_option", default=None, help="Project ID. Defaults to JUDGMENT_PROJECT_ID or saved context.") +@click.option("--project", "project_name_option", default=None, help="Project name to resolve.") +@click.argument("_args", nargs=-1, metavar='[[ORG_ID] PROJECT_ID] NAME PROMPT') +@click.option("--options", "options", required=True, help='JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.\n\n**Shape:**\n```\n[\n {"name":"