From e1532b31932418ed3126f3a606b205c072161d51 Mon Sep 17 00:00:00 2001 From: Cascade Bot Date: Wed, 5 Aug 2026 14:47:57 +0000 Subject: [PATCH] fix(backends): honor inherited model label + warn on engine/model incompatibility --- docs/adding-engines.md | 21 ++++- src/backends/catalog.ts | 28 +++++- src/backends/claude-code/index.ts | 13 ++- src/backends/claude-code/models.ts | 8 ++ src/backends/codex/index.ts | 4 +- src/backends/codex/models.ts | 11 +++ src/backends/types.ts | 8 ++ tests/unit/backends/catalog.test.ts | 60 +++++++++++++ tests/unit/web/agent-config-utils.test.ts | 86 +++++++++++++++++++ tests/unit/web/model-field.test.ts | 29 +++++++ .../projects/agent-config-detail.tsx | 25 +++++- .../components/projects/agent-config-types.ts | 10 +++ .../components/projects/agent-config-utils.ts | 23 +++++ .../projects/project-harness-form.tsx | 2 +- web/src/components/settings/model-field.tsx | 25 +++++- 15 files changed, 340 insertions(+), 13 deletions(-) diff --git a/docs/adding-engines.md b/docs/adding-engines.md index f7a872f65..2e8929954 100644 --- a/docs/adding-engines.md +++ b/docs/adding-engines.md @@ -72,14 +72,33 @@ export const MY_ENGINE_DEFINITION: AgentEngineDefinition = { ], modelSelection: { type: 'select', // or 'free-text' for open-ended model strings - defaultValueLabel: 'Default (v1.0)', + // Derive the label from your DEFAULT_*_MODEL constant + catalog lookup instead + // of hardcoding it, so the displayed default can never drift from the actually + // resolved default on the next model bump (MNG-1772). See catalog.ts's + // `defaultModelLabel(models, defaultId)` helper. + defaultValueLabel: defaultModelLabel(MY_ENGINE_MODELS, DEFAULT_MY_ENGINE_MODEL), options: MY_ENGINE_MODELS, // Imported from ./my-engine/models.ts + // Optional: model-ID prefixes your engine accepts in addition to catalog IDs. + // Single-source these next to your model list (e.g. MY_ENGINE_ACCEPTED_PREFIXES) + // and consume them in your resolveModel() so the runtime acceptance rules and + // the dashboard's config-time incompatibility warning share one definition. + // Must be a plain string[] — it serializes across the agentConfigs.engines tRPC + // query and is mirrored by the frontend's isModelCompatibleWithEngine(). + acceptedModelPrefixes: MY_ENGINE_ACCEPTED_PREFIXES, }, logLabel: 'My Engine Log', // Optional: add 'settings' if your engine has configurable fields }; ``` +> **Anti-drift + config-time warning contract (MNG-1772).** For `select`-type +> engines, deriving `defaultValueLabel` from the default-model constant keeps the +> UI honest, and exposing `acceptedModelPrefixes` lets the agent-config detail +> panel warn *before* dispatch when an inherited model is incompatible with the +> engine (instead of crashing mid-run). A guard test in +> `tests/unit/backends/catalog.test.ts` asserts the displayed default always +> matches the resolved default for every `select` engine. + Add it to `DEFAULT_ENGINE_CATALOG` at the bottom of the same file: ```typescript diff --git a/src/backends/catalog.ts b/src/backends/catalog.ts index 8a68516dc..59db6e77c 100644 --- a/src/backends/catalog.ts +++ b/src/backends/catalog.ts @@ -1,7 +1,25 @@ -import { CLAUDE_CODE_MODELS } from './claude-code/models.js'; -import { CODEX_MODELS } from './codex/models.js'; +import { + CLAUDE_CODE_ACCEPTED_PREFIXES, + CLAUDE_CODE_MODELS, + DEFAULT_CLAUDE_CODE_MODEL, +} from './claude-code/models.js'; +import { CODEX_ACCEPTED_PREFIXES, CODEX_MODELS, DEFAULT_CODEX_MODEL } from './codex/models.js'; import type { AgentEngineDefinition } from './types.js'; +/** + * Derive the `select`-engine empty-option label from the engine's default model + * constant, looking the display label up in the model catalog. This kills the + * hand-synced-literal drift class (MNG-1772/MNG-1770): the displayed default + * always matches the actually-resolved `DEFAULT_*_MODEL` on the next model bump. + */ +export function defaultModelLabel( + models: ReadonlyArray<{ value: string; label: string }>, + defaultId: string, +): string { + const match = models.find((m) => m.value === defaultId); + return `Default (${match?.label ?? defaultId})`; +} + export const LLMIST_ENGINE_DEFINITION: AgentEngineDefinition = { id: 'llmist', label: 'LLMist', @@ -35,8 +53,9 @@ export const CLAUDE_CODE_ENGINE_DEFINITION: AgentEngineDefinition = { ], modelSelection: { type: 'select', - defaultValueLabel: 'Default (Sonnet 5)', + defaultValueLabel: defaultModelLabel(CLAUDE_CODE_MODELS, DEFAULT_CLAUDE_CODE_MODEL), options: CLAUDE_CODE_MODELS, + acceptedModelPrefixes: CLAUDE_CODE_ACCEPTED_PREFIXES, }, logLabel: 'Claude Code Log', settings: { @@ -94,8 +113,9 @@ export const CODEX_ENGINE_DEFINITION: AgentEngineDefinition = { ], modelSelection: { type: 'select', - defaultValueLabel: 'Default (GPT-5.4)', + defaultValueLabel: defaultModelLabel(CODEX_MODELS, DEFAULT_CODEX_MODEL), options: CODEX_MODELS, + acceptedModelPrefixes: CODEX_ACCEPTED_PREFIXES, }, logLabel: 'Codex Log', settings: { diff --git a/src/backends/claude-code/index.ts b/src/backends/claude-code/index.ts index 83c3b128c..d4cc29e26 100644 --- a/src/backends/claude-code/index.ts +++ b/src/backends/claude-code/index.ts @@ -21,7 +21,11 @@ import { consumeStream, filterContextImages, } from './messageProcessing.js'; -import { CLAUDE_CODE_MODEL_IDS, DEFAULT_CLAUDE_CODE_MODEL } from './models.js'; +import { + CLAUDE_CODE_ACCEPTED_PREFIXES, + CLAUDE_CODE_MODEL_IDS, + DEFAULT_CLAUDE_CODE_MODEL, +} from './models.js'; import { ClaudeCodeSettingsSchema, resolveClaudeCodeSettings } from './settings.js'; export { @@ -40,8 +44,11 @@ export { buildPromptWithImages, formatErrorMessage } from './messageProcessing.j */ export function resolveClaudeModel(cascadeModel: string): string { if (CLAUDE_CODE_MODEL_IDS.includes(cascadeModel)) return cascadeModel; - if (cascadeModel.startsWith('claude-')) return cascadeModel; - if (cascadeModel.startsWith('anthropic:')) return cascadeModel.replace('anthropic:', ''); + if (CLAUDE_CODE_ACCEPTED_PREFIXES.some((prefix) => cascadeModel.startsWith(prefix))) { + return cascadeModel.startsWith('anthropic:') + ? cascadeModel.replace('anthropic:', '') + : cascadeModel; + } throw new Error( `Model "${cascadeModel}" is not compatible with the Claude Code engine. Configure a Claude-compatible model (e.g. "${DEFAULT_CLAUDE_CODE_MODEL}") or switch to a different engine.`, diff --git a/src/backends/claude-code/models.ts b/src/backends/claude-code/models.ts index 0d34b424b..8ba84189d 100644 --- a/src/backends/claude-code/models.ts +++ b/src/backends/claude-code/models.ts @@ -20,3 +20,11 @@ export const CLAUDE_CODE_MODELS = [ export const CLAUDE_CODE_MODEL_IDS: string[] = CLAUDE_CODE_MODELS.map((m) => m.value); export const DEFAULT_CLAUDE_CODE_MODEL = 'claude-sonnet-5'; + +/** + * Model-ID prefixes the Claude Code engine accepts in addition to catalog IDs. + * Single source of truth consumed by `resolveClaudeModel` (runtime acceptance) + * and surfaced on the engine definition as `acceptedModelPrefixes` so the + * dashboard can mirror the compatibility check without duplicating logic. + */ +export const CLAUDE_CODE_ACCEPTED_PREFIXES = ['claude-', 'anthropic:'] as const; diff --git a/src/backends/codex/index.ts b/src/backends/codex/index.ts index 9004b3cbb..6ff1dcb01 100644 --- a/src/backends/codex/index.ts +++ b/src/backends/codex/index.ts @@ -19,7 +19,7 @@ import { buildSystemPrompt, buildTaskPrompt } from '../shared/nativeToolPrompts. import type { AgentEngineResult, AgentExecutionPlan, LogWriter } from '../types.js'; import type { UsageSummary } from './jsonlParser.js'; import { extractUsage, parseCodexEvent } from './jsonlParser.js'; -import { CODEX_MODEL_IDS, DEFAULT_CODEX_MODEL } from './models.js'; +import { CODEX_ACCEPTED_PREFIXES, CODEX_MODEL_IDS, DEFAULT_CODEX_MODEL } from './models.js'; import { CODEX_COMPLETION_OUTPUT_SCHEMA, parseCodexCompletionReport } from './outputSchema.js'; import { assertHeadlessCodexSettings, @@ -508,7 +508,7 @@ function resolveCodexModel(cascadeModel: string): string { // and would silently persist zero cost. Add new models to CODEX_MODEL_IDS in // src/backends/codex/models.ts AND add a pricing row to MODEL_PRICING in // src/utils/llmMetrics.ts before accepting them here. - if (cascadeModel.startsWith('openai:')) { + if (CODEX_ACCEPTED_PREFIXES.some((prefix) => cascadeModel.startsWith(prefix))) { const bareId = cascadeModel.replace('openai:', ''); if (CODEX_MODEL_IDS.includes(bareId)) return bareId; } diff --git a/src/backends/codex/models.ts b/src/backends/codex/models.ts index beab7fcee..84e306cd5 100644 --- a/src/backends/codex/models.ts +++ b/src/backends/codex/models.ts @@ -13,3 +13,14 @@ export const CODEX_MODELS = [ export const CODEX_MODEL_IDS: string[] = CODEX_MODELS.map((model) => model.value); export const DEFAULT_CODEX_MODEL = 'gpt-5.4'; + +/** + * Model-ID prefixes the Codex engine accepts in addition to catalog IDs. + * Single source of truth consumed by `resolveCodexModel` (runtime acceptance) + * and surfaced on the engine definition as `acceptedModelPrefixes` so the + * dashboard can mirror the compatibility check without duplicating logic. + * + * Note: an `openai:`-prefixed model still only resolves when its bare ID is a + * known catalog ID (see `resolveCodexModel`); the prefix alone is not enough. + */ +export const CODEX_ACCEPTED_PREFIXES = ['openai:'] as const; diff --git a/src/backends/types.ts b/src/backends/types.ts index 744d93871..b382b7619 100644 --- a/src/backends/types.ts +++ b/src/backends/types.ts @@ -156,6 +156,14 @@ export interface AgentEngineDefinition { type: 'select'; defaultValueLabel: string; options: ReadonlyArray<{ value: string; label: string }>; + /** + * Model-ID prefixes this engine accepts in addition to its catalog + * options (e.g. `['claude-', 'anthropic:']`). Must be a plain + * `string[]` so it serializes across the `agentConfigs.engines` tRPC + * query — the frontend mirrors the runtime compatibility check via + * `isModelCompatibleWithEngine` without duplicating logic. + */ + acceptedModelPrefixes?: readonly string[]; }; readonly logLabel: string; readonly settings?: AgentEngineSettingsDefinition; diff --git a/tests/unit/backends/catalog.test.ts b/tests/unit/backends/catalog.test.ts index b929389b1..f9308e1ea 100644 --- a/tests/unit/backends/catalog.test.ts +++ b/tests/unit/backends/catalog.test.ts @@ -3,9 +3,15 @@ import { CLAUDE_CODE_ENGINE_DEFINITION, CODEX_ENGINE_DEFINITION, DEFAULT_ENGINE_CATALOG, + defaultModelLabel, LLMIST_ENGINE_DEFINITION, OPENCODE_ENGINE_DEFINITION, } from '../../../src/backends/catalog.js'; +import { + CLAUDE_CODE_MODELS, + DEFAULT_CLAUDE_CODE_MODEL, +} from '../../../src/backends/claude-code/models.js'; +import { CODEX_MODELS, DEFAULT_CODEX_MODEL } from '../../../src/backends/codex/models.js'; import type { AgentEngineDefinition } from '../../../src/backends/types.js'; describe('DEFAULT_ENGINE_CATALOG', () => { @@ -50,6 +56,60 @@ describe('DEFAULT_ENGINE_CATALOG', () => { }); }); +// ─── defaultModelLabel + displayed-default==resolved-default guard ──────────── +describe('defaultModelLabel', () => { + it('renders "Default (