Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion docs/adding-engines.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,33 @@ export const MY_ENGINE_DEFINITION: AgentEngineDefinition = {
],
modelSelection: {
type: 'select', // or 'free-text' for open-ended model strings
defaultValueLabel: 'Default (v1.0)',
// Derive the label from your DEFAULT_*_MODEL constant + catalog lookup instead
// of hardcoding it, so the displayed default can never drift from the actually
// resolved default on the next model bump (MNG-1772). See catalog.ts's
// `defaultModelLabel(models, defaultId)` helper.
defaultValueLabel: defaultModelLabel(MY_ENGINE_MODELS, DEFAULT_MY_ENGINE_MODEL),
options: MY_ENGINE_MODELS, // Imported from ./my-engine/models.ts
// Optional: model-ID prefixes your engine accepts in addition to catalog IDs.
// Single-source these next to your model list (e.g. MY_ENGINE_ACCEPTED_PREFIXES)
// and consume them in your resolveModel() so the runtime acceptance rules and
// the dashboard's config-time incompatibility warning share one definition.
// Must be a plain string[] — it serializes across the agentConfigs.engines tRPC
// query and is mirrored by the frontend's isModelCompatibleWithEngine().
acceptedModelPrefixes: MY_ENGINE_ACCEPTED_PREFIXES,
},
logLabel: 'My Engine Log',
// Optional: add 'settings' if your engine has configurable fields
};
```

> **Anti-drift + config-time warning contract (MNG-1772).** For `select`-type
> engines, deriving `defaultValueLabel` from the default-model constant keeps the
> UI honest, and exposing `acceptedModelPrefixes` lets the agent-config detail
> panel warn *before* dispatch when an inherited model is incompatible with the
> engine (instead of crashing mid-run). A guard test in
> `tests/unit/backends/catalog.test.ts` asserts the displayed default always
> matches the resolved default for every `select` engine.

Add it to `DEFAULT_ENGINE_CATALOG` at the bottom of the same file:

```typescript
Expand Down
28 changes: 24 additions & 4 deletions src/backends/catalog.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,25 @@
import { CLAUDE_CODE_MODELS } from './claude-code/models.js';
import { CODEX_MODELS } from './codex/models.js';
import {
CLAUDE_CODE_ACCEPTED_PREFIXES,
CLAUDE_CODE_MODELS,
DEFAULT_CLAUDE_CODE_MODEL,
} from './claude-code/models.js';
import { CODEX_ACCEPTED_PREFIXES, CODEX_MODELS, DEFAULT_CODEX_MODEL } from './codex/models.js';
import type { AgentEngineDefinition } from './types.js';

/**
* Derive the `select`-engine empty-option label from the engine's default model
* constant, looking the display label up in the model catalog. This kills the
* hand-synced-literal drift class (MNG-1772/MNG-1770): the displayed default
* always matches the actually-resolved `DEFAULT_*_MODEL` on the next model bump.
*/
export function defaultModelLabel(
models: ReadonlyArray<{ value: string; label: string }>,
defaultId: string,
): string {
const match = models.find((m) => m.value === defaultId);
return `Default (${match?.label ?? defaultId})`;
}

export const LLMIST_ENGINE_DEFINITION: AgentEngineDefinition = {
id: 'llmist',
label: 'LLMist',
Expand Down Expand Up @@ -35,8 +53,9 @@ export const CLAUDE_CODE_ENGINE_DEFINITION: AgentEngineDefinition = {
],
modelSelection: {
type: 'select',
defaultValueLabel: 'Default (Sonnet 5)',
defaultValueLabel: defaultModelLabel(CLAUDE_CODE_MODELS, DEFAULT_CLAUDE_CODE_MODEL),
options: CLAUDE_CODE_MODELS,
acceptedModelPrefixes: CLAUDE_CODE_ACCEPTED_PREFIXES,
},
logLabel: 'Claude Code Log',
settings: {
Expand Down Expand Up @@ -94,8 +113,9 @@ export const CODEX_ENGINE_DEFINITION: AgentEngineDefinition = {
],
modelSelection: {
type: 'select',
defaultValueLabel: 'Default (GPT-5.4)',
defaultValueLabel: defaultModelLabel(CODEX_MODELS, DEFAULT_CODEX_MODEL),
options: CODEX_MODELS,
acceptedModelPrefixes: CODEX_ACCEPTED_PREFIXES,
},
logLabel: 'Codex Log',
settings: {
Expand Down
13 changes: 10 additions & 3 deletions src/backends/claude-code/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ import {
consumeStream,
filterContextImages,
} from './messageProcessing.js';
import { CLAUDE_CODE_MODEL_IDS, DEFAULT_CLAUDE_CODE_MODEL } from './models.js';
import {
CLAUDE_CODE_ACCEPTED_PREFIXES,
CLAUDE_CODE_MODEL_IDS,
DEFAULT_CLAUDE_CODE_MODEL,
} from './models.js';
import { ClaudeCodeSettingsSchema, resolveClaudeCodeSettings } from './settings.js';

export {
Expand All @@ -40,8 +44,11 @@ export { buildPromptWithImages, formatErrorMessage } from './messageProcessing.j
*/
export function resolveClaudeModel(cascadeModel: string): string {
if (CLAUDE_CODE_MODEL_IDS.includes(cascadeModel)) return cascadeModel;
if (cascadeModel.startsWith('claude-')) return cascadeModel;
if (cascadeModel.startsWith('anthropic:')) return cascadeModel.replace('anthropic:', '');
if (CLAUDE_CODE_ACCEPTED_PREFIXES.some((prefix) => cascadeModel.startsWith(prefix))) {
return cascadeModel.startsWith('anthropic:')
? cascadeModel.replace('anthropic:', '')
: cascadeModel;
}

throw new Error(
`Model "${cascadeModel}" is not compatible with the Claude Code engine. Configure a Claude-compatible model (e.g. "${DEFAULT_CLAUDE_CODE_MODEL}") or switch to a different engine.`,
Expand Down
8 changes: 8 additions & 0 deletions src/backends/claude-code/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,11 @@ export const CLAUDE_CODE_MODELS = [
export const CLAUDE_CODE_MODEL_IDS: string[] = CLAUDE_CODE_MODELS.map((m) => m.value);

export const DEFAULT_CLAUDE_CODE_MODEL = 'claude-sonnet-5';

/**
* Model-ID prefixes the Claude Code engine accepts in addition to catalog IDs.
* Single source of truth consumed by `resolveClaudeModel` (runtime acceptance)
* and surfaced on the engine definition as `acceptedModelPrefixes` so the
* dashboard can mirror the compatibility check without duplicating logic.
*/
export const CLAUDE_CODE_ACCEPTED_PREFIXES = ['claude-', 'anthropic:'] as const;
4 changes: 2 additions & 2 deletions src/backends/codex/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { buildSystemPrompt, buildTaskPrompt } from '../shared/nativeToolPrompts.
import type { AgentEngineResult, AgentExecutionPlan, LogWriter } from '../types.js';
import type { UsageSummary } from './jsonlParser.js';
import { extractUsage, parseCodexEvent } from './jsonlParser.js';
import { CODEX_MODEL_IDS, DEFAULT_CODEX_MODEL } from './models.js';
import { CODEX_ACCEPTED_PREFIXES, CODEX_MODEL_IDS, DEFAULT_CODEX_MODEL } from './models.js';
import { CODEX_COMPLETION_OUTPUT_SCHEMA, parseCodexCompletionReport } from './outputSchema.js';
import {
assertHeadlessCodexSettings,
Expand Down Expand Up @@ -508,7 +508,7 @@ function resolveCodexModel(cascadeModel: string): string {
// and would silently persist zero cost. Add new models to CODEX_MODEL_IDS in
// src/backends/codex/models.ts AND add a pricing row to MODEL_PRICING in
// src/utils/llmMetrics.ts before accepting them here.
if (cascadeModel.startsWith('openai:')) {
if (CODEX_ACCEPTED_PREFIXES.some((prefix) => cascadeModel.startsWith(prefix))) {
const bareId = cascadeModel.replace('openai:', '');
if (CODEX_MODEL_IDS.includes(bareId)) return bareId;
}
Expand Down
11 changes: 11 additions & 0 deletions src/backends/codex/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,14 @@ export const CODEX_MODELS = [
export const CODEX_MODEL_IDS: string[] = CODEX_MODELS.map((model) => model.value);

export const DEFAULT_CODEX_MODEL = 'gpt-5.4';

/**
* Model-ID prefixes the Codex engine accepts in addition to catalog IDs.
* Single source of truth consumed by `resolveCodexModel` (runtime acceptance)
* and surfaced on the engine definition as `acceptedModelPrefixes` so the
* dashboard can mirror the compatibility check without duplicating logic.
*
* Note: an `openai:`-prefixed model still only resolves when its bare ID is a
* known catalog ID (see `resolveCodexModel`); the prefix alone is not enough.
*/
export const CODEX_ACCEPTED_PREFIXES = ['openai:'] as const;
8 changes: 8 additions & 0 deletions src/backends/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,14 @@ export interface AgentEngineDefinition {
type: 'select';
defaultValueLabel: string;
options: ReadonlyArray<{ value: string; label: string }>;
/**
* Model-ID prefixes this engine accepts in addition to its catalog
* options (e.g. `['claude-', 'anthropic:']`). Must be a plain
* `string[]` so it serializes across the `agentConfigs.engines` tRPC
* query — the frontend mirrors the runtime compatibility check via
* `isModelCompatibleWithEngine` without duplicating logic.
*/
acceptedModelPrefixes?: readonly string[];
};
readonly logLabel: string;
readonly settings?: AgentEngineSettingsDefinition;
Expand Down
60 changes: 60 additions & 0 deletions tests/unit/backends/catalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,15 @@ import {
CLAUDE_CODE_ENGINE_DEFINITION,
CODEX_ENGINE_DEFINITION,
DEFAULT_ENGINE_CATALOG,
defaultModelLabel,
LLMIST_ENGINE_DEFINITION,
OPENCODE_ENGINE_DEFINITION,
} from '../../../src/backends/catalog.js';
import {
CLAUDE_CODE_MODELS,
DEFAULT_CLAUDE_CODE_MODEL,
} from '../../../src/backends/claude-code/models.js';
import { CODEX_MODELS, DEFAULT_CODEX_MODEL } from '../../../src/backends/codex/models.js';
import type { AgentEngineDefinition } from '../../../src/backends/types.js';

describe('DEFAULT_ENGINE_CATALOG', () => {
Expand Down Expand Up @@ -50,6 +56,60 @@ describe('DEFAULT_ENGINE_CATALOG', () => {
});
});

// ─── defaultModelLabel + displayed-default==resolved-default guard ────────────
describe('defaultModelLabel', () => {
it('renders "Default (<label>)" using the catalog label lookup', () => {
const models = [
{ value: 'claude-sonnet-5', label: 'Claude Sonnet 5' },
{ value: 'gpt-5.4', label: 'GPT-5.4' },
];
expect(defaultModelLabel(models, 'claude-sonnet-5')).toBe('Default (Claude Sonnet 5)');
expect(defaultModelLabel(models, 'gpt-5.4')).toBe('Default (GPT-5.4)');
});

it('falls back to the raw id when the id is not in the catalog', () => {
expect(defaultModelLabel([{ value: 'a', label: 'A' }], 'missing')).toBe('Default (missing)');
});
});

/**
* Anti-drift guard (MNG-1772): the label displayed for the empty option of each
* `select` engine must be derived from that engine's DEFAULT_*_MODEL. If someone
* bumps the default model without touching the label (or vice versa), this fails
* loudly — the "displayed default != resolved default" defect class this issue
* was about cannot silently reappear.
*/
describe('select engines: displayed default == resolved default', () => {
it('claude-code defaultValueLabel matches DEFAULT_CLAUDE_CODE_MODEL catalog label', () => {
const label = CLAUDE_CODE_MODELS.find((m) => m.value === DEFAULT_CLAUDE_CODE_MODEL)?.label;
expect(label).toBeDefined();
if (CLAUDE_CODE_ENGINE_DEFINITION.modelSelection.type === 'select') {
expect(CLAUDE_CODE_ENGINE_DEFINITION.modelSelection.defaultValueLabel).toBe(
`Default (${label})`,
);
}
});

it('codex defaultValueLabel matches DEFAULT_CODEX_MODEL catalog label', () => {
const label = CODEX_MODELS.find((m) => m.value === DEFAULT_CODEX_MODEL)?.label;
expect(label).toBeDefined();
if (CODEX_ENGINE_DEFINITION.modelSelection.type === 'select') {
expect(CODEX_ENGINE_DEFINITION.modelSelection.defaultValueLabel).toBe(`Default (${label})`);
}
});

it('select engines expose acceptedModelPrefixes as a plain string[]', () => {
for (const engine of DEFAULT_ENGINE_CATALOG) {
if (engine.modelSelection.type !== 'select') continue;
const prefixes = engine.modelSelection.acceptedModelPrefixes;
expect(Array.isArray(prefixes)).toBe(true);
for (const prefix of prefixes ?? []) {
expect(typeof prefix).toBe('string');
}
}
});
});

// ─── Individual engine definitions ────────────────────────────────────────────
describe('LLMIST_ENGINE_DEFINITION', () => {
it('has correct id and label', () => {
Expand Down
86 changes: 86 additions & 0 deletions tests/unit/web/agent-config-utils.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { describe, expect, it } from 'vitest';
import type { ResolvedTrigger } from '../../../src/api/routers/_shared/triggerTypes.js';
import type { Engine } from '../../../web/src/components/projects/agent-config-types.js';
import {
countActiveTriggers,
engineHasCredentials,
isModelCompatibleWithEngine,
} from '../../../web/src/components/projects/agent-config-utils.js';

// ============================================================================
Expand Down Expand Up @@ -76,6 +78,90 @@ describe('engineHasCredentials', () => {
});
});

// ============================================================================
// isModelCompatibleWithEngine
// ============================================================================

describe('isModelCompatibleWithEngine', () => {
const claudeCode: Engine = {
id: 'claude-code',
label: 'Claude Code',
modelSelection: {
type: 'select',
defaultValueLabel: 'Default (Claude Sonnet 5)',
options: [
{ value: 'claude-sonnet-5', label: 'Claude Sonnet 5' },
{ value: 'claude-opus-5', label: 'Claude Opus 5' },
],
acceptedModelPrefixes: ['claude-', 'anthropic:'],
},
};

const codex: Engine = {
id: 'codex',
label: 'Codex',
modelSelection: {
type: 'select',
defaultValueLabel: 'Default (GPT-5.4)',
options: [{ value: 'gpt-5.4', label: 'GPT-5.4' }],
acceptedModelPrefixes: ['openai:'],
},
};

const freeText: Engine = {
id: 'llmist',
label: 'LLMist',
modelSelection: { type: 'free-text' },
};

it('flags an openrouter model as incompatible with claude-code', () => {
expect(
isModelCompatibleWithEngine('openrouter:google/gemini-3-flash-preview', claudeCode),
).toBe(false);
});

it('accepts a catalog id for claude-code', () => {
expect(isModelCompatibleWithEngine('claude-sonnet-5', claudeCode)).toBe(true);
});

it('accepts a non-catalog claude- prefixed model for claude-code', () => {
expect(isModelCompatibleWithEngine('claude-opus-5', claudeCode)).toBe(true);
});

it('accepts an anthropic: prefixed model for claude-code', () => {
expect(isModelCompatibleWithEngine('anthropic:claude-sonnet-4-5', claudeCode)).toBe(true);
});

it('flags an openai model as incompatible with claude-code', () => {
expect(isModelCompatibleWithEngine('openai:gpt-5.4', claudeCode)).toBe(false);
});

it('accepts catalog ids and openai: prefixes for codex', () => {
expect(isModelCompatibleWithEngine('gpt-5.4', codex)).toBe(true);
expect(isModelCompatibleWithEngine('openai:gpt-5.4', codex)).toBe(true);
});

it('flags an openrouter model as incompatible with codex', () => {
expect(isModelCompatibleWithEngine('openrouter:google/gemini-3-flash-preview', codex)).toBe(
false,
);
});

it('treats every model as compatible with a free-text engine', () => {
expect(isModelCompatibleWithEngine('openrouter:google/gemini-3-flash-preview', freeText)).toBe(
true,
);
});

it('treats an empty model (inherit) as compatible', () => {
expect(isModelCompatibleWithEngine('', claudeCode)).toBe(true);
});

it('treats an engine without modelSelection as compatible', () => {
expect(isModelCompatibleWithEngine('anything', { id: 'x', label: 'X' })).toBe(true);
});
});

// ============================================================================
// countActiveTriggers
// ============================================================================
Expand Down
29 changes: 29 additions & 0 deletions tests/unit/web/model-field.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest';
import { resolveSelectEmptyLabel } from '../../../web/src/components/settings/model-field.js';
import {
addPrefix,
formatContext,
Expand All @@ -7,6 +8,34 @@ import {
stripPrefix,
} from '../../../web/src/lib/openrouter-utils.js';

// ────────────────────────────────────────────────────────────────────────────
// resolveSelectEmptyLabel — honors the caller's inheritance-aware defaultLabel
// on the ModelField `select` branch (MNG-1772). The runtime chain is
// override → per-agent → project.model with no engine-default step, so the
// empty option must describe inheritance, not a fictional engine default.
// ────────────────────────────────────────────────────────────────────────────
describe('resolveSelectEmptyLabel', () => {
it('returns the caller defaultLabel when provided', () => {
expect(
resolveSelectEmptyLabel(
'Inherit from project (openrouter:google/gemini-3-flash-preview)',
'Default (Claude Sonnet 5)',
),
).toBe('Inherit from project (openrouter:google/gemini-3-flash-preview)');
});

it('falls back to the engine defaultValueLabel when defaultLabel is undefined', () => {
expect(resolveSelectEmptyLabel(undefined, 'Default (Claude Sonnet 5)')).toBe(
'Default (Claude Sonnet 5)',
);
});

it('prefers an empty-string defaultLabel only when it is nullish (uses ?? semantics)', () => {
// Empty string is a real value under ?? — kept for callers that pass ''.
expect(resolveSelectEmptyLabel('', 'Default (GPT-5.4)')).toBe('');
});
});

// Tests import directly from the shared utility module used by the production
// component, so implementation drift between tests and production is impossible.

Expand Down
Loading
Loading