Skip to content

test(provider): centralize catalog model pins, add a delisting tripwire #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Aayam Bansal (aayambansal) merged 1 commit into from
Jul 5, 2026
+64 −36
Merged

test(provider): centralize catalog model pins, add a delisting tripwire #91

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 64 additions & 36 deletions backend/cli/test/provider/provider.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,34 @@ import { Instance } from "../../src/project/instance"
import { Provider } from "../../src/provider/provider"
import { Env } from "../../src/env"

/* Pinned against the live models.dev catalog. When models.dev delists one of
these ids, the "pinned catalog models still exist upstream" test below fails
with instructions — update the pin here and every test follows. Previous
pin claude-sonnet-4-20250514 was delisted upstream on 2026-07-05 and broke
10 tests at once. */
const SONNET = "claude-sonnet-4-6"
const OPUS = "claude-opus-4-5"

test("pinned catalog models still exist upstream", async () => {
await using tmp = await tmpdir({})
await Instance.provide({
directory: tmp.path,
init: async () => {
Env.set("ANTHROPIC_API_KEY", "test-api-key")
},
fn: async () => {
const providers = await Provider.list()
const models = Object.keys(providers["anthropic"]?.models ?? {})
for (const id of [SONNET, OPUS]) {
if (!models.includes(id))
throw new Error(
`models.dev no longer lists anthropic/${id} — update the SONNET/OPUS pins at the top of this file`,
)
}
},
})
})

function clearManagedLLMEnv() {
for (const key of [
"ANTHROPIC_API_KEY",
Expand Down Expand Up @@ -157,7 +185,7 @@ test("model whitelist filters models for provider", async () => {
$schema: "https://syntheticsciences.ai/config.json",
provider: {
anthropic: {
whitelist: ["claude-sonnet-4-20250514"],
whitelist: [SONNET],
},
},
}),
Expand All @@ -173,7 +201,7 @@ test("model whitelist filters models for provider", async () => {
const providers = await Provider.list()
expect(providers["anthropic"]).toBeDefined()
const models = Object.keys(providers["anthropic"].models)
expect(models).toContain("claude-sonnet-4-20250514")
expect(models).toContain(SONNET)
expect(models.length).toBe(1)
},
})
Expand All @@ -188,7 +216,7 @@ test("model blacklist excludes specific models", async () => {
$schema: "https://syntheticsciences.ai/config.json",
provider: {
anthropic: {
blacklist: ["claude-sonnet-4-20250514"],
blacklist: [SONNET],
},
},
}),
Expand All @@ -204,7 +232,7 @@ test("model blacklist excludes specific models", async () => {
const providers = await Provider.list()
expect(providers["anthropic"]).toBeDefined()
const models = Object.keys(providers["anthropic"].models)
expect(models).not.toContain("claude-sonnet-4-20250514")
expect(models).not.toContain(SONNET)
},
})
})
Expand All @@ -220,7 +248,7 @@ test("custom model alias via config", async () => {
anthropic: {
models: {
"my-alias": {
id: "claude-sonnet-4-20250514",
id: SONNET,
name: "My Custom Alias",
},
},
Expand Down Expand Up @@ -336,10 +364,10 @@ test("getModel returns model for valid provider/model", async () => {
Env.set("ANTHROPIC_API_KEY", "test-api-key")
},
fn: async () => {
const model = await Provider.getModel("anthropic", "claude-sonnet-4-20250514")
const model = await Provider.getModel("anthropic", SONNET)
expect(model).toBeDefined()
expect(model.providerID).toBe("anthropic")
expect(model.id).toBe("claude-sonnet-4-20250514")
expect(model.id).toBe(SONNET)
const language = await Provider.getLanguage(model)
expect(language).toBeDefined()
},
Expand Down Expand Up @@ -430,7 +458,7 @@ test("defaultModel respects config model setting", async () => {
path.join(dir, "openscience.json"),
JSON.stringify({
$schema: "https://syntheticsciences.ai/config.json",
model: "anthropic/claude-sonnet-4-20250514",
model: `anthropic/${SONNET}`,
}),
)
},
Expand All @@ -443,7 +471,7 @@ test("defaultModel respects config model setting", async () => {
fn: async () => {
const model = await Provider.defaultModel()
expect(model.providerID).toBe("anthropic")
expect(model.modelID).toBe("claude-sonnet-4-20250514")
expect(model.modelID).toBe(SONNET)
},
})
})
Expand Down Expand Up @@ -538,7 +566,7 @@ test("model options are merged from existing model", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
options: {
customOption: "custom-value",
},
Expand All @@ -557,7 +585,7 @@ test("model options are merged from existing model", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.options.customOption).toBe("custom-value")
},
})
Expand Down Expand Up @@ -647,7 +675,7 @@ test("getModel uses realIdByKey for aliased models", async () => {
anthropic: {
models: {
"my-sonnet": {
id: "claude-sonnet-4-20250514",
id: SONNET,
name: "My Sonnet Alias",
},
},
Expand Down Expand Up @@ -762,7 +790,7 @@ test("model inherits properties from existing database model", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
name: "Custom Name for Sonnet",
},
},
Expand All @@ -779,7 +807,7 @@ test("model inherits properties from existing database model", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.name).toBe("Custom Name for Sonnet")
expect(model.capabilities.toolcall).toBe(true)
expect(model.capabilities.attachment).toBe(true)
Expand Down Expand Up @@ -846,8 +874,8 @@ test("whitelist and blacklist can be combined", async () => {
$schema: "https://syntheticsciences.ai/config.json",
provider: {
anthropic: {
whitelist: ["claude-sonnet-4-20250514", "claude-opus-4-20250514"],
blacklist: ["claude-opus-4-20250514"],
whitelist: [SONNET, OPUS],
blacklist: [OPUS],
},
},
}),
Expand All @@ -863,8 +891,8 @@ test("whitelist and blacklist can be combined", async () => {
const providers = await Provider.list()
expect(providers["anthropic"]).toBeDefined()
const models = Object.keys(providers["anthropic"].models)
expect(models).toContain("claude-sonnet-4-20250514")
expect(models).not.toContain("claude-opus-4-20250514")
expect(models).toContain(SONNET)
expect(models).not.toContain(OPUS)
expect(models.length).toBe(1)
},
})
Expand Down Expand Up @@ -983,7 +1011,7 @@ test("getSmallModel respects config small_model override", async () => {
path.join(dir, "openscience.json"),
JSON.stringify({
$schema: "https://syntheticsciences.ai/config.json",
small_model: "anthropic/claude-sonnet-4-20250514",
small_model: `anthropic/${SONNET}`,
}),
)
},
Expand All @@ -997,7 +1025,7 @@ test("getSmallModel respects config small_model override", async () => {
const model = await Provider.getSmallModel("anthropic")
expect(model).toBeDefined()
expect(model?.providerID).toBe("anthropic")
expect(model?.id).toBe("claude-sonnet-4-20250514")
expect(model?.id).toBe(SONNET)
},
})
})
Expand Down Expand Up @@ -1178,7 +1206,7 @@ test("model alias name defaults to alias key when id differs", async () => {
anthropic: {
models: {
sonnet: {
id: "claude-sonnet-4-20250514",
id: SONNET,
// no name specified - should default to "sonnet" (the key)
},
},
Expand Down Expand Up @@ -1294,7 +1322,7 @@ test("model cost overrides existing cost values", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
cost: {
input: 999,
output: 888,
Expand All @@ -1314,7 +1342,7 @@ test("model cost overrides existing cost values", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.cost.input).toBe(999)
expect(model.cost.output).toBe(888)
},
Expand Down Expand Up @@ -1575,8 +1603,8 @@ test("getModel returns consistent results", async () => {
Env.set("ANTHROPIC_API_KEY", "test-api-key")
},
fn: async () => {
const model1 = await Provider.getModel("anthropic", "claude-sonnet-4-20250514")
const model2 = await Provider.getModel("anthropic", "claude-sonnet-4-20250514")
const model1 = await Provider.getModel("anthropic", SONNET)
const model2 = await Provider.getModel("anthropic", SONNET)
expect(model1.providerID).toEqual(model2.providerID)
expect(model1.id).toEqual(model2.id)
expect(model1).toEqual(model2)
Expand Down Expand Up @@ -1940,7 +1968,7 @@ test("model variants are generated for reasoning models", async () => {
fn: async () => {
const providers = await Provider.list()
// Claude sonnet 4 has reasoning capability
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.capabilities.reasoning).toBe(true)
expect(model.variants).toBeDefined()
expect(Object.keys(model.variants!).length).toBeGreaterThan(0)
Expand All @@ -1958,7 +1986,7 @@ test("model variants can be disabled via config", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
variants: {
high: { disabled: true },
},
Expand All @@ -1977,7 +2005,7 @@ test("model variants can be disabled via config", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.variants).toBeDefined()
expect(model.variants!["high"]).toBeUndefined()
// max variant should still exist
Expand All @@ -1996,7 +2024,7 @@ test("model variants can be customized via config", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
variants: {
high: {
thinking: {
Expand All @@ -2020,7 +2048,7 @@ test("model variants can be customized via config", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.variants!["high"]).toBeDefined()
expect(model.variants!["high"].thinking.budgetTokens).toBe(20000)
},
Expand All @@ -2037,7 +2065,7 @@ test("disabled key is stripped from variant config", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
variants: {
max: {
disabled: false,
Expand All @@ -2059,7 +2087,7 @@ test("disabled key is stripped from variant config", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.variants!["max"]).toBeDefined()
expect(model.variants!["max"].disabled).toBeUndefined()
expect(model.variants!["max"].customField).toBe("test")
Expand All @@ -2077,7 +2105,7 @@ test("all variants can be disabled via config", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
variants: {
low: { disabled: true },
medium: { disabled: true },
Expand All @@ -2099,7 +2127,7 @@ test("all variants can be disabled via config", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.variants).toBeDefined()
expect(Object.keys(model.variants!).length).toBe(0)
},
Expand All @@ -2116,7 +2144,7 @@ test("variant config merges with generated variants", async () => {
provider: {
anthropic: {
models: {
"claude-sonnet-4-20250514": {
[SONNET]: {
variants: {
high: {
extraOption: "custom-value",
Expand All @@ -2137,7 +2165,7 @@ test("variant config merges with generated variants", async () => {
},
fn: async () => {
const providers = await Provider.list()
const model = providers["anthropic"].models["claude-sonnet-4-20250514"]
const model = providers["anthropic"].models[SONNET]
expect(model.variants!["high"]).toBeDefined()
// Should have both the generated thinking config and the custom option
expect(model.variants!["high"].thinking).toBeDefined()
Expand Down
Loading