diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index 8d23ee9b8..55d02c1ce 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -36,7 +36,7 @@ import { writePerTestArtifacts as writeCorePerTestArtifacts, writeInitialRunSummaryArtifact, } from '@agentv/core'; -import type { TargetDefinition } from '@agentv/core'; +import type { ProviderDefinition } from '@agentv/core'; import { type MaterializedTaskBundlePaths, @@ -206,7 +206,7 @@ function createTaskBundleArtifactsWriter(options?: { const taskBundle = await materializeTaskBundle({ test: sourceTest, targetName: targetSelection.targetName, - targetDefinitions: targetSelection.definitions as readonly TargetDefinition[], + targetDefinitions: targetSelection.definitions as readonly ProviderDefinition[], outputDir: testDir, cwd: options?.cwd, repoRoot: options?.repoRoot, diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index 44c43399c..cbe7522b5 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -2,9 +2,9 @@ import path from 'node:path'; import { type EvalTargetRef, type EvalTargetSpec, - type TargetDefinition, + type ProviderDefinition, loadTestSuite, - readTargetDefinitions, + readProviderDefinitions, } from '@agentv/core'; import { array, command, multioption, option, optional, positional, string } from 'cmd-ts'; @@ -28,7 +28,7 @@ function unique(values: readonly string[]): readonly string[] { return result; } -function targetReferenceNames(target: TargetDefinition): readonly string[] { +function targetReferenceNames(target: ProviderDefinition): readonly string[] { const references: string[] = []; for (const key of ['use_target', 'grader_target'] as const) { const value = target[key]; @@ -51,7 +51,7 @@ function targetReferenceNames(target: TargetDefinition): readonly string[] { function ensureTargetGraph( targetName: string, - definitions: readonly TargetDefinition[], + definitions: readonly ProviderDefinition[], targetsFilePath: string, ): void { const byName = new Map(definitions.map((definition) => [definition.name, definition])); @@ -82,9 +82,9 @@ function ensureTargetGraph( } function definitionsWithEvalTargetRefs( - definitions: readonly TargetDefinition[], + definitions: readonly ProviderDefinition[], targetRefs: readonly EvalTargetRef[] | undefined, -): readonly TargetDefinition[] { +): readonly ProviderDefinition[] { if (!targetRefs) { return definitions; } @@ -94,16 +94,16 @@ function definitionsWithEvalTargetRefs( if (ref.definition && !result.some((definition) => definition.name === ref.name)) { result.push(ref.definition); } else if (ref.use_target && !result.some((definition) => definition.name === ref.name)) { - result.push({ name: ref.name, use_target: ref.use_target } as TargetDefinition); + result.push({ name: ref.name, use_target: ref.use_target } as ProviderDefinition); } } return result; } function definitionsWithEvalTargetSpec( - definitions: readonly TargetDefinition[], + definitions: readonly ProviderDefinition[], targetSpec: EvalTargetSpec | undefined, -): readonly TargetDefinition[] { +): readonly ProviderDefinition[] { if (!targetSpec?.definition) { return definitions; } @@ -124,7 +124,7 @@ function definitionsWithEvalTargetSpec( ...base, ...targetSpec.definition, name: targetSpec.name, - } as TargetDefinition; + } as ProviderDefinition; return [effective, ...definitions.filter((definition) => definition.name !== targetSpec.name)]; } @@ -210,7 +210,7 @@ export const evalBundleCommand = command({ ); } - let definitions: readonly TargetDefinition[]; + let definitions: readonly ProviderDefinition[]; let targetNames: readonly string[]; if (suite.inlineTarget) { definitions = [suite.inlineTarget]; @@ -224,7 +224,7 @@ export const evalBundleCommand = command({ }); definitions = definitionsWithEvalTargetSpec( definitionsWithEvalTargetRefs( - await readTargetDefinitions(targetsFilePath), + await readProviderDefinitions(targetsFilePath), suite.targetRefs, ), suite.targetSpec, diff --git a/apps/cli/src/commands/eval/interactive.ts b/apps/cli/src/commands/eval/interactive.ts index 4a442c96a..793d3276c 100644 --- a/apps/cli/src/commands/eval/interactive.ts +++ b/apps/cli/src/commands/eval/interactive.ts @@ -1,6 +1,6 @@ import { existsSync } from 'node:fs'; import path from 'node:path'; -import { listTargetNames, readTargetDefinitions } from '@agentv/core'; +import { listProviderLabels, readProviderDefinitions } from '@agentv/core'; import { checkbox, confirm, number, search, select } from '@inquirer/prompts'; import { PROVIDER_FILE_CANDIDATES, fileExists } from '../../utils/targets.js'; @@ -209,8 +209,8 @@ async function promptTargetSelection(cwd: string, firstEvalPath: string): Promis return 'default'; } - const definitions = await readTargetDefinitions(targetsPath); - const targetNames = listTargetNames(definitions); + const definitions = await readProviderDefinitions(targetsPath); + const targetNames = listProviderLabels(definitions); if (targetNames.length === 0) { return 'default'; diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 2f94a2325..32f6c5333 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -16,11 +16,11 @@ import { type ExperimentArtifactMetadata, type ExperimentConfig, type FailOnError, - type ResolvedTarget, + type ProviderDefinition, + type ResolvedProviderBackend, ResponseCache, RunBudgetTracker, type RunRuntimeSourceMetadata, - type TargetDefinition, type TrialsConfig, buildExperimentArtifactMetadata, buildTraceFromMessages, @@ -31,7 +31,7 @@ import { loadTestSuite, loadTestSuiteFromYamlObject, loadTsConfig, - resolveTargetDefinition, + resolveProviderDefinition, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToCodexLogEntries, @@ -1205,7 +1205,7 @@ function createDisplayIdTracker(): { getOrAssign(testCaseKey: string): number } * Azure uses `deploymentName`; most other providers use `model`. * CLI and mock providers have no model field. */ -function extractModelName(target: ResolvedTarget): string | undefined { +function extractModelName(target: ResolvedProviderBackend): string | undefined { if (target.kind === 'azure') { return target.config.deploymentName; } @@ -1218,7 +1218,7 @@ function extractModelName(target: ResolvedTarget): string | undefined { /** * Build the inline label suffix (e.g. `[provider=azure, model=gpt-4]`). */ -function buildTargetLabelSuffix(providerLabel: string, target: ResolvedTarget): string { +function buildTargetLabelSuffix(providerLabel: string, target: ResolvedProviderBackend): string { const parts = [`provider=${providerLabel}`]; const model = extractModelName(target); if (model) parts.push(`model=${model}`); @@ -1345,7 +1345,7 @@ async function prepareFileMetadata(params: { readonly cwd: string; readonly options: NormalizedOptions; readonly providerCatalogPath?: string; - readonly providerDefinitions?: readonly TargetDefinition[]; + readonly providerDefinitions?: readonly ProviderDefinition[]; readonly providerDefinitionsSource?: string; readonly suiteFilter?: string | readonly string[]; }): Promise<{ @@ -1362,7 +1362,7 @@ async function prepareFileMetadata(params: { readonly threshold?: number; readonly tags?: readonly string[]; readonly providerFactory?: ( - target: import('@agentv/core').ResolvedTarget, + target: import('@agentv/core').ResolvedProviderBackend, ) => import('@agentv/core').Provider; }> { const { @@ -1454,7 +1454,7 @@ async function prepareFileMetadata(params: { ]; } else if (suite.inlineTarget && effectiveOptions.cliTargets.length === 0) { const targetDefinition = suite.inlineTarget; - const resolvedTarget = resolveTargetDefinition(targetDefinition, process.env, testFilePath, { + const resolvedTarget = resolveProviderDefinition(targetDefinition, process.env, testFilePath, { emitDeprecationWarnings: false, }); selections = [ @@ -1470,7 +1470,7 @@ async function prepareFileMetadata(params: { }, ]; } else if (suite.providerFactory && effectiveOptions.cliTargets.length === 0) { - const taskTarget: ResolvedTarget = { + const taskTarget: ResolvedProviderBackend = { kind: 'mock', name: 'custom-task', graderTarget: undefined, @@ -1647,7 +1647,7 @@ async function runSingleEvalFile(params: { readonly failOnError?: FailOnError; readonly threshold?: number; readonly providerFactory?: ( - target: import('@agentv/core').ResolvedTarget, + target: import('@agentv/core').ResolvedProviderBackend, ) => import('@agentv/core').Provider; }): Promise<{ results: EvaluationResult[] }> { const { @@ -2156,7 +2156,7 @@ export async function runEvalCommand( readonly threshold?: number; readonly tags?: readonly string[]; readonly providerFactory?: ( - target: import('@agentv/core').ResolvedTarget, + target: import('@agentv/core').ResolvedProviderBackend, ) => import('@agentv/core').Provider; } >(); @@ -2359,7 +2359,7 @@ export async function runEvalCommand( // --transcript: create a shared TranscriptProvider and validate entry count let transcriptProviderFactory: - | ((target: import('@agentv/core').ResolvedTarget) => import('@agentv/core').Provider) + | ((target: import('@agentv/core').ResolvedProviderBackend) => import('@agentv/core').Provider) | undefined; if (options.transcript) { const { TranscriptProvider } = await import('@agentv/core'); diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index 7f109f66a..84c03a0f7 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -1,11 +1,11 @@ import { type EvalTargetSpec, - type ResolvedTarget, - type TargetDefinition, - listTargetNames, - readTargetDefinitions, + type ProviderDefinition, + type ResolvedProviderBackend, + listProviderLabels, + readProviderDefinitions, readTestSuiteMetadata, - resolveTargetDefinition, + resolveProviderDefinition, } from '@agentv/core'; import { validateTargetsFile } from '@agentv/core/evaluation/validation'; import { discoverTargetsFile } from '../../utils/targets.js'; @@ -32,14 +32,14 @@ function isTTY(): boolean { */ function resolveUseTarget( name: string, - definitions: readonly TargetDefinition[], + definitions: readonly ProviderDefinition[], env: NodeJS.ProcessEnv, targetsFilePath: string, -): TargetDefinition { +): ProviderDefinition { const maxDepth = 5; - let current: TargetDefinition | undefined = definitions.find((d) => d.name === name); + let current: ProviderDefinition | undefined = definitions.find((d) => d.name === name); if (!current) { - const available = listTargetNames(definitions).join(', '); + const available = listProviderLabels(definitions).join(', '); throw new Error( `Provider '${name}' not found in ${targetsFilePath}. Available providers: ${available}`, ); @@ -56,9 +56,11 @@ function resolveUseTarget( const resolved: string = envMatch ? (env[envMatch[1]] ?? '') : raw; if (resolved.trim().length === 0) break; - const next: TargetDefinition | undefined = definitions.find((d) => d.name === resolved.trim()); + const next: ProviderDefinition | undefined = definitions.find( + (d) => d.name === resolved.trim(), + ); if (!next) { - const available = listTargetNames(definitions).join(', '); + const available = listProviderLabels(definitions).join(', '); throw new Error( `Provider '${name}' use_target '${resolved.trim()}' not found in ${targetsFilePath}. Available providers: ${available}`, ); @@ -82,8 +84,8 @@ export async function readTestSuiteTargets( } export interface TargetSelection { - readonly definitions: readonly TargetDefinition[]; - readonly resolvedTarget: ResolvedTarget; + readonly definitions: readonly ProviderDefinition[]; + readonly resolvedTarget: ResolvedProviderBackend; readonly targetName: string; readonly targetLabel?: string; readonly targetSource: 'cli' | 'test-file' | 'default'; @@ -97,7 +99,7 @@ export interface TargetSelectionOptions { readonly repoRoot: string; readonly cwd: string; readonly explicitTargetsPath?: string; - readonly providerDefinitions?: readonly TargetDefinition[]; + readonly providerDefinitions?: readonly ProviderDefinition[]; readonly providerDefinitionsSource?: string; readonly requireExplicitProviderCatalog?: boolean; readonly allowLegacyTargetFiles?: boolean; @@ -111,14 +113,14 @@ export interface TargetSelectionOptions { async function readProviderCatalog(options: { readonly explicitTargetsPath?: string; - readonly providerDefinitions?: readonly TargetDefinition[]; + readonly providerDefinitions?: readonly ProviderDefinition[]; readonly providerDefinitionsSource?: string; readonly requireExplicitProviderCatalog?: boolean; readonly testFilePath: string; readonly repoRoot: string; readonly cwd: string; readonly allowLegacyTargetFiles?: boolean; -}): Promise<{ readonly definitions: readonly TargetDefinition[]; readonly sourcePath: string }> { +}): Promise<{ readonly definitions: readonly ProviderDefinition[]; readonly sourcePath: string }> { if (!options.explicitTargetsPath && options.providerDefinitions) { return { definitions: options.providerDefinitions, @@ -139,7 +141,7 @@ async function readProviderCatalog(options: { }); await validateProviderCatalogFile(targetsFilePath); return { - definitions: await readTargetDefinitions(targetsFilePath), + definitions: await readProviderDefinitions(targetsFilePath), sourcePath: targetsFilePath, }; } @@ -191,19 +193,19 @@ function pickTargetName(options: { } function withModelOverride( - target: TargetDefinition, + target: ProviderDefinition, modelOverride: string | undefined, -): TargetDefinition { +): ProviderDefinition { const model = modelOverride?.trim(); return model && model.length > 0 ? { ...target, model } : target; } function overlayTargetDefinition(params: { readonly spec: EvalTargetSpec | undefined; - readonly definitions: readonly TargetDefinition[]; + readonly definitions: readonly ProviderDefinition[]; readonly env: NodeJS.ProcessEnv; readonly targetsFilePath: string; -}): TargetDefinition | undefined { +}): ProviderDefinition | undefined { const { spec, definitions, env, targetsFilePath } = params; if (!spec?.definition) { return undefined; @@ -220,9 +222,9 @@ function overlayTargetDefinition(params: { } function definitionsWithEffectiveTarget( - definitions: readonly TargetDefinition[], - effective: TargetDefinition, -): readonly TargetDefinition[] { + definitions: readonly ProviderDefinition[], + effective: ProviderDefinition, +): readonly ProviderDefinition[] { return [effective, ...definitions.filter((definition) => definition.name !== effective.name)]; } @@ -272,7 +274,7 @@ export async function selectTarget(options: TargetSelectionOptions): Promise d.name === ref.name)) { definitions.push(ref.definition); } else if (ref.use_target && !fileDefinitions.some((d) => d.name === ref.name)) { - definitions.push({ name: ref.name, use_target: ref.use_target } as TargetDefinition); + definitions.push({ name: ref.name, use_target: ref.use_target } as ProviderDefinition); } } } @@ -367,7 +369,7 @@ export async function selectMultipleTargets( const targetLabel = labelsMap.get(name); try { - const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath, { + const resolvedTarget = resolveProviderDefinition(targetDefinition, env, testFilePath, { emitDeprecationWarnings: false, }); results.push({ diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 1b026dc6b..6a8dda903 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -7,7 +7,7 @@ import { type EvalSourceReference, type EvalTest, type JsonObject, - type TargetDefinition, + type ProviderDefinition, type TestMessage, type WorkspaceConfig, parseYamlValue, @@ -76,13 +76,13 @@ export interface TaskBundleTargetSelection { readonly evalFileAbsolutePath?: string; readonly targetName: string; readonly resolvedTargetName?: string; - readonly definitions: readonly TargetDefinition[]; + readonly definitions: readonly ProviderDefinition[]; } export interface MaterializeTaskBundleOptions { readonly test: EvalTest; readonly targetName: string; - readonly targetDefinitions: readonly TargetDefinition[]; + readonly targetDefinitions: readonly ProviderDefinition[]; readonly outputDir: string; readonly cwd?: string; readonly repoRoot?: string; @@ -557,7 +557,7 @@ function buildEvalCase( return moveInputToVars(testCase); } -function targetReferenceNames(target: TargetDefinition): readonly string[] { +function targetReferenceNames(target: ProviderDefinition): readonly string[] { const references: string[] = []; for (const key of ['use_target', 'grader_target'] as const) { const value = target[key]; @@ -580,10 +580,10 @@ function targetReferenceNames(target: TargetDefinition): readonly string[] { function selectTargetDefinitions( targetName: string, - definitions: readonly TargetDefinition[], -): readonly TargetDefinition[] { + definitions: readonly ProviderDefinition[], +): readonly ProviderDefinition[] { const byName = new Map(definitions.map((definition) => [definition.name, definition])); - const selected: TargetDefinition[] = []; + const selected: ProviderDefinition[] = []; const seen = new Set(); function visit(name: string): void { @@ -636,8 +636,8 @@ function bundledEvalFileName(evalFilePath: string): string { function uniqueTargetDefinitions( selections: readonly TaskBundleTargetSelection[], -): readonly TargetDefinition[] { - const selected: TargetDefinition[] = []; +): readonly ProviderDefinition[] { + const selected: ProviderDefinition[] = []; const seen = new Set(); for (const selection of selections) { @@ -653,7 +653,7 @@ function uniqueTargetDefinitions( return selected; } -function serializeTargetDefinition(definition: TargetDefinition): Record { +function serializeTargetDefinition(definition: ProviderDefinition): Record { const providerSpec = typeof definition.provider_spec === 'string' && definition.provider_spec.trim().length > 0 ? definition.provider_spec.trim() @@ -702,7 +702,7 @@ function serializeTargetDefinition(definition: TargetDefinition): Record[] { return definitions.map((definition) => serializeTargetDefinition(definition)); } diff --git a/apps/cli/src/commands/grade/index.ts b/apps/cli/src/commands/grade/index.ts index a385d297f..7793f8dee 100644 --- a/apps/cli/src/commands/grade/index.ts +++ b/apps/cli/src/commands/grade/index.ts @@ -11,7 +11,7 @@ import path from 'node:path'; import { EXECUTION_TRACE_SCHEMA_VERSION, type PreparedAttemptMetadata, - type ResolvedTarget, + type ResolvedProviderBackend, type Trace, type TranscriptJsonLine, buildTraceFromMessages, @@ -555,7 +555,7 @@ async function gradePreparedAttempt(options: { const target = { ...selection.resolvedTarget, name: manifest.target, - } as ResolvedTarget; + } as ResolvedProviderBackend; const response = options.responsePath !== undefined diff --git a/apps/cli/src/commands/prepare/index.ts b/apps/cli/src/commands/prepare/index.ts index 6cef80519..7dfefb00c 100644 --- a/apps/cli/src/commands/prepare/index.ts +++ b/apps/cli/src/commands/prepare/index.ts @@ -11,7 +11,7 @@ import { type JsonObject, type PreparedEvalWorkspace, type PreparedWorkspaceRepoPin, - type ResolvedTarget, + type ResolvedProviderBackend, type TargetHooksConfig, deriveCategory, loadTestSuite, @@ -263,7 +263,7 @@ async function selectPrepareTarget(options: { readonly target: string; readonly targetRefs?: readonly EvalTargetRef[]; }): Promise<{ - readonly resolvedTarget: ResolvedTarget; + readonly resolvedTarget: ResolvedProviderBackend; readonly targetHooks?: TargetHooksConfig; }> { const selections = await selectMultipleTargets({ @@ -282,7 +282,7 @@ async function selectPrepareTarget(options: { resolvedTarget: { ...selection.resolvedTarget, name: options.target, - } as ResolvedTarget, + } as ResolvedProviderBackend, ...(selection.targetHooks !== undefined && { targetHooks: selection.targetHooks }), }; } diff --git a/apps/cli/src/commands/results/eval-runner.ts b/apps/cli/src/commands/results/eval-runner.ts index 159c44f99..8abba035f 100644 --- a/apps/cli/src/commands/results/eval-runner.ts +++ b/apps/cli/src/commands/results/eval-runner.ts @@ -23,7 +23,7 @@ import { type ChildProcess, execFileSync, spawn } from 'node:child_process'; import { type WriteStream, createWriteStream, existsSync, mkdirSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { listTargetNames, readTargetDefinitions } from '@agentv/core'; +import { listProviderLabels, readProviderDefinitions } from '@agentv/core'; import type { Context } from 'hono'; import type { Hono } from 'hono'; @@ -131,8 +131,8 @@ async function discoverTargetsInProject(cwd: string): Promise if (!targetsFilePath) return []; try { - const definitions = await readTargetDefinitions(targetsFilePath); - return listTargetNames(definitions); + const definitions = await readProviderDefinitions(targetsFilePath); + return listProviderLabels(definitions); } catch { return []; } diff --git a/apps/cli/src/commands/runs/rerun.ts b/apps/cli/src/commands/runs/rerun.ts index 19185c447..5e3994abf 100644 --- a/apps/cli/src/commands/runs/rerun.ts +++ b/apps/cli/src/commands/runs/rerun.ts @@ -2,7 +2,10 @@ import { constants } from 'node:fs'; import { access, readFile } from 'node:fs/promises'; import path from 'node:path'; -import { parseYamlValue, readTargetDefinitions as readCoreTargetDefinitions } from '@agentv/core'; +import { + parseYamlValue, + readProviderDefinitions as readCoreProviderDefinitions, +} from '@agentv/core'; import { array, command, @@ -115,7 +118,7 @@ async function readTaskTarget(evalPath: string, fallback: string): Promise[]> { - const definitions = await readCoreTargetDefinitions(providersPath); + const definitions = await readCoreProviderDefinitions(providersPath); return definitions.map((definition) => definition as unknown as Record); } diff --git a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx index 985f80fe9..1e48888b3 100644 --- a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx +++ b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx @@ -64,7 +64,7 @@ Use `createBuiltinProviderRegistry()` to get a registry pre-loaded with all buil import { createBuiltinProviderRegistry, type ProviderFactoryFn, - type ResolvedTarget, + type ResolvedProviderBackend, type Provider, type ProviderRequest, type ProviderResponse, @@ -72,7 +72,7 @@ import { const registry = createBuiltinProviderRegistry(); -registry.register('my-provider', (target: ResolvedTarget): Provider => { +registry.register('my-provider', (target: ResolvedProviderBackend): Provider => { return { id: `my-provider:${target.name}`, kind: 'cli', // use 'cli' as the kind for custom providers @@ -90,12 +90,12 @@ registry.register('my-provider', (target: ResolvedTarget): Provider => { The `register()` method takes two arguments: 1. **kind** (`string`) -- A unique identifier for your provider. This is the value used in `provider:` in providers.yaml. -2. **factory** (`ProviderFactoryFn`) -- A function that receives a `ResolvedTarget` and returns a `Provider` instance. +2. **factory** (`ProviderFactoryFn`) -- A function that receives a `ResolvedProviderBackend` and returns a `Provider` instance. The factory function signature: ```typescript -type ProviderFactoryFn = (target: ResolvedTarget) => Provider; +type ProviderFactoryFn = (target: ResolvedProviderBackend) => Provider; ``` ## Example: Wrapping an HTTP API @@ -108,7 +108,7 @@ import { type Provider, type ProviderRequest, type ProviderResponse, - type ResolvedTarget, + type ResolvedProviderBackend, } from '@agentv/core'; class HttpAgentProvider implements Provider { @@ -164,7 +164,7 @@ class HttpAgentProvider implements Provider { // Register the provider const registry = createBuiltinProviderRegistry(); -registry.register('http-agent', (target: ResolvedTarget) => { +registry.register('http-agent', (target: ResolvedProviderBackend) => { const config = target.config as { baseUrl: string; apiKey: string }; return new HttpAgentProvider(target.name, { baseUrl: config.baseUrl ?? 'http://localhost:8080', diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index a3541437f..00a0ee7a2 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -75,9 +75,9 @@ import type { EvalMetadata } from './metadata.js'; import { runEvaluation } from './orchestrator.js'; import { createFunctionProvider } from './providers/function-provider.js'; import type { ProviderFactoryFn } from './providers/provider-registry.js'; -import { readTargetDefinitions } from './providers/targets-file.js'; -import { type ResolvedTarget, resolveTargetDefinition } from './providers/targets.js'; -import type { TargetDefinition } from './providers/types.js'; +import { readProviderDefinitions } from './providers/targets-file.js'; +import { type ResolvedProviderBackend, resolveProviderDefinition } from './providers/targets.js'; +import type { ProviderDefinition } from './providers/types.js'; import { INLINE_ASSERT_FN } from './registry/builtin-graders.js'; import { writeArtifactsFromResults } from './run-artifacts.js'; import type { @@ -177,7 +177,7 @@ export interface EvalConfig { /** Prompt templates for inline tests. Use tests[].vars for per-row values. */ readonly prompts?: readonly (string | readonly { role: string; content: string }[])[]; /** Target provider configuration */ - readonly target?: TargetDefinition; + readonly target?: ProviderDefinition; /** Custom task function — mutually exclusive with target */ readonly task?: (input: string) => string | Promise; /** Suite-level assert entries applied to all tests */ @@ -221,7 +221,7 @@ export interface MaterializedEvalConfig { readonly budgetUsd?: number; readonly threshold?: number; readonly metadata?: EvalMetadata; - readonly target?: TargetDefinition; + readonly target?: ProviderDefinition; readonly task?: (input: string) => string | Promise; readonly providerFactory?: ProviderFactoryFn; } @@ -324,7 +324,7 @@ export async function evaluate(config: EvalConfig): Promise { // files participate even when the command is launched from a parent folder. await loadEnvHierarchy(repoRoot, testFilePath); - let resolvedTarget: ResolvedTarget; + let resolvedTarget: ResolvedProviderBackend; let providerFactory: ProviderFactoryFn | undefined; if (config.task || materialized.providerFactory) { providerFactory = config.task @@ -337,7 +337,7 @@ export async function evaluate(config: EvalConfig): Promise { }; } else { // Resolve target — inline definition or auto-discover from providers.yaml - let targetDef: TargetDefinition; + let targetDef: ProviderDefinition; if (config.target) { targetDef = config.target; } else if (materialized.target) { @@ -345,7 +345,7 @@ export async function evaluate(config: EvalConfig): Promise { } else { targetDef = (await discoverDefaultTarget(repoRoot)) ?? { name: 'default', provider: 'mock' }; } - resolvedTarget = resolveTargetDefinition(targetDef); + resolvedTarget = resolveProviderDefinition(targetDef); } const collectedResults: EvaluationResult[] = []; @@ -748,7 +748,7 @@ const PROVIDER_FILE_CANDIDATES = ['.agentv/providers.yaml', '.agentv/providers.y /** * Auto-discover the 'default' provider from providers.yaml in the repo tree. */ -async function discoverDefaultTarget(repoRoot: string): Promise { +async function discoverDefaultTarget(repoRoot: string): Promise { const cwd = process.cwd(); const chain = buildDirectoryChain(path.join(cwd, '_placeholder'), repoRoot); @@ -757,7 +757,7 @@ async function discoverDefaultTarget(repoRoot: string): Promise d.name === 'default'); if (defaultTarget) return defaultTarget; } catch { diff --git a/packages/core/src/evaluation/graders/types.ts b/packages/core/src/evaluation/graders/types.ts index 8055e7f3d..26f1af312 100644 --- a/packages/core/src/evaluation/graders/types.ts +++ b/packages/core/src/evaluation/graders/types.ts @@ -1,4 +1,4 @@ -import type { ResolvedTarget } from '../providers/targets.js'; +import type { ResolvedProviderBackend } from '../providers/targets.js'; import type { ChatPrompt, Message, Provider } from '../providers/types.js'; import type { TokenUsage, Trace } from '../trace.js'; import type { @@ -25,7 +25,7 @@ export interface EvaluationContext { readonly candidateValue?: unknown; /** JSON-safe provider response metadata available to transforms. */ readonly responseMetadata?: JsonObject; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly provider: Provider; readonly attempt: number; readonly promptInputs: { diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 9811f2841..3e694a243 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -12,7 +12,7 @@ import { import { getAgentvConfigDir } from '../../paths.js'; import { createEvalConfigEnv, interpolateEnv } from '../interpolation.js'; import { normalizeProviderDefinition } from '../providers/targets.js'; -import type { TargetDefinition } from '../providers/types.js'; +import type { ProviderDefinition } from '../providers/types.js'; import type { EvalTargetRef, FailOnError, @@ -106,7 +106,7 @@ export type AgentVConfig = { /** Resolved file path when top-level `providers` was authored as a file:// reference. */ readonly providerCatalogPath?: string; /** Provider definitions resolved from top-level `providers`, including inline arrays and file refs. */ - readonly providerDefinitions?: readonly TargetDefinition[]; + readonly providerDefinitions?: readonly ProviderDefinition[]; } & ComposableConfigGraph; /** @@ -314,7 +314,7 @@ function parseConfigObject( function parseProviderDefinitions( rawProviders: unknown, configPath: string, -): readonly TargetDefinition[] | undefined { +): readonly ProviderDefinition[] | undefined { if (rawProviders === undefined) { return undefined; } @@ -565,7 +565,7 @@ function parseEvalProviderRef(raw: unknown, location: string): EvalTargetRef { const definition = normalizeProviderDefinition( Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'hooks')), { location }, - ) as TargetDefinition; + ) as ProviderDefinition; return { name: definition.name, diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index 1bb46bf75..c58ac913f 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -12,7 +12,7 @@ import { pathToFileURL } from 'node:url'; import { type EvalConfig as ProgrammaticEvalConfig, materializeEvalConfig } from '../evaluate.js'; import { createFunctionProvider } from '../providers/function-provider.js'; import type { ProviderFactoryFn } from '../providers/provider-registry.js'; -import type { TargetDefinition } from '../providers/types.js'; +import type { ProviderDefinition } from '../providers/types.js'; import { type EvalSuiteResult, loadTestSuiteFromYamlObject } from '../yaml-parser.js'; const SDK_EVAL_SUITE_SYMBOL = Symbol.for('@agentv/sdk/eval-suite'); @@ -63,7 +63,7 @@ export interface TsEvalResult { } export interface TsEvalSuiteResult extends EvalSuiteResult { - readonly inlineTarget?: TargetDefinition; + readonly inlineTarget?: ProviderDefinition; readonly providerFactory?: ProviderFactoryFn; } diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index ae4e2e940..23aae2461 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -26,9 +26,9 @@ import { import { createBuiltinProviderRegistry, createProvider } from './providers/index.js'; import { discoverProviders } from './providers/provider-discovery.js'; import { - type ResolvedTarget, - resolveDelegatedTargetDefinition, - resolveTargetDefinition, + type ResolvedProviderBackend, + resolveDelegatedProviderDefinition, + resolveProviderDefinition, } from './providers/targets.js'; import type { ChatMessage, @@ -36,10 +36,10 @@ import type { EnvLookup, Message, Provider, + ProviderDefinition, ProviderRequest, ProviderResponse, ProviderStreamCallbacks, - TargetDefinition, } from './providers/types.js'; import { LLM_GRADER_CAPABLE_KINDS, @@ -179,10 +179,10 @@ function mergeTextOutput(left: string | undefined, right: string | undefined): s } interface EvaluationRuntimeOptions { - readonly target: ResolvedTarget; - readonly targets?: readonly TargetDefinition[]; + readonly target: ResolvedProviderBackend; + readonly targets?: readonly ProviderDefinition[]; readonly env?: EnvLookup; - readonly providerFactory?: (target: ResolvedTarget) => Provider; + readonly providerFactory?: (target: ResolvedProviderBackend) => Provider; readonly evalFilePath?: string; readonly graderTarget?: string; /** Config-level fallback grader target name (`.agentv/config.yaml`'s `defaults.grader`), used when a target has no `grader_target` and no CLI `--grader-target` override is given. */ @@ -191,8 +191,10 @@ interface EvaluationRuntimeOptions { } interface EvaluationRuntime { - readonly getOrCreateProvider: (resolved: ResolvedTarget) => Provider; - readonly resolveGraderProvider: (targetContext: ResolvedTarget) => Promise; + readonly getOrCreateProvider: (resolved: ResolvedProviderBackend) => Provider; + readonly resolveGraderProvider: ( + targetContext: ResolvedProviderBackend, + ) => Promise; readonly targetResolver: (name: string) => Provider | undefined; readonly availableTargets: readonly string[]; } @@ -208,10 +210,10 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR defaultGraderTarget, model: cliModel, } = options; - const resolvedTargetsByName = new Map(); + const resolvedTargetsByName = new Map(); resolvedTargetsByName.set(target.name, target); - const targetDefinitions = new Map(); + const targetDefinitions = new Map(); for (const definition of targets ?? []) { targetDefinitions.set(definition.name, definition); } @@ -219,7 +221,7 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR const envLookup: EnvLookup = env ?? process.env; const providerCache = new Map(); - const getOrCreateProvider = (resolved: ResolvedTarget): Provider => { + const getOrCreateProvider = (resolved: ResolvedProviderBackend): Provider => { const existing = providerCache.get(resolved.name); if (existing) { return existing; @@ -230,21 +232,21 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR return instance; }; - const resolveTargetByName = (name: string): ResolvedTarget | undefined => { + const resolveTargetByName = (name: string): ResolvedProviderBackend | undefined => { if (resolvedTargetsByName.has(name)) { return resolvedTargetsByName.get(name); } - const definition = resolveDelegatedTargetDefinition(name, targetDefinitions, envLookup); + const definition = resolveDelegatedProviderDefinition(name, targetDefinitions, envLookup); if (!definition) { return undefined; } - const resolved = resolveTargetDefinition(definition, envLookup, evalFilePath ?? ''); + const resolved = resolveProviderDefinition(definition, envLookup, evalFilePath ?? ''); resolvedTargetsByName.set(name, resolved); return resolved; }; const resolveGraderProvider = async ( - targetContext: ResolvedTarget, + targetContext: ResolvedProviderBackend, ): Promise => { // CLI --grader-target takes highest priority. if (cliGraderTarget) { @@ -424,8 +426,8 @@ export interface EvaluationCache { export interface RunEvalCaseOptions { readonly evalCase: EvalTest; readonly provider: Provider; - readonly providerFactory?: (target: ResolvedTarget) => Provider; - readonly target: ResolvedTarget; + readonly providerFactory?: (target: ResolvedProviderBackend) => Provider; + readonly target: ResolvedProviderBackend; readonly evaluators: Partial> & { readonly 'llm-grader': Grader }; readonly now?: () => Date; readonly maxRetries?: number; @@ -504,10 +506,10 @@ export interface ProgressEvent { export interface RunEvaluationOptions { readonly testFilePath: string; readonly repoRoot: URL | string; - readonly target: ResolvedTarget; - readonly targets?: readonly TargetDefinition[]; + readonly target: ResolvedProviderBackend; + readonly targets?: readonly ProviderDefinition[]; readonly env?: EnvLookup; - readonly providerFactory?: (target: ResolvedTarget) => Provider; + readonly providerFactory?: (target: ResolvedProviderBackend) => Provider; readonly evaluators?: Partial>; readonly maxRetries?: number; readonly agentTimeoutMs?: number; @@ -573,11 +575,11 @@ export interface PreparedAttemptMetadata { export interface GradePreparedEvalCaseOptions { readonly evalCase: EvalTest; - readonly target: ResolvedTarget; - readonly targets?: readonly TargetDefinition[]; + readonly target: ResolvedProviderBackend; + readonly targets?: readonly ProviderDefinition[]; readonly env?: EnvLookup; readonly evaluators?: Partial>; - readonly providerFactory?: (target: ResolvedTarget) => Provider; + readonly providerFactory?: (target: ResolvedProviderBackend) => Provider; readonly now?: () => Date; readonly agentTimeoutMs?: number; readonly graderTarget?: string; @@ -592,7 +594,7 @@ export interface GradePreparedEvalCaseOptions { readonly preparedAttempt: PreparedAttemptMetadata; } -function createPreparedProvider(target: ResolvedTarget): Provider { +function createPreparedProvider(target: ResolvedProviderBackend): Provider { return { id: `prepared:${target.name}`, kind: target.kind, @@ -1588,7 +1590,7 @@ export async function runEvaluation( async function runBatchEvaluation(options: { readonly evalCases: readonly EvalTest[]; readonly provider: Provider; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly evaluatorRegistry: Partial> & { readonly 'llm-grader': Grader; }; @@ -1597,7 +1599,9 @@ async function runBatchEvaluation(options: { readonly onProgress?: (event: ProgressEvent) => MaybePromise; readonly onResult?: (result: EvaluationResult) => MaybePromise; readonly verbose?: boolean; - readonly resolveGraderProvider: (target: ResolvedTarget) => Promise; + readonly resolveGraderProvider: ( + target: ResolvedProviderBackend, + ) => Promise; readonly agentTimeoutMs?: number; readonly targetResolver?: (name: string) => Provider | undefined; readonly availableTargets?: readonly string[]; @@ -1829,7 +1833,7 @@ async function maybeRecordReplayFixture(options: { readonly evalCase: EvalTest; readonly evalFilePath?: string; readonly repoRoot?: string; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly attempt: number; readonly response: ProviderResponse; readonly nowFn: () => Date; @@ -2727,7 +2731,7 @@ async function prepareCandidateForGrading(options: { async function evaluateCandidate(options: { readonly evalCase: EvalTest; readonly candidate: string; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly provider: Provider; readonly evaluators: Partial> & { readonly 'llm-grader': Grader }; readonly typeRegistry: import('./registry/grader-registry.js').GraderRegistry; @@ -2920,7 +2924,7 @@ async function runEvaluatorsForCase(options: { readonly candidate: string; readonly candidateValue?: unknown; readonly responseMetadata?: JsonObject; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly provider: Provider; readonly evaluators: Partial> & { readonly 'llm-grader': Grader }; readonly typeRegistry: import('./registry/grader-registry.js').GraderRegistry; @@ -3128,7 +3132,7 @@ async function runEvaluatorList(options: { readonly candidate: string; readonly candidateValue?: unknown; readonly responseMetadata?: JsonObject; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly provider: Provider; readonly evaluatorRegistry: Partial> & { readonly 'llm-grader': Grader; @@ -3390,7 +3394,7 @@ function filterEvalCases( function buildEvaluatorRegistry( overrides: Partial> | undefined, - resolveGraderProvider: (target: ResolvedTarget) => Promise, + resolveGraderProvider: (target: ResolvedProviderBackend) => Promise, ): Partial> & { readonly 'llm-grader': Grader } { const llmGrader = overrides?.['llm-grader'] ?? @@ -3422,7 +3426,7 @@ function buildEvaluatorRegistry( async function runConversationMode(options: { readonly evalCase: EvalTest; readonly provider: Provider; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly evaluators: Partial> & { readonly 'llm-grader': Grader }; readonly typeRegistry: import('./registry/grader-registry.js').GraderRegistry; readonly graderProvider?: Provider; @@ -3813,7 +3817,7 @@ async function invokeProvider( provider: Provider, options: { readonly evalCase: EvalTest; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly promptInputs: PromptInputs; readonly attempt: number; readonly evalFilePath?: string; @@ -3991,7 +3995,7 @@ function extractProviderError(response: ProviderResponse): string | undefined { function createCacheKey( provider: Provider, - target: ResolvedTarget, + target: ResolvedProviderBackend, evalCase: EvalTest, promptInputs: PromptInputs, ): string { diff --git a/packages/core/src/evaluation/prepared-workspace.ts b/packages/core/src/evaluation/prepared-workspace.ts index 3ed8f72d7..be5830ae2 100644 --- a/packages/core/src/evaluation/prepared-workspace.ts +++ b/packages/core/src/evaluation/prepared-workspace.ts @@ -15,7 +15,7 @@ import { randomUUID } from 'node:crypto'; import path from 'node:path'; import micromatch from 'micromatch'; -import type { ResolvedTarget } from './providers/targets.js'; +import type { ResolvedProviderBackend } from './providers/targets.js'; import type { ChatPrompt } from './providers/types.js'; import { AGENT_PROVIDER_KINDS } from './providers/types.js'; import type { EvalTest, JsonObject, RepoConfig, TargetHooksConfig } from './types.js'; @@ -33,7 +33,7 @@ import { type PromptInputs, buildPromptInputs, loadTests } from './yaml-parser.j export interface PrepareEvalWorkspaceOptions { readonly testFilePath: string; readonly repoRoot: URL | string; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly targetHooks?: TargetHooksConfig; readonly evalCases?: readonly EvalTest[]; /** Exact test id to prepare. */ @@ -130,7 +130,7 @@ function selectSingleCase(options: { return selected[0]; } -function promptModeForTarget(target: ResolvedTarget): 'agent' | 'lm' { +function promptModeForTarget(target: ResolvedProviderBackend): 'agent' | 'lm' { return AGENT_PROVIDER_KINDS.includes(target.kind) || target.kind === 'cli' ? 'agent' : 'lm'; } diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 18f5ba8e9..6623bca64 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -16,19 +16,19 @@ import { PiRpcProvider } from './pi-rpc.js'; import { ProviderRegistry } from './provider-registry.js'; import { ReplayProvider } from './replay.js'; import { SdkChildProvider } from './sdk-child-provider.js'; -import type { ResolvedTarget } from './targets.js'; +import type { ResolvedProviderBackend } from './targets.js'; import { - COMMON_TARGET_SETTINGS, - resolveDelegatedTargetDefinition, - resolveTargetDefinition, + COMMON_PROVIDER_SETTINGS, + resolveDelegatedProviderDefinition, + resolveProviderDefinition, } from './targets.js'; import type { EnvLookup, Provider, + ProviderDefinition, ProviderKind, ProviderRequest, ProviderResponse, - TargetDefinition, } from './types.js'; import { VSCodeProvider } from './vscode-provider.js'; @@ -41,7 +41,7 @@ export type { ProviderResponse, ProviderStreamCallbacks, ProviderTokenUsage, - TargetDefinition, + ProviderDefinition, ToolCall, } from './types.js'; @@ -66,12 +66,12 @@ export type { PiRpcResolvedConfig, ReplayResolvedConfig, ReplayResolvedSource, - ResolvedTarget, + ResolvedProviderBackend, VSCodeResolvedConfig, } from './targets.js'; -export { COMMON_TARGET_SETTINGS, resolveDelegatedTargetDefinition, resolveTargetDefinition }; -export { readTargetDefinitions, listTargetNames } from './targets-file.js'; +export { COMMON_PROVIDER_SETTINGS, resolveDelegatedProviderDefinition, resolveProviderDefinition }; +export { readProviderDefinitions, listProviderLabels } from './targets-file.js'; export { ensureVSCodeSubagents, type EnsureSubagentsOptions, @@ -161,11 +161,11 @@ class UnsupportedSandboxProvider implements Provider { } } -function usesSandboxRuntime(target: ResolvedTarget): boolean { +function usesSandboxRuntime(target: ResolvedProviderBackend): boolean { return target.runtime?.mode === 'sandbox'; } -function unsupportedSandboxProvider(target: ResolvedTarget): Provider { +function unsupportedSandboxProvider(target: ResolvedProviderBackend): Provider { return new UnsupportedSandboxProvider( target.kind as ProviderKind, target.name, @@ -261,14 +261,14 @@ const defaultProviderRegistry = createBuiltinProviderRegistry(); * Create a provider from a resolved target using the default registry. * Custom providers can be registered via `createBuiltinProviderRegistry().register()`. */ -export function createProvider(target: ResolvedTarget): Provider { +export function createProvider(target: ResolvedProviderBackend): Provider { return defaultProviderRegistry.create(target); } export function resolveAndCreateProvider( - definition: TargetDefinition, + definition: ProviderDefinition, env: EnvLookup = process.env, ): Provider { - const resolved = resolveTargetDefinition(definition, env); + const resolved = resolveProviderDefinition(definition, env); return createProvider(resolved); } diff --git a/packages/core/src/evaluation/providers/provider-registry.ts b/packages/core/src/evaluation/providers/provider-registry.ts index c9a37c4fd..30310a596 100644 --- a/packages/core/src/evaluation/providers/provider-registry.ts +++ b/packages/core/src/evaluation/providers/provider-registry.ts @@ -7,13 +7,13 @@ * dropping files in `.agentv/providers/`. */ -import type { ResolvedTarget } from './targets.js'; +import type { ResolvedProviderBackend } from './targets.js'; import type { Provider } from './types.js'; /** * Factory function that creates a Provider instance from a resolved target. */ -export type ProviderFactoryFn = (target: ResolvedTarget) => Provider; +export type ProviderFactoryFn = (target: ResolvedProviderBackend) => Provider; /** * Registry of provider factory functions keyed by provider kind. @@ -49,7 +49,7 @@ export class ProviderRegistry { * Create a provider instance from a resolved target. * Falls back to CLI provider for unknown kinds (custom provider escape hatch). */ - create(target: ResolvedTarget): Provider { + create(target: ResolvedProviderBackend): Provider { const factory = this.factories.get(target.kind); if (!factory) { throw new Error( diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index 0edf80d7d..885350c34 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { parseYamlValue } from '../yaml-loader.js'; import { normalizeProviderDefinition } from './targets.js'; import { TARGETS_SCHEMA_V2 } from './types.js'; -import type { TargetDefinition } from './types.js'; +import type { ProviderDefinition } from './types.js'; function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -36,7 +36,7 @@ function assertProviderDefinition( value: unknown, index: number, filePath: string, -): TargetDefinition { +): ProviderDefinition { if (!isRecord(value)) { throw new Error(`providers entry at index ${index} in ${filePath} must be an object`); } @@ -59,9 +59,9 @@ async function fileExists(filePath: string): Promise { } } -export async function readTargetDefinitions( +export async function readProviderDefinitions( filePath: string, -): Promise { +): Promise { const absolutePath = path.resolve(filePath); if (!(await fileExists(absolutePath))) { throw new Error(`providers.yaml not found at ${absolutePath}`); @@ -77,6 +77,6 @@ export async function readTargetDefinitions( return definitions; } -export function listTargetNames(definitions: readonly TargetDefinition[]): readonly string[] { +export function listProviderLabels(definitions: readonly ProviderDefinition[]): readonly string[] { return definitions.map((definition) => definition.name); } diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index b17baf1ff..464c34a93 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; import { renderEnvTemplateString } from '../interpolation.js'; import type { TargetRuntimeConfig, TargetRuntimeMode } from './sandbox-runner.js'; -import type { EnvLookup, TargetDefinition } from './types.js'; +import type { EnvLookup, ProviderDefinition } from './types.js'; // --------------------------------------------------------------------------- // Zod Schemas for CLI Provider Configuration @@ -686,7 +686,7 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -export interface NormalizeTargetDefinitionOptions { +export interface NormalizeInternalProviderDefinitionOptions { readonly defaultName?: string; } @@ -822,13 +822,13 @@ function assertCrossPlatformExecProviderSpec(spec: string, providerId: string): /** * Converts the public Promptfoo-shaped provider object into AgentV's internal - * target definition. Public YAML uses `providers[].id` for the backend/provider + * provider definition. Public YAML uses `providers[].id` for the backend/provider * spec and `providers[].label` for the stable AgentV result/selection key. */ export function normalizeProviderDefinition( definition: unknown, options: NormalizeProviderDefinitionOptions = {}, -): TargetDefinition { +): ProviderDefinition { const location = options.location ?? 'provider'; if (!isRecord(definition)) { throw new Error(`Invalid ${location}: provider must be an object.`); @@ -868,7 +868,7 @@ export function normalizeProviderDefinition( const publicSpec = normalizePublicProviderId(providerId); const authoredConfig = isRecord(rest.config) ? rest.config : {}; - return normalizeTargetDefinition({ + return normalizeInternalProviderDefinition({ ...rest, config: { ...publicSpec.config, @@ -879,17 +879,13 @@ export function normalizeProviderDefinition( }); } -/** - * Converts the authored target object into AgentV's internal target definition. - * Authored YAML uses `id` as the stable AgentV target identity. The runtime - * continues to use `name` as its resolver and artifact key. - */ -export function normalizeTargetDefinition( +/** Normalizes an internal provider definition object for resolver use. */ +export function normalizeInternalProviderDefinition( definition: unknown, - options: NormalizeTargetDefinitionOptions = {}, -): TargetDefinition { + options: NormalizeInternalProviderDefinitionOptions = {}, +): ProviderDefinition { if (!isRecord(definition)) { - throw new Error('Target definition must be an object'); + throw new Error('Provider definition must be an object'); } assertNoTargetTestbedFields(definition); @@ -903,7 +899,7 @@ export function normalizeTargetDefinition( typeof rawName === 'string' && rawName.trim().length > 0 ? rawName.trim() : undefined; const name = id ?? legacyName ?? label ?? options.defaultName; if (!name || name.trim().length === 0) { - throw new Error("Target definition is missing a valid 'id' field"); + throw new Error("Provider definition is missing a valid 'id' field"); } const config = isRecord(definition.config) ? definition.config : {}; @@ -913,22 +909,22 @@ export function normalizeTargetDefinition( ...(id !== undefined ? { id } : {}), label: label ?? name, name, - } as unknown as TargetDefinition; + } as unknown as ProviderDefinition; } function assertNoTargetTestbedFields(definition: Record): void { if (definition.environment !== undefined) { throw new Error( - 'Target definitions cannot include environment; author environment at suite/test/case scope.', + 'Provider definitions cannot include environment; author environment at suite/test/case scope.', ); } if (definition.container !== undefined) { throw new Error( - 'Target definitions cannot include container setup; use an environment recipe.', + 'Provider definitions cannot include container setup; use an environment recipe.', ); } if (definition.install !== undefined) { - throw new Error('Target definitions cannot include install steps; use environment.setup.'); + throw new Error('Provider definitions cannot include install steps; use environment.setup.'); } } @@ -954,7 +950,7 @@ function collectDeprecatedCamelCaseWarnings( return warnings; } -function assertNoDeprecatedCamelCaseTargetFields(definition: TargetDefinition): void { +function assertNoDeprecatedCamelCaseTargetFields(definition: ProviderDefinition): void { const warning = findDeprecatedCamelCaseTargetWarnings( definition, `target "${definition.name}"`, @@ -972,7 +968,7 @@ function assertNoDeprecatedCamelCaseTargetFields(definition: TargetDefinition): ); } -function assertNoRemovedTargetFields(definition: TargetDefinition): void { +function assertNoRemovedTargetFields(definition: ProviderDefinition): void { const rawDefinition = definition as unknown as Record; if (Object.prototype.hasOwnProperty.call(rawDefinition, 'judge_target')) { throw new Error( @@ -1031,8 +1027,8 @@ export type CliHealthcheck = Readonly; // Note: CliResolvedConfig is a type alias derived from CliNormalizedConfig (see above), // which itself is inferred from CliTargetConfigSchema for type safety and single source of truth. -/** Base fields shared by all resolved targets. */ -interface ResolvedTargetBase { +/** Base fields shared by all resolved provider backends. */ +interface ResolvedProviderBackendBase { readonly name: string; readonly label?: string; readonly runtime?: TargetRuntimeConfig; @@ -1052,51 +1048,80 @@ interface ResolvedTargetBase { readonly fallbackTargets?: readonly string[]; } -export type ResolvedTarget = - | (ResolvedTargetBase & { readonly kind: 'openai'; readonly config: OpenAIResolvedConfig }) - | (ResolvedTargetBase & { +export type ResolvedProviderBackend = + | (ResolvedProviderBackendBase & { + readonly kind: 'openai'; + readonly config: OpenAIResolvedConfig; + }) + | (ResolvedProviderBackendBase & { readonly kind: 'openrouter'; readonly config: OpenRouterResolvedConfig; }) - | (ResolvedTargetBase & { readonly kind: 'azure'; readonly config: AzureResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'anthropic'; readonly config: AnthropicResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'gemini'; readonly config: GeminiResolvedConfig }) - | (ResolvedTargetBase & { + | (ResolvedProviderBackendBase & { readonly kind: 'azure'; readonly config: AzureResolvedConfig }) + | (ResolvedProviderBackendBase & { + readonly kind: 'anthropic'; + readonly config: AnthropicResolvedConfig; + }) + | (ResolvedProviderBackendBase & { + readonly kind: 'gemini'; + readonly config: GeminiResolvedConfig; + }) + | (ResolvedProviderBackendBase & { readonly kind: 'codex-cli' | 'codex-app-server' | 'codex-sdk'; readonly config: CodexResolvedConfig; }) - | (ResolvedTargetBase & { + | (ResolvedProviderBackendBase & { readonly kind: 'copilot-sdk'; readonly config: CopilotSdkResolvedConfig; }) - | (ResolvedTargetBase & { + | (ResolvedProviderBackendBase & { readonly kind: 'copilot-cli'; readonly config: CopilotCliResolvedConfig; }) - | (ResolvedTargetBase & { + | (ResolvedProviderBackendBase & { readonly kind: 'pi-sdk' | 'pi-coding-agent'; readonly config: PiCodingAgentResolvedConfig; }) - | (ResolvedTargetBase & { readonly kind: 'pi-cli'; readonly config: PiCliResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'pi-rpc'; readonly config: PiRpcResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'claude-cli'; readonly config: ClaudeResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'claude-sdk'; readonly config: ClaudeResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'mock'; readonly config: MockResolvedConfig }) - | (ResolvedTargetBase & { + | (ResolvedProviderBackendBase & { + readonly kind: 'pi-cli'; + readonly config: PiCliResolvedConfig; + }) + | (ResolvedProviderBackendBase & { + readonly kind: 'pi-rpc'; + readonly config: PiRpcResolvedConfig; + }) + | (ResolvedProviderBackendBase & { + readonly kind: 'claude-cli'; + readonly config: ClaudeResolvedConfig; + }) + | (ResolvedProviderBackendBase & { + readonly kind: 'claude-sdk'; + readonly config: ClaudeResolvedConfig; + }) + | (ResolvedProviderBackendBase & { readonly kind: 'mock'; readonly config: MockResolvedConfig }) + | (ResolvedProviderBackendBase & { readonly kind: 'vscode' | 'vscode-insiders'; readonly config: VSCodeResolvedConfig; }) - | (ResolvedTargetBase & { readonly kind: 'agentv'; readonly config: AgentVResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'cli'; readonly config: CliResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'transcript'; readonly config: Record }) - | (ResolvedTargetBase & { readonly kind: 'replay'; readonly config: ReplayResolvedConfig }); + | (ResolvedProviderBackendBase & { + readonly kind: 'agentv'; + readonly config: AgentVResolvedConfig; + }) + | (ResolvedProviderBackendBase & { readonly kind: 'cli'; readonly config: CliResolvedConfig }) + | (ResolvedProviderBackendBase & { + readonly kind: 'transcript'; + readonly config: Record; + }) + | (ResolvedProviderBackendBase & { + readonly kind: 'replay'; + readonly config: ReplayResolvedConfig; + }); /** - * Optional settings accepted on ALL target definitions regardless of provider. - * Exported so the targets validator can reuse the same list — adding a field - * here automatically makes it valid in targets.yaml without a separate update. + * Optional settings accepted on all provider definitions regardless of backend. + * Exported so provider validation can reuse the same list. */ -export const COMMON_TARGET_SETTINGS = [ +export const COMMON_PROVIDER_SETTINGS = [ 'runtime', 'batch_requests', 'subagent_mode_allowed', @@ -1208,11 +1233,11 @@ function resolveTargetRuntime( ); } -export function resolveDelegatedTargetDefinition( +export function resolveDelegatedProviderDefinition( name: string, - definitions: ReadonlyMap, + definitions: ReadonlyMap, env: EnvLookup = process.env, -): TargetDefinition | undefined { +): ProviderDefinition | undefined { let definition = definitions.get(name); if (!definition) { return undefined; @@ -1278,21 +1303,21 @@ export function resolveDelegatedTargetDefinition( ); } -export function resolveTargetDefinition( - definition: TargetDefinition, +export function resolveProviderDefinition( + definition: ProviderDefinition, env: EnvLookup = process.env, evalFilePath?: string, options?: { readonly emitDeprecationWarnings?: boolean }, -): ResolvedTarget { +): ResolvedProviderBackend { void options; - const normalizedDefinition = normalizeTargetDefinition(definition); + const normalizedDefinition = normalizeInternalProviderDefinition(definition); assertNoRemovedTargetFields(normalizedDefinition); assertNoDeprecatedCamelCaseTargetFields(normalizedDefinition); const parsed = BASE_TARGET_SCHEMA.parse(normalizedDefinition); if (!parsed.provider) { throw new Error( - `${parsed.name}: 'provider' is required (targets with use_target must be resolved before calling resolveTargetDefinition)`, + `${parsed.name}: 'provider' is required (provider definitions with use_target must be resolved before calling resolveProviderDefinition)`, ); } const provider = resolveString( diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index db0c53337..7f388addd 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -449,12 +449,12 @@ export interface Provider { export type EnvLookup = Readonly>; -export interface TargetDefinition { - /** Internal canonical target identity. Authored YAML uses `id`; loaders map id -> name. */ +export interface ProviderDefinition { + /** Internal canonical provider label. Authored YAML uses providers[].label, falling back to providers[].id. */ readonly name: string; - /** Authored AgentV target identity. */ + /** Normalized AgentV provider label retained for resolver compatibility. */ readonly id?: string | undefined; - /** Internal compatibility display label; authored `label` is rejected at the boundary. */ + /** Display label used by result identity and selection. */ readonly label?: string | undefined; /** Promptfoo-shaped provider options bag. Provider settings are flattened at the boundary. */ readonly config?: unknown | undefined; @@ -465,7 +465,7 @@ export interface TargetDefinition { readonly provider?: ProviderKind | string; /** Original public providers[].id backend/spec string after public provider normalization. */ readonly provider_spec?: string | undefined; - // Delegation: resolve this target as another named target. + // Delegation: resolve this provider definition as another named provider. // Supports ${{ ENV_VAR }} syntax (e.g., use_target: ${{ AGENT_TARGET }}). readonly use_target?: string | unknown | undefined; readonly grader_target?: string | undefined; diff --git a/packages/core/src/evaluation/replay-fixtures.ts b/packages/core/src/evaluation/replay-fixtures.ts index d076f2214..9858fa5de 100644 --- a/packages/core/src/evaluation/replay-fixtures.ts +++ b/packages/core/src/evaluation/replay-fixtures.ts @@ -17,7 +17,7 @@ import path from 'node:path'; import { z } from 'zod'; import { deriveSkillCallsFromMessages, skillCallMetadata } from './providers/skill-calls.js'; -import type { ResolvedTarget } from './providers/targets.js'; +import type { ResolvedProviderBackend } from './providers/targets.js'; import type { Message, ProviderResponse, ProviderTokenUsage, ToolCall } from './providers/types.js'; import type { EvalTest } from './types.js'; @@ -135,7 +135,7 @@ export interface BuildReplayFixtureRecordOptions { readonly evalCase: EvalTest; readonly evalFilePath: string; readonly repoRoot: string; - readonly target: ResolvedTarget; + readonly target: ResolvedProviderBackend; readonly sourceTarget?: string; readonly attempt: number; readonly variant?: string; @@ -496,7 +496,7 @@ function buildFixtureId(input: { } function buildSourceMetadata( - target: ResolvedTarget, + target: ResolvedProviderBackend, sourceTarget: string, ): Record { return dropUndefined({ @@ -507,7 +507,7 @@ function buildSourceMetadata( }); } -function extractModelName(target: ResolvedTarget): string | undefined { +function extractModelName(target: ResolvedProviderBackend): string | undefined { const config = target.config as Record; if (typeof config.model === 'string') { return config.model; diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index bc8755b78..fda3aa4f5 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -351,7 +351,7 @@ export type EvalTargetRef = { /** Delegate to another named target (same as use_target in targets.yaml) */ readonly use_target?: string; /** Inline target definition normalized from a promptfoo-shaped target object. */ - readonly definition?: import('./providers/types.js').TargetDefinition; + readonly definition?: import('./providers/types.js').ProviderDefinition; /** Per-target hooks for workspace customization */ readonly hooks?: TargetHooksConfig; }; diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index ec4c79dd1..f28f60a94 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import { CLI_PLACEHOLDERS, - COMMON_TARGET_SETTINGS, + COMMON_PROVIDER_SETTINGS, findDeprecatedCamelCaseTargetWarnings, normalizeProviderDefinition, } from '../providers/targets.js'; @@ -63,9 +63,9 @@ function validateLegacyEnvTemplates( } } -// Cross-provider settings derived from the schema source of truth in targets.ts. -// Adding a field to COMMON_TARGET_SETTINGS automatically makes it valid here. -const COMMON_SETTINGS = new Set(COMMON_TARGET_SETTINGS); +// Cross-provider settings derived from the provider-definition schema source of truth. +// Adding a field to COMMON_PROVIDER_SETTINGS automatically makes it valid here. +const COMMON_SETTINGS = new Set(COMMON_PROVIDER_SETTINGS); const RETRY_SETTINGS = new Set([ 'max_retries', diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 57c028644..3e6d4730d 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -63,7 +63,7 @@ import { } from './loaders/shorthand-expansion.js'; import { parseTransformSpec } from './loaders/transform-parser.js'; import { parseMetadata } from './metadata.js'; -import type { TargetDefinition } from './providers/types.js'; +import type { ProviderDefinition } from './providers/types.js'; import type { AgentRulesExtensionConfig, AgentRulesPaths, @@ -1301,7 +1301,7 @@ export type EvalSuiteResult = { /** Internal normalized run controls derived from flat eval YAML. */ readonly experimentConfig?: ExperimentConfig; /** Inline target definition from a TS eval config. */ - readonly inlineTarget?: import('./providers/types.js').TargetDefinition; + readonly inlineTarget?: import('./providers/types.js').ProviderDefinition; /** Custom provider factory from a TS eval config task(). */ readonly providerFactory?: import('./providers/provider-registry.js').ProviderFactoryFn; }; @@ -1314,7 +1314,7 @@ export type EvalDefaultTestDefaults = { export type EvalTargetSpec = { readonly name: string; readonly extends?: string; - readonly definition?: TargetDefinition; + readonly definition?: ProviderDefinition; readonly hooks?: TargetHooksConfig; }; diff --git a/packages/core/test/evaluation/conversation-mode.test.ts b/packages/core/test/evaluation/conversation-mode.test.ts index 6cd477995..ee4f09025 100644 --- a/packages/core/test/evaluation/conversation-mode.test.ts +++ b/packages/core/test/evaluation/conversation-mode.test.ts @@ -15,7 +15,7 @@ import os from 'node:os'; import path from 'node:path'; import { runEvalCase } from '../../src/evaluation/orchestrator.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Provider, ProviderRequest, @@ -71,7 +71,7 @@ class ErrorOnFirstProvider implements Provider { } } -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/dependency-scheduling.test.ts b/packages/core/test/evaluation/dependency-scheduling.test.ts index b08f873e2..0bfb8f12f 100644 --- a/packages/core/test/evaluation/dependency-scheduling.test.ts +++ b/packages/core/test/evaluation/dependency-scheduling.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test'; import { runEvaluation } from '../../src/evaluation/orchestrator.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Provider, ProviderRequest, @@ -30,7 +30,7 @@ class FixedProvider implements Provider { } } -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/env-injection.test.ts b/packages/core/test/evaluation/env-injection.test.ts index 65e746ffc..2f7224e80 100644 --- a/packages/core/test/evaluation/env-injection.test.ts +++ b/packages/core/test/evaluation/env-injection.test.ts @@ -9,7 +9,7 @@ import { parseShellExportsEnv, runEnvFromEntries, } from '../../src/evaluation/env-injection.js'; -import { resolveTargetDefinition } from '../../src/evaluation/providers/targets.js'; +import { resolveProviderDefinition } from '../../src/evaluation/providers/targets.js'; describe('parseShellExportsEnv', () => { it('parses export and bare KEY=value lines, stripping quotes', () => { @@ -230,7 +230,7 @@ describe('env injection feeds {{ env.* }} target interpolation', () => { await loadEnvPathFiles(['.env'], tempDir); - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { id: 'oracle', provider: 'openai', @@ -265,7 +265,7 @@ describe('env injection feeds {{ env.* }} target interpolation', () => { { cwd: process.cwd() }, ); - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { id: 'oracle', provider: 'openai', diff --git a/packages/core/test/evaluation/evaluators_variables.test.ts b/packages/core/test/evaluation/evaluators_variables.test.ts index 929f52397..2b345c4eb 100644 --- a/packages/core/test/evaluation/evaluators_variables.test.ts +++ b/packages/core/test/evaluation/evaluators_variables.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test'; import { LlmGrader } from '../../src/evaluation/graders.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Provider, ProviderRequest, @@ -35,7 +35,7 @@ const baseTestCase: EvalTest = { evaluator: 'llm-grader', }; -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/execution-metrics.test.ts b/packages/core/test/evaluation/execution-metrics.test.ts index 6dbc6900e..44fc19058 100644 --- a/packages/core/test/evaluation/execution-metrics.test.ts +++ b/packages/core/test/evaluation/execution-metrics.test.ts @@ -3,7 +3,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { ScriptGrader } from '../../src/evaluation/graders.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import { type TraceComputeResult, type TraceSummary, @@ -253,7 +253,7 @@ describe('Code Grader Metrics Integration', () => { evaluator: 'script', }; - const baseTarget: ResolvedTarget = { + const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/execution-status.test.ts b/packages/core/test/evaluation/execution-status.test.ts index cf6b92b97..5e4e4d6a9 100644 --- a/packages/core/test/evaluation/execution-status.test.ts +++ b/packages/core/test/evaluation/execution-status.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { runEvalCase } from '../../src/evaluation/orchestrator.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Provider, ProviderResponse } from '../../src/evaluation/providers/types.js'; import type { EvalTest } from '../../src/evaluation/types.js'; @@ -53,7 +53,7 @@ const baseTestCase: EvalTest = { evaluator: 'llm-grader', }; -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/extensions.test.ts b/packages/core/test/evaluation/extensions.test.ts index 61909e73b..8b1e15eda 100644 --- a/packages/core/test/evaluation/extensions.test.ts +++ b/packages/core/test/evaluation/extensions.test.ts @@ -6,7 +6,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { runEvaluation } from '../../src/evaluation/orchestrator.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Provider, ProviderRequest, @@ -14,7 +14,7 @@ import type { } from '../../src/evaluation/providers/types.js'; import { loadTestSuite, loadTests } from '../../src/evaluation/yaml-parser.js'; -const target: ResolvedTarget = { +const target: ResolvedProviderBackend = { name: 'mock', kind: 'mock', config: {}, diff --git a/packages/core/test/evaluation/graders.test.ts b/packages/core/test/evaluation/graders.test.ts index bcf659b9a..a82051942 100644 --- a/packages/core/test/evaluation/graders.test.ts +++ b/packages/core/test/evaluation/graders.test.ts @@ -13,7 +13,7 @@ import { TokenUsageGrader, } from '../../src/evaluation/graders.js'; import { assembleLlmGraderPrompt } from '../../src/evaluation/graders/llm-grader-prompt.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Provider, ProviderRequest, @@ -92,7 +92,7 @@ const baseTestCase: EvalTest = { evaluator: 'llm-grader', }; -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/graders/composite-threshold.test.ts b/packages/core/test/evaluation/graders/composite-threshold.test.ts index bbba8942e..84921586f 100644 --- a/packages/core/test/evaluation/graders/composite-threshold.test.ts +++ b/packages/core/test/evaluation/graders/composite-threshold.test.ts @@ -7,7 +7,7 @@ import type { Grader, GraderFactory, } from '../../../src/evaluation/graders/types.js'; -import type { ResolvedTarget } from '../../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../../src/evaluation/providers/targets.js'; import type { EvalTest, GraderConfig } from '../../../src/evaluation/types.js'; const baseTestCase: EvalTest = { @@ -21,7 +21,7 @@ const baseTestCase: EvalTest = { criteria: 'Test outcome', }; -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/graders/execution-metrics.test.ts b/packages/core/test/evaluation/graders/execution-metrics.test.ts index 33a931dc2..8b477c210 100644 --- a/packages/core/test/evaluation/graders/execution-metrics.test.ts +++ b/packages/core/test/evaluation/graders/execution-metrics.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test'; import { ExecutionMetricsGrader } from '../../../src/evaluation/graders/execution-metrics.js'; -import type { ResolvedTarget } from '../../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../../src/evaluation/providers/targets.js'; import type { TokenUsage, TraceSummary } from '../../../src/evaluation/trace.js'; import type { EvalTest, ExecutionMetricsGraderConfig } from '../../../src/evaluation/types.js'; @@ -16,7 +16,7 @@ const baseTestCase: EvalTest = { criteria: 'Test outcome', }; -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/llm-grader-multimodal.test.ts b/packages/core/test/evaluation/llm-grader-multimodal.test.ts index 99e23587a..bcbc13192 100644 --- a/packages/core/test/evaluation/llm-grader-multimodal.test.ts +++ b/packages/core/test/evaluation/llm-grader-multimodal.test.ts @@ -14,7 +14,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Message, ProviderRequest } from '../../src/evaluation/providers/types.js'; import type { EvalTest } from '../../src/evaluation/types.js'; @@ -37,7 +37,7 @@ const baseTestCase: EvalTest = { evaluator: 'llm-grader', }; -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 3a749b579..fe3ef2c86 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -20,7 +20,7 @@ import { runEvaluation, } from '../../src/evaluation/orchestrator.js'; import { ReplayProvider } from '../../src/evaluation/providers/index.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Message, Provider, @@ -148,7 +148,7 @@ const baseTestCase: EvalTest = { evaluator: 'llm-grader', }; -const baseTarget: ResolvedTarget = { +const baseTarget: ResolvedProviderBackend = { kind: 'mock', name: 'mock', config: { response: '{}' }, @@ -432,7 +432,7 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, let liveTargetCalls = 0; let graderRuns = 0; - const replayTarget: ResolvedTarget = { + const replayTarget: ResolvedProviderBackend = { kind: 'replay', name: 'replay_coding_agent', config: { @@ -533,7 +533,7 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, let liveTargetCalls = 0; let graderRuns = 0; - const replayTarget: ResolvedTarget = { + const replayTarget: ResolvedProviderBackend = { kind: 'replay', name: 'replay_coding_agent', config: { @@ -1058,7 +1058,7 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, const previousPath = process.env.PATH; process.env.PATH = `${binDir}${path.delimiter}${previousPath ?? ''}`; - const resolvedTargets: ResolvedTarget[] = []; + const resolvedTargets: ResolvedProviderBackend[] = []; const provider = new CapturingCliProvider('docker-cli', { output: [{ role: 'assistant', content: 'OK' }], }); @@ -2841,7 +2841,7 @@ describe('criteria with assertions runs only declared evaluators (#452)', () => responses: [{ output: [{ role: 'assistant', content: 'hello world' }] }], }); - const targetWithGrader: ResolvedTarget = { + const targetWithGrader: ResolvedProviderBackend = { ...baseTarget, graderTarget: 'grader-target', }; @@ -2867,7 +2867,7 @@ describe('criteria with assertions runs only declared evaluators (#452)', () => responses: [{ output: [{ role: 'assistant', content: 'hello world' }] }], }); - const targetWithGrader: ResolvedTarget = { + const targetWithGrader: ResolvedProviderBackend = { ...baseTarget, graderTarget: 'grader-target', }; @@ -2958,7 +2958,7 @@ describe('criteria with assertions runs only declared evaluators (#452)', () => responses: [{ output: [{ role: 'assistant', content: 'hello world' }] }], }); - const targetWithGrader: ResolvedTarget = { + const targetWithGrader: ResolvedProviderBackend = { ...baseTarget, graderTarget: 'grader-target', }; diff --git a/packages/core/test/evaluation/prepared-workspace.test.ts b/packages/core/test/evaluation/prepared-workspace.test.ts index 37ba97cca..a1008d0b0 100644 --- a/packages/core/test/evaluation/prepared-workspace.test.ts +++ b/packages/core/test/evaluation/prepared-workspace.test.ts @@ -6,7 +6,7 @@ import os from 'node:os'; import path from 'node:path'; import { prepareEvalWorkspace } from '../../src/evaluation/prepared-workspace.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { EvalTest, WorkspaceConfig } from '../../src/evaluation/types.js'; import { WorkspaceSetupError } from '../../src/evaluation/workspace/setup.js'; @@ -76,7 +76,7 @@ function evalCase(workspace: WorkspaceConfig): EvalTest { }; } -const target: ResolvedTarget = { +const target: ResolvedProviderBackend = { kind: 'mock', name: 'mock-target', config: { response: '{}' }, diff --git a/packages/core/test/evaluation/providers/fallback-targets.test.ts b/packages/core/test/evaluation/providers/fallback-targets.test.ts index 0279acbb2..8d6c92f7d 100644 --- a/packages/core/test/evaluation/providers/fallback-targets.test.ts +++ b/packages/core/test/evaluation/providers/fallback-targets.test.ts @@ -8,8 +8,8 @@ import { describe, expect, it } from 'bun:test'; import { runEvalCase } from '../../../src/evaluation/orchestrator.js'; -import { resolveTargetDefinition } from '../../../src/evaluation/providers/targets.js'; -import type { ResolvedTarget } from '../../../src/evaluation/providers/targets.js'; +import { resolveProviderDefinition } from '../../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../../src/evaluation/providers/targets.js'; import type { Provider, ProviderResponse } from '../../../src/evaluation/providers/types.js'; import type { EvalTest } from '../../../src/evaluation/types.js'; @@ -68,14 +68,14 @@ const MOCK_EVAL_CASE: EvalTest = { criteria: 'Answer correctly', }; -const MOCK_TARGET: ResolvedTarget = { +const MOCK_TARGET: ResolvedProviderBackend = { kind: 'mock', name: 'primary', config: { response: 'ok' }, fallbackTargets: ['fallback-a', 'fallback-b'], }; -const MOCK_TARGET_NO_FALLBACK: ResolvedTarget = { +const MOCK_TARGET_NO_FALLBACK: ResolvedProviderBackend = { kind: 'mock', name: 'primary', config: { response: 'ok' }, @@ -122,10 +122,10 @@ const evaluators = { }; // --------------------------------------------------------------------------- -// resolveTargetDefinition tests +// resolveProviderDefinition tests // --------------------------------------------------------------------------- -describe('resolveTargetDefinition - fallback_targets', () => { +describe('resolveProviderDefinition - fallback_targets', () => { const env = { TEST_KEY: 'sk-test-key', TEST_MODEL: 'gpt-4o-mini', @@ -140,7 +140,7 @@ describe('resolveTargetDefinition - fallback_targets', () => { fallback_targets: ['azure-llm', 'gemini-flash'], }; - const resolved = resolveTargetDefinition(definition, env); + const resolved = resolveProviderDefinition(definition, env); expect(resolved.fallbackTargets).toEqual(['azure-llm', 'gemini-flash']); }); @@ -153,7 +153,7 @@ describe('resolveTargetDefinition - fallback_targets', () => { fallbackTargets: ['backup-1'], }; - expect(() => resolveTargetDefinition(definition, env)).toThrow( + expect(() => resolveProviderDefinition(definition, env)).toThrow( /fallbackTargets.*fallback_targets/i, ); }); @@ -166,7 +166,7 @@ describe('resolveTargetDefinition - fallback_targets', () => { model: '{{ env.TEST_MODEL }}', }; - const resolved = resolveTargetDefinition(definition, env); + const resolved = resolveProviderDefinition(definition, env); expect(resolved.fallbackTargets).toBeUndefined(); }); }); diff --git a/packages/core/test/evaluation/providers/pi-runtime.test.ts b/packages/core/test/evaluation/providers/pi-runtime.test.ts index b230344a9..13e3e01a2 100644 --- a/packages/core/test/evaluation/providers/pi-runtime.test.ts +++ b/packages/core/test/evaluation/providers/pi-runtime.test.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { createProvider, - resolveTargetDefinition, + resolveProviderDefinition, } from '../../../src/evaluation/providers/index.js'; import { PiCliProvider } from '../../../src/evaluation/providers/pi-cli.js'; import type { PiProcessRunOptions } from '../../../src/evaluation/providers/pi-process.js'; @@ -270,7 +270,7 @@ describe('Pi coding-agent runtime providers', () => { }); it('resolves pi-cli and pi-rpc command argv without disturbing plain LLM providers', async () => { - const piCli = resolveTargetDefinition( + const piCli = resolveProviderDefinition( { id: 'pi-cli-id', name: 'pi-cli-id', @@ -283,7 +283,7 @@ describe('Pi coding-agent runtime providers', () => { } as never, {}, ); - const piRpc = resolveTargetDefinition( + const piRpc = resolveProviderDefinition( { id: 'pi-rpc-id', name: 'pi-rpc-id', @@ -298,7 +298,7 @@ describe('Pi coding-agent runtime providers', () => { } as never, { OPENAI_BASE_URL: 'http://127.0.0.1:10531/v1', OPENAI_API_KEY: 'local-key' }, ); - const llm = resolveTargetDefinition( + const llm = resolveProviderDefinition( { name: 'mock-llm', provider: 'mock', response: 'still works' }, {}, ); diff --git a/packages/core/test/evaluation/providers/sandbox-runtime.test.ts b/packages/core/test/evaluation/providers/sandbox-runtime.test.ts index d8f0b34c6..2094e889c 100644 --- a/packages/core/test/evaluation/providers/sandbox-runtime.test.ts +++ b/packages/core/test/evaluation/providers/sandbox-runtime.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from 'bun:test'; import { createProvider } from '../../../src/evaluation/providers/index.js'; -import type { ResolvedTarget } from '../../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../../src/evaluation/providers/targets.js'; describe('sandbox target runtime', () => { it('returns deliberate unsupported envelopes for sandbox coding-agent adapters', async () => { - const target: ResolvedTarget = { + const target: ResolvedProviderBackend = { name: 'codex-sandbox', kind: 'codex-cli', runtime: { mode: 'sandbox', image: 'agentv-codex:sha256' }, diff --git a/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts b/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts index e8e6e741a..49a3b0afa 100644 --- a/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts +++ b/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts @@ -1,6 +1,6 @@ import path from 'node:path'; import { describe, expect, it } from 'vitest'; -import { resolveTargetDefinition } from '../../../src/evaluation/providers/targets.js'; +import { resolveProviderDefinition } from '../../../src/evaluation/providers/targets.js'; describe('CLI cwd fallback to eval directory', () => { it('uses eval directory as fallback when cwd env var is empty', () => { @@ -16,7 +16,7 @@ describe('CLI cwd fallback to eval directory', () => { }; const evalFilePath = '/path/to/evals/my-test/test.yaml'; - const resolved = resolveTargetDefinition(definition, env, evalFilePath); + const resolved = resolveProviderDefinition(definition, env, evalFilePath); expect(resolved.kind).toBe('cli'); if (resolved.kind === 'cli') { @@ -35,7 +35,7 @@ describe('CLI cwd fallback to eval directory', () => { const env = {}; // Env var not defined const evalFilePath = '/workspace/evals/tool-trajectory/demo.yaml'; - const resolved = resolveTargetDefinition(definition, env, evalFilePath); + const resolved = resolveProviderDefinition(definition, env, evalFilePath); expect(resolved.kind).toBe('cli'); if (resolved.kind === 'cli') { @@ -56,7 +56,7 @@ describe('CLI cwd fallback to eval directory', () => { }; const evalFilePath = '/path/to/evals/my-test/test.yaml'; - const resolved = resolveTargetDefinition(definition, env, evalFilePath); + const resolved = resolveProviderDefinition(definition, env, evalFilePath); expect(resolved.kind).toBe('cli'); if (resolved.kind === 'cli') { @@ -74,7 +74,7 @@ describe('CLI cwd fallback to eval directory', () => { const env = {}; // No evalFilePath provided - const resolved = resolveTargetDefinition(definition, env); + const resolved = resolveProviderDefinition(definition, env); expect(resolved.kind).toBe('cli'); if (resolved.kind === 'cli') { @@ -92,7 +92,7 @@ describe('CLI cwd fallback to eval directory', () => { const env = {}; const evalFilePath = '/path/to/evals/my-test/test.yaml'; - const resolved = resolveTargetDefinition(definition, env, evalFilePath); + const resolved = resolveProviderDefinition(definition, env, evalFilePath); expect(resolved.kind).toBe('cli'); if (resolved.kind === 'cli') { @@ -112,7 +112,7 @@ describe('CLI cwd fallback to eval directory', () => { const env = {}; const evalFilePath = '/path/to/evals/my-test/test.yaml'; - const resolved = resolveTargetDefinition(definition, env, evalFilePath); + const resolved = resolveProviderDefinition(definition, env, evalFilePath); expect(resolved.kind).toBe('cli'); if (resolved.kind === 'cli') { diff --git a/packages/core/test/evaluation/providers/targets-file.test.ts b/packages/core/test/evaluation/providers/targets-file.test.ts index e03fd3ac2..b954f87a9 100644 --- a/packages/core/test/evaluation/providers/targets-file.test.ts +++ b/packages/core/test/evaluation/providers/targets-file.test.ts @@ -3,9 +3,9 @@ import { mkdir, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import { readTargetDefinitions } from '../../../src/evaluation/providers/targets-file.js'; +import { readProviderDefinitions } from '../../../src/evaluation/providers/targets-file.js'; -describe('readTargetDefinitions', () => { +describe('readProviderDefinitions', () => { let tempDir: string | undefined; afterEach(async () => { @@ -33,7 +33,7 @@ describe('readTargetDefinitions', () => { reasoning_effort: low `); - const definitions = await readTargetDefinitions(filePath); + const definitions = await readProviderDefinitions(filePath); expect(definitions).toEqual([ expect.objectContaining({ @@ -53,7 +53,7 @@ describe('readTargetDefinitions', () => { label: mini `); - const definitions = await readTargetDefinitions(filePath); + const definitions = await readProviderDefinitions(filePath); expect(definitions).toEqual([ expect.objectContaining({ @@ -83,7 +83,7 @@ describe('readTargetDefinitions', () => { - id: openai:codex-desktop `); - const definitions = await readTargetDefinitions(filePath); + const definitions = await readProviderDefinitions(filePath); expect(definitions).toEqual([ expect.objectContaining({ @@ -156,7 +156,7 @@ describe('readTargetDefinitions', () => { - id: exec:./script.sh `); - await expect(readTargetDefinitions(filePath)).rejects.toThrow(/cross-platform commands/); + await expect(readProviderDefinitions(filePath)).rejects.toThrow(/cross-platform commands/); }); it('rejects removed top-level targets', async () => { @@ -165,7 +165,7 @@ describe('readTargetDefinitions', () => { provider: mock `); - await expect(readTargetDefinitions(filePath)).rejects.toThrow(/uses removed 'targets'/); + await expect(readProviderDefinitions(filePath)).rejects.toThrow(/uses removed 'targets'/); }); it('rejects authored name in favor of label', async () => { @@ -174,7 +174,7 @@ describe('readTargetDefinitions', () => { id: mock `); - await expect(readTargetDefinitions(filePath)).rejects.toThrow(/providers\[\]\.label/); + await expect(readProviderDefinitions(filePath)).rejects.toThrow(/providers\[\]\.label/); }); it('rejects authored provider field in favor of id', async () => { @@ -183,6 +183,6 @@ describe('readTargetDefinitions', () => { provider: mock `); - await expect(readTargetDefinitions(filePath)).rejects.toThrow(/providers\[\]\.id/); + await expect(readProviderDefinitions(filePath)).rejects.toThrow(/providers\[\]\.id/); }); }); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index cc3c1c19d..6a3c4c047 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -38,21 +38,21 @@ mock.module('@earendil-works/pi-ai', () => ({ })); const providerModule = await import('../../../src/evaluation/providers/index.js'); -const { resolveDelegatedTargetDefinition, resolveTargetDefinition, createProvider } = +const { resolveDelegatedProviderDefinition, resolveProviderDefinition, createProvider } = providerModule; const { normalizeProviderDefinition } = await import( '../../../src/evaluation/providers/targets.js' ); const { extractLastAssistantContent } = await import('../../../src/evaluation/providers/types.js'); -describe('resolveDelegatedTargetDefinition', () => { +describe('resolveDelegatedProviderDefinition', () => { it('throws a helpful error when an env-backed use_target variable is missing', () => { const definitions = new Map([ ['grader', { name: 'grader', use_target: '{{ env.GRADER_TARGET }}' }], ['azure', { name: 'azure', provider: 'azure' }], ]); - expect(() => resolveDelegatedTargetDefinition('grader', definitions, {})).toThrow( + expect(() => resolveDelegatedProviderDefinition('grader', definitions, {})).toThrow( /GRADER_TARGET is not set/i, ); }); @@ -63,7 +63,7 @@ describe('resolveDelegatedTargetDefinition', () => { ]); expect(() => - resolveDelegatedTargetDefinition('grader', definitions, { + resolveDelegatedProviderDefinition('grader', definitions, { GRADER_TARGET: 'azure', }), ).toThrow(/resolved to "azure".*no target named "azure" exists/i); @@ -75,7 +75,7 @@ describe('resolveDelegatedTargetDefinition', () => { ['azure', { name: 'azure', provider: 'azure' }], ]); - expect(() => resolveDelegatedTargetDefinition('grader', definitions, {})).toThrow( + expect(() => resolveDelegatedProviderDefinition('grader', definitions, {})).toThrow( /removed legacy use_target syntax.*Use {{ env\.GRADER_TARGET }}/, ); }); @@ -87,7 +87,7 @@ describe('resolveDelegatedTargetDefinition', () => { ['azure', { name: 'azure', provider: 'azure' }], ]); - const resolved = resolveDelegatedTargetDefinition('grader', definitions, { + const resolved = resolveDelegatedProviderDefinition('grader', definitions, { GRADER_TARGET: 'llm', }); @@ -95,14 +95,14 @@ describe('resolveDelegatedTargetDefinition', () => { }); }); -describe('resolveTargetDefinition', () => { +describe('resolveProviderDefinition', () => { beforeEach(() => { piCompleteMock.mockClear(); piGetModelMock.mockClear(); }); it('uses authored target id as AgentV target identity', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { id: 'primary-sut', provider: 'mock', @@ -120,7 +120,7 @@ describe('resolveTargetDefinition', () => { }); it('uses clean target id as identity when label and legacy name are absent', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { id: 'sandbox-cli', provider: 'cli', @@ -140,7 +140,7 @@ describe('resolveTargetDefinition', () => { }); it('treats provider as backend kind while id remains target identity', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { id: 'candidate-agent', provider: 'openai', @@ -161,7 +161,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves Promptfoo-style colon provider specs without splitting stable identity', () => { - const openaiTarget = resolveTargetDefinition( + const openaiTarget = resolveProviderDefinition( normalizeProviderDefinition({ id: 'openai:responses:gpt-5.4', label: 'gpt5-responses', @@ -174,7 +174,7 @@ describe('resolveTargetDefinition', () => { expect(openaiTarget.config.model).toBe('gpt-5.4'); expect(openaiTarget.config.apiFormat).toBe('responses'); - const anthropicTarget = resolveTargetDefinition( + const anthropicTarget = resolveProviderDefinition( normalizeProviderDefinition({ id: 'anthropic:messages:claude-sonnet-4-6', config: { api_key: '{{ env.ANTHROPIC_API_KEY }}' }, @@ -185,14 +185,14 @@ describe('resolveTargetDefinition', () => { expect(anthropicTarget.kind).toBe('anthropic'); expect(anthropicTarget.config.model).toBe('claude-sonnet-4-6'); - const execTarget = resolveTargetDefinition( + const execTarget = resolveProviderDefinition( normalizeProviderDefinition({ id: 'exec:node ./provider.js' }), ); expect(execTarget.name).toBe('exec:node ./provider.js'); expect(execTarget.kind).toBe('cli'); expect(execTarget.config.command).toBe('node ./provider.js'); - const codexSdkTarget = resolveTargetDefinition( + const codexSdkTarget = resolveProviderDefinition( normalizeProviderDefinition({ id: 'openai:codex-sdk:gpt-5.4-codex', label: 'codex-sdk', @@ -203,7 +203,7 @@ describe('resolveTargetDefinition', () => { expect(codexSdkTarget.kind).toBe('codex-sdk'); expect(codexSdkTarget.config.model).toBe('gpt-5.4-codex'); - const codexAppServerTarget = resolveTargetDefinition( + const codexAppServerTarget = resolveProviderDefinition( normalizeProviderDefinition({ id: 'openai:codex-desktop:gpt-5.4-codex', label: 'codex-local', @@ -223,7 +223,7 @@ describe('resolveTargetDefinition', () => { } satisfies Record; expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'default', provider: 'azure', @@ -243,7 +243,7 @@ describe('resolveTargetDefinition', () => { AZURE_DEPLOYMENT_NAME: 'gpt-4o', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'default', provider: 'azure', @@ -270,7 +270,7 @@ describe('resolveTargetDefinition', () => { MY_MODEL: 'literal-model', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'test', provider: 'azure', @@ -295,7 +295,7 @@ describe('resolveTargetDefinition', () => { MY_MODEL: 'literal-model', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'no-spaces', provider: 'azure', @@ -318,7 +318,7 @@ describe('resolveTargetDefinition', () => { const env = {} satisfies Record; expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'broken', provider: 'azure', @@ -339,7 +339,7 @@ describe('resolveTargetDefinition', () => { CUSTOM_VERSION: 'api-version=2024-08-01-preview', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'azure-version', provider: 'azure', @@ -367,7 +367,7 @@ describe('resolveTargetDefinition', () => { } satisfies Record; expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'azure-with-api-format', provider: 'azure', @@ -387,7 +387,7 @@ describe('resolveTargetDefinition', () => { } satisfies Record; expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'openai-with-judge-target', provider: 'openai', @@ -402,7 +402,7 @@ describe('resolveTargetDefinition', () => { it('rejects removed copilot-log target provider surface', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'old-copilot-log', provider: 'copilot-log', @@ -415,7 +415,7 @@ describe('resolveTargetDefinition', () => { it('rejects removed log_format target aliases', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'copilot-log-format', provider: 'copilot-cli', @@ -426,7 +426,7 @@ describe('resolveTargetDefinition', () => { ).toThrow(/log_format.*has been removed.*stream_log/i); expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'claude-log-output-format', provider: 'claude-cli', @@ -438,7 +438,7 @@ describe('resolveTargetDefinition', () => { }); it('maps canonical stream_log values to agent logger config', () => { - const raw = resolveTargetDefinition( + const raw = resolveProviderDefinition( { name: 'copilot-raw-log', provider: 'copilot-cli', @@ -453,7 +453,7 @@ describe('resolveTargetDefinition', () => { expect(raw.config.streamLog).toBe('raw'); expect(raw.config.logFormat).toBe('json'); - const summary = resolveTargetDefinition( + const summary = resolveProviderDefinition( { name: 'claude-summary-log', provider: 'claude-cli', @@ -468,7 +468,7 @@ describe('resolveTargetDefinition', () => { expect(summary.config.streamLog).toBe('summary'); expect(summary.config.logFormat).toBe('summary'); - const disabled = resolveTargetDefinition( + const disabled = resolveProviderDefinition( { name: 'pi-disabled-log', provider: 'pi-cli', @@ -491,7 +491,7 @@ describe('resolveTargetDefinition', () => { AZURE_DEPLOYMENT_NAME: 'gpt-4o', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'azure-default-version', provider: 'azure', @@ -516,7 +516,7 @@ describe('resolveTargetDefinition', () => { } satisfies Record; expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'broken', provider: 'azure', @@ -530,7 +530,7 @@ describe('resolveTargetDefinition', () => { }); it('supports vscode configuration with executable/wait/dry_run', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'editor', provider: 'vscode', @@ -556,7 +556,7 @@ describe('resolveTargetDefinition', () => { VSCODE_CMD: '/custom/path/to/code', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'editor', provider: 'vscode', @@ -576,7 +576,7 @@ describe('resolveTargetDefinition', () => { it('resolves vscode executable from literal path', () => { const env = {} satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'editor', provider: 'vscode', @@ -596,7 +596,7 @@ describe('resolveTargetDefinition', () => { it('vscode defaults to code when no executable specified', () => { const env = {} satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'editor', provider: 'vscode', @@ -617,7 +617,7 @@ describe('resolveTargetDefinition', () => { GOOGLE_API_KEY: 'gemini-secret', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'gemini-target', provider: 'gemini', @@ -643,7 +643,7 @@ describe('resolveTargetDefinition', () => { GOOGLE_GEMINI_MODEL: 'gemini-2.5-pro', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'gemini-pro', provider: 'gemini', @@ -669,7 +669,7 @@ describe('resolveTargetDefinition', () => { GOOGLE_API_KEY: 'gemini-secret', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'gemini-flash', provider: 'gemini', @@ -697,7 +697,7 @@ describe('resolveTargetDefinition', () => { OPENAI_MODEL: 'gpt-5.4', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'openai-target', provider: 'openai', @@ -727,7 +727,7 @@ describe('resolveTargetDefinition', () => { OPENAI_MODEL: 'gpt-5.4', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'openai-target', provider: 'openai', @@ -752,7 +752,7 @@ describe('resolveTargetDefinition', () => { it('rejects inline {{ env.* }} templates in secret fields', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'openai-target', provider: 'openai', @@ -771,7 +771,7 @@ describe('resolveTargetDefinition', () => { it('rejects composed {{ env.* }} templates in secret fields', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'openai-target', provider: 'openai', @@ -791,7 +791,7 @@ describe('resolveTargetDefinition', () => { it('rejects literal defaults in secret field env templates', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'openai-target', provider: 'openai', @@ -813,7 +813,7 @@ describe('resolveTargetDefinition', () => { OPENROUTER_MODEL: 'openai/gpt-5-mini', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'openrouter-target', provider: 'openrouter', @@ -836,7 +836,7 @@ describe('resolveTargetDefinition', () => { it('throws when google api key is missing', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'broken-gemini', provider: 'gemini', @@ -848,7 +848,7 @@ describe('resolveTargetDefinition', () => { }); it('honors batch_requests flag in settings', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'batched', provider: 'mock', @@ -863,7 +863,7 @@ describe('resolveTargetDefinition', () => { it('rejects removed provider_batching flag in settings', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'batched', provider: 'mock', @@ -880,7 +880,7 @@ describe('resolveTargetDefinition', () => { CLI_TOKEN: 'secret-token', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'shell-cli', provider: 'cli', @@ -904,7 +904,7 @@ describe('resolveTargetDefinition', () => { }); it('accepts PROMPT_FILE as a supported cli placeholder', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'shell-cli-prompt-file', provider: 'cli', @@ -923,7 +923,7 @@ describe('resolveTargetDefinition', () => { it('throws for unknown cli placeholders', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'bad-cli', provider: 'cli', @@ -940,7 +940,7 @@ describe('resolveTargetDefinition', () => { CODEX_MODEL: 'gpt-4', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'codex-cli', provider: 'codex-cli', @@ -970,7 +970,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves codex-cli reasoning_effort from env', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'codex-cli', provider: 'codex-cli', @@ -994,7 +994,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves codex-cli OpenAI-compatible endpoint settings', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'codex-local-openai', provider: 'codex-cli', @@ -1034,7 +1034,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves codex-sdk as an explicit SDK provider kind', () => { - const target = resolveTargetDefinition({ + const target = resolveProviderDefinition({ name: 'codex-sdk-target', provider: 'codex-sdk', model: 'gpt-5-codex', @@ -1049,7 +1049,7 @@ describe('resolveTargetDefinition', () => { it('rejects unsupported codex reasoning_effort values', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'codex', provider: 'codex-cli', @@ -1063,7 +1063,7 @@ describe('resolveTargetDefinition', () => { it('rejects bare codex provider alias', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'codex', provider: 'codex', @@ -1076,7 +1076,7 @@ describe('resolveTargetDefinition', () => { it('rejects ambiguous Claude and Copilot provider aliases', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'copilot-alias-removed', provider: 'copilot', @@ -1086,7 +1086,7 @@ describe('resolveTargetDefinition', () => { ).toThrow(/ambiguous provider 'copilot'.*copilot-cli.*copilot-sdk/i); expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'claude-alias-removed', provider: 'claude', @@ -1097,7 +1097,7 @@ describe('resolveTargetDefinition', () => { }); it('claude-cli defaults command to claude', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'claude-default', provider: 'claude-cli', @@ -1115,7 +1115,7 @@ describe('resolveTargetDefinition', () => { }); it('claude-cli accepts custom command argv', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'claude-custom', provider: 'claude-cli', @@ -1135,7 +1135,7 @@ describe('resolveTargetDefinition', () => { it('claude-cli rejects removed executable and args fields', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'claude-custom', provider: 'claude-cli', @@ -1147,7 +1147,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves copilot-cli as its own provider kind', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-cli-target', provider: 'copilot-cli', @@ -1169,7 +1169,7 @@ describe('resolveTargetDefinition', () => { }); it('copilot-cli defaults executable to copilot', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-cli-default', provider: 'copilot-cli', @@ -1193,7 +1193,7 @@ describe('resolveTargetDefinition', () => { OPTIONAL_BEARER_TOKEN: 'bearer-secret', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-cli-openai-flat', provider: 'copilot-cli', @@ -1228,7 +1228,7 @@ describe('resolveTargetDefinition', () => { OPENAI_API_KEY: 'openai-secret', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-sdk-openai-flat', provider: 'copilot-sdk', @@ -1261,7 +1261,7 @@ describe('resolveTargetDefinition', () => { WIRE_MODEL: 'gpt-5.3-codex-spark', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-sdk-openai-wire-model', provider: 'copilot-sdk', @@ -1292,7 +1292,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves copilot-sdk args field', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-sdk-with-args', provider: 'copilot-sdk', @@ -1311,7 +1311,7 @@ describe('resolveTargetDefinition', () => { it('copilot flat config rejects literal api_key', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'copilot-literal-key', provider: 'copilot-sdk', @@ -1325,7 +1325,7 @@ describe('resolveTargetDefinition', () => { it('copilot flat config rejects literal bearer_token', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'copilot-literal-bearer', provider: 'copilot-sdk', @@ -1342,7 +1342,7 @@ describe('resolveTargetDefinition', () => { MY_TOKEN: 'bearer-secret', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-sdk-bearer', provider: 'copilot-sdk', @@ -1366,7 +1366,7 @@ describe('resolveTargetDefinition', () => { FOUNDRY_KEY: 'foundry-secret', } satisfies Record; - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-sdk-responses', provider: 'copilot-sdk', @@ -1388,7 +1388,7 @@ describe('resolveTargetDefinition', () => { }); it('copilot-sdk without base_url has no custom provider', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'copilot-sdk-plain', provider: 'copilot-sdk', @@ -1407,7 +1407,7 @@ describe('resolveTargetDefinition', () => { it('rejects camelCase target fields', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'deprecated-camel-case', provider: 'openai', @@ -1426,7 +1426,7 @@ describe('resolveTargetDefinition', () => { }); it('preserves Azure deployment-scoped OpenAI base URLs', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'azure-chat', provider: 'openai', @@ -1452,7 +1452,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves agentv target with model and default temperature', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'agentv-grader', provider: 'agentv', @@ -1471,7 +1471,7 @@ describe('resolveTargetDefinition', () => { }); it('resolves agentv target with explicit temperature', () => { - const target = resolveTargetDefinition( + const target = resolveProviderDefinition( { name: 'agentv-warm', provider: 'agentv', @@ -1492,7 +1492,7 @@ describe('resolveTargetDefinition', () => { it('throws when agentv target is missing model', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'agentv-no-model', provider: 'agentv', @@ -1504,7 +1504,7 @@ describe('resolveTargetDefinition', () => { it('resolves replay targets from execution trace sources', () => { const evalFilePath = '/workspace/evals/sample.eval.yaml'; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'replay-from-execution-trace', provider: 'replay', @@ -1535,7 +1535,7 @@ describe('resolveTargetDefinition', () => { it('rejects replay targets with multiple replay sources', () => { expect(() => - resolveTargetDefinition( + resolveProviderDefinition( { name: 'ambiguous-replay', provider: 'replay', @@ -1563,7 +1563,7 @@ describe('createProvider', () => { OPENAI_MODEL: 'gpt-5.4', } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'openai-target', provider: 'openai', @@ -1590,7 +1590,7 @@ describe('createProvider', () => { OPENAI_API_KEY: 'k', OPENAI_MODEL: 'gpt-5', } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'openai-resp', provider: 'openai', @@ -1613,7 +1613,7 @@ describe('createProvider', () => { OPENROUTER_API_KEY: 'openrouter-key', OPENROUTER_MODEL: 'openai/gpt-5-mini', } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'openrouter-target', provider: 'openrouter', @@ -1636,7 +1636,7 @@ describe('createProvider', () => { ANTHROPIC_API_KEY: 'k', ANTHROPIC_MODEL: 'claude-sonnet-4', } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'anthropic-target', provider: 'anthropic', @@ -1660,7 +1660,7 @@ describe('createProvider', () => { it('routes gemini targets through pi-ai google-generative-ai', async () => { const env = { GOOGLE_API_KEY: 'gemini-key' } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'gemini-target', provider: 'gemini', api_key: '{{ env.GOOGLE_API_KEY }}' }, env, ); @@ -1676,7 +1676,7 @@ describe('createProvider', () => { AZURE_OPENAI_API_KEY: 'key', AZURE_DEPLOYMENT_NAME: 'gpt-4o', } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'azure-target', provider: 'azure', @@ -1703,7 +1703,7 @@ describe('createProvider', () => { AZURE_DEPLOYMENT_NAME: 'gpt-4o', } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-azure', provider: 'pi-coding-agent', @@ -1726,7 +1726,7 @@ describe('createProvider', () => { }); it('resolves pi-coding-agent reasoning_effort level from target config', () => { - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-openai-codex', provider: 'pi-coding-agent', @@ -1744,7 +1744,7 @@ describe('createProvider', () => { }); it('resolves pi-sdk as an explicit SDK provider kind', () => { - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-sdk-agent', provider: 'pi-sdk', @@ -1765,7 +1765,7 @@ describe('createProvider', () => { AZURE_DEPLOYMENT_NAME: 'gpt-4o', } satisfies Record; - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-cli-azure', provider: 'pi-cli', @@ -1786,7 +1786,7 @@ describe('createProvider', () => { }); it('normalizes pi-cli openai base_url targets to azure for PI CLI compatibility', () => { - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-cli-openai-compatible', provider: 'pi-cli', @@ -1806,7 +1806,7 @@ describe('createProvider', () => { }); it('defaults pi-cli base_url targets to azure when subprovider is omitted', () => { - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-cli-local-endpoint', provider: 'pi-cli', @@ -1827,7 +1827,7 @@ describe('createProvider', () => { }); it('keeps pi-cli openai targets on openai when no base_url is configured', () => { - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-cli-openai', provider: 'pi-cli', @@ -1845,7 +1845,7 @@ describe('createProvider', () => { }); it('resolves pi-cli reasoning_effort level from env-backed config', () => { - const resolved = resolveTargetDefinition( + const resolved = resolveProviderDefinition( { name: 'pi-cli-openai-codex', provider: 'pi-cli', diff --git a/packages/core/test/evaluation/tool-trajectory-grader.test.ts b/packages/core/test/evaluation/tool-trajectory-grader.test.ts index f7074f8cd..a4946fff9 100644 --- a/packages/core/test/evaluation/tool-trajectory-grader.test.ts +++ b/packages/core/test/evaluation/tool-trajectory-grader.test.ts @@ -2,14 +2,14 @@ import { describe, expect, it } from 'bun:test'; import { ToolTrajectoryGrader } from '../../src/evaluation/graders.js'; import type { EvaluationContext } from '../../src/evaluation/graders.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Message, Provider } from '../../src/evaluation/providers/types.js'; import type { ToolTrajectoryGraderConfig, TraceSummary } from '../../src/evaluation/trace.js'; import { computeTraceSummary } from '../../src/evaluation/trace.js'; import type { EvalTest } from '../../src/evaluation/types.js'; // Minimal mock objects -const mockTarget: ResolvedTarget = { +const mockTarget: ResolvedProviderBackend = { name: 'mock', kind: 'mock', config: {}, diff --git a/packages/core/test/evaluation/transform-runtime.test.ts b/packages/core/test/evaluation/transform-runtime.test.ts index 20652d1e1..f3b535382 100644 --- a/packages/core/test/evaluation/transform-runtime.test.ts +++ b/packages/core/test/evaluation/transform-runtime.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { runEvalCase, runEvaluation } from '../../src/evaluation/orchestrator.js'; -import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { ResolvedProviderBackend } from '../../src/evaluation/providers/targets.js'; import type { Provider, ProviderRequest, @@ -33,7 +33,7 @@ class StaticProvider implements Provider { } } -const target: ResolvedTarget = { +const target: ResolvedProviderBackend = { name: 'mock', kind: 'mock', config: { response: 'raw' },