diff --git a/packages/core/src/evaluation/providers/pi-process.ts b/packages/core/src/evaluation/providers/pi-process.ts index 01e99f9c1..dcd3f2b06 100644 --- a/packages/core/src/evaluation/providers/pi-process.ts +++ b/packages/core/src/evaluation/providers/pi-process.ts @@ -15,6 +15,8 @@ export interface PiProcessRunOptions { readonly env: NodeJS.ProcessEnv; readonly signal?: AbortSignal; readonly stdin?: string; + readonly stdinEnd?: 'after_write' | 'manual'; + readonly completeOnStdout?: (stdout: string) => boolean; readonly onStdoutChunk?: (chunk: string) => void; readonly onStderrChunk?: (chunk: string) => void; } @@ -211,9 +213,21 @@ export async function defaultPiProcessRunner( } child.stdout.setEncoding('utf8'); + let stdinEnded = false; + const endStdin = (): void => { + if (stdinEnded || child.stdin.destroyed) { + return; + } + stdinEnded = true; + child.stdin.end(); + }; + child.stdout.on('data', (chunk) => { stdout += chunk; options.onStdoutChunk?.(chunk); + if (options.completeOnStdout?.(stdout)) { + endStdin(); + } }); child.stderr.setEncoding('utf8'); @@ -261,7 +275,12 @@ export async function defaultPiProcessRunner( }); }); - child.stdin.end(options.stdin ?? ''); + if (options.stdin !== undefined) { + child.stdin.write(options.stdin); + } + if (options.stdinEnd !== 'manual') { + endStdin(); + } }); } diff --git a/packages/core/src/evaluation/providers/pi-rpc.ts b/packages/core/src/evaluation/providers/pi-rpc.ts index d9fd6f5bd..7061e3284 100644 --- a/packages/core/src/evaluation/providers/pi-rpc.ts +++ b/packages/core/src/evaluation/providers/pi-rpc.ts @@ -9,7 +9,13 @@ import { classifyPiProcessFailure, defaultPiProcessRunner, piProcessFailureMessage, + splitPiCommand, } from './pi-process.js'; +import { + extractAzureResourceName, + resolveCliProvider, + resolveEnvKeyName, +} from './pi-provider-aliases.js'; import { extractPiTextContent, toFiniteNumber } from './pi-utils.js'; import { normalizeInputFiles } from './preread.js'; import type { PiRpcResolvedConfig } from './targets.js'; @@ -49,7 +55,7 @@ export class PiRpcProvider implements Provider { const startedAt = Date.now(); const cwd = this.resolveCwd(request.cwd); - const command = ensureRpcMode(this.config.command); + const command = this.buildCommand(); const inputFiles = normalizeInputFiles(request.inputFiles); const rpcRequest = buildRpcRequest({ request, @@ -61,9 +67,11 @@ export class PiRpcProvider implements Provider { command, cwd, timeoutMs: this.config.timeoutMs, - env: buildPiRuntimeEnv({ runtime: this.config.runtime, targetName: this.targetName }), + env: this.buildEnv(), signal: request.signal, stdin: `${JSON.stringify(rpcRequest)}\n`, + stdinEnd: 'manual', + completeOnStdout: (stdout) => hasRpcAgentEnd(stdout, rpcRequest.id), }); if (result.timedOut || result.exitCode !== 0 || result.signal || result.spawnErrorCode) { @@ -100,6 +108,18 @@ export class PiRpcProvider implements Provider { }); } + const resultError = rpcResultError(parsed.result); + if (resultError) { + return this.buildRpcTaskFailureResponse({ + result, + command, + cwd, + startedAt, + message: resultError, + events: parsed.events, + }); + } + const output = messagesFromRpcResult(parsed.result); const tokenUsage = tokenUsageFromRpcResult(parsed.result); const endedAt = Date.now(); @@ -150,6 +170,90 @@ export class PiRpcProvider implements Provider { return process.cwd(); } + private buildCommand(): readonly string[] { + const args: string[] = []; + if (this.config.subprovider) { + args.push('--provider', resolveCliProvider(this.config.subprovider)); + } + if (this.config.model) { + args.push('--model', this.config.model); + } + if (this.config.apiKey && this.config.subprovider?.toLowerCase() !== 'azure') { + args.push('--api-key', this.config.apiKey); + } + if (this.config.systemPrompt) { + args.push('--system-prompt', this.config.systemPrompt); + } + + if (!hasModeFlag(this.config.command)) { + args.push('--mode', 'rpc'); + } + args.push('--no-session'); + + if (this.config.tools) { + args.push('--tools', this.config.tools); + } + if (this.config.thinking) { + args.push('--thinking', this.config.thinking); + } + + return splitPiCommand(this.config.command, args); + } + + private buildEnv(): NodeJS.ProcessEnv { + const env = buildPiRuntimeEnv({ + runtime: this.config.runtime, + targetName: this.targetName, + }); + + const provider = this.config.subprovider?.toLowerCase() ?? 'google'; + if (provider === 'azure') { + if (this.config.apiKey) { + env.AZURE_OPENAI_API_KEY = this.config.apiKey; + } + if (this.config.baseUrl) { + if (/^https?:\/\//.test(this.config.baseUrl)) { + env.AZURE_OPENAI_BASE_URL = this.config.baseUrl; + } else { + env.AZURE_OPENAI_RESOURCE_NAME = extractAzureResourceName(this.config.baseUrl); + } + } + } else if (this.config.apiKey) { + const envKey = resolveEnvKeyName(provider); + if (envKey) { + env[envKey] = this.config.apiKey; + } + } + + if (this.config.subprovider) { + const resolvedProvider = resolveCliProvider(this.config.subprovider); + const providerOwnPrefixes: Record = { + openrouter: ['OPENROUTER_'], + anthropic: ['ANTHROPIC_'], + openai: ['OPENAI_'], + 'azure-openai-responses': ['AZURE_OPENAI_'], + google: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'], + gemini: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'], + groq: ['GROQ_'], + xai: ['XAI_'], + }; + const ownPrefixes = providerOwnPrefixes[resolvedProvider] ?? []; + const allOtherPrefixes = Object.entries(providerOwnPrefixes) + .filter(([key]) => key !== resolvedProvider) + .flatMap(([, prefixes]) => prefixes); + for (const key of Object.keys(env)) { + if ( + allOtherPrefixes.some((prefix) => key.startsWith(prefix)) && + !ownPrefixes.some((prefix) => key.startsWith(prefix)) + ) { + delete env[key]; + } + } + } + + return env; + } + private buildProcessErrorResponse(params: { readonly result: PiProcessRunResult; readonly command: readonly string[]; @@ -253,10 +357,10 @@ export class PiRpcProvider implements Provider { } type RpcRequest = { - readonly jsonrpc: '2.0'; readonly id: string; - readonly method: 'run'; - readonly params: Record; + readonly type: 'prompt'; + readonly message: string; + readonly streamingBehavior?: 'followUp'; }; type ParsedRpcOutput = { @@ -270,37 +374,18 @@ function buildRpcRequest(params: { readonly config: PiRpcResolvedConfig; readonly inputFiles?: readonly string[]; }): RpcRequest { - const rpcParams: Record = { - prompt: params.request.question, - system_prompt: params.config.systemPrompt ?? params.request.systemPrompt, - model: params.config.model, - subprovider: params.config.subprovider, - tools: params.config.tools, - thinking: params.config.thinking, - input_files: params.inputFiles, - metadata: params.request.metadata, - }; - for (const key of Object.keys(rpcParams)) { - if (rpcParams[key] === undefined) { - delete rpcParams[key]; - } - } + const prefix = params.request.systemPrompt ? `${params.request.systemPrompt}\n\n` : ''; + const suffix = + params.inputFiles && params.inputFiles.length > 0 + ? `\n\nInput files:\n${params.inputFiles.map((file) => `@${file}`).join('\n')}` + : ''; return { - jsonrpc: '2.0', id: randomUUID(), - method: 'run', - params: rpcParams, + type: 'prompt', + message: `${prefix}${params.request.question}${suffix}`, }; } -function ensureRpcMode(command: readonly string[]): readonly string[] { - const hasMode = command.some( - (arg, index) => - arg === '--mode' || arg.startsWith('--mode=') || command[index - 1] === '--mode', - ); - return hasMode ? command : [...command, '--mode', 'rpc']; -} - function parseRpcOutput(stdout: string, requestId: string): ParsedRpcOutput { const lines = stdout .split(/\r?\n/) @@ -326,6 +411,17 @@ function parseRpcOutput(stdout: string, requestId: string): ParsedRpcOutput { throw new Error(`invalid JSON protocol message: ${formatError(parseError)}`); } + if (message.id === requestId && message.type === 'response') { + if (message.success === false) { + error = rpcErrorMessage(message.error ?? message.message); + } + continue; + } + if (message.type === 'agent_end') { + result = message; + events.push(message); + continue; + } if (message.id === requestId && Object.prototype.hasOwnProperty.call(message, 'result')) { result = message.result; continue; @@ -354,6 +450,32 @@ function parseRpcOutput(stdout: string, requestId: string): ParsedRpcOutput { return { result, events }; } +function hasRpcAgentEnd(stdout: string, requestId: string): boolean { + for (const line of stdout.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + const parsed = JSON.parse(trimmed) as Record; + if (parsed.type === 'agent_end') { + return true; + } + if (parsed.id === requestId && parsed.type === 'response' && parsed.success === false) { + return true; + } + } catch { + // Keep waiting; parseRpcOutput will report malformed lines after exit. + } + } + return false; +} + +function hasModeFlag(command: readonly string[]): boolean { + return command.some( + (arg, index) => + arg === '--mode' || arg.startsWith('--mode=') || command[index - 1] === '--mode', + ); +} + function messagesFromRpcResult(result: unknown): readonly Message[] { if (result && typeof result === 'object') { const record = result as Record; @@ -374,6 +496,30 @@ function messagesFromRpcResult(result: unknown): readonly Message[] { return [{ role: 'assistant', content: JSON.stringify(result) }]; } +function rpcResultError(result: unknown): string | undefined { + if (!result || typeof result !== 'object') { + return undefined; + } + const record = result as Record; + if (!Array.isArray(record.messages)) { + return undefined; + } + const errored = [...record.messages].reverse().find((message) => { + if (!message || typeof message !== 'object') return false; + const msg = message as Record; + return msg.role === 'assistant' && (msg.stopReason === 'error' || msg.stop_reason === 'error'); + }); + if (!errored || typeof errored !== 'object') { + return undefined; + } + const msg = errored as Record; + return ( + stringField(msg, 'errorMessage') ?? + stringField(msg, 'error_message') ?? + 'Pi RPC assistant message ended with stopReason=error' + ); +} + function convertRpcMessage(value: unknown): Message | undefined { if (!value || typeof value !== 'object' || Array.isArray(value)) { return undefined; @@ -403,7 +549,7 @@ function tokenUsageFromRpcResult(result: unknown): ProviderTokenUsage | undefine | Record | undefined; if (!usage || typeof usage !== 'object') { - return undefined; + return tokenUsageFromMessages(record.messages); } const input = toFiniteNumber(usage.input ?? usage.input_tokens ?? usage.inputTokens); const output = toFiniteNumber(usage.output ?? usage.output_tokens ?? usage.outputTokens); @@ -420,6 +566,49 @@ function tokenUsageFromRpcResult(result: unknown): ProviderTokenUsage | undefine }; } +function tokenUsageFromMessages(messages: unknown): ProviderTokenUsage | undefined { + if (!Array.isArray(messages)) { + return undefined; + } + + let totalInput = 0; + let totalOutput = 0; + let totalCached = 0; + let totalReasoning = 0; + let found = false; + + for (const message of messages) { + if (!message || typeof message !== 'object') { + continue; + } + const usage = (message as Record).usage; + if (!usage || typeof usage !== 'object') { + continue; + } + found = true; + const record = usage as Record; + totalInput += toFiniteNumber(record.input) ?? 0; + totalOutput += toFiniteNumber(record.output) ?? 0; + totalCached += + toFiniteNumber(record.cacheRead) ?? + toFiniteNumber(record.cache_read) ?? + toFiniteNumber(record.cached) ?? + 0; + totalReasoning += + toFiniteNumber(record.reasoning) ?? toFiniteNumber(record.reasoning_tokens) ?? 0; + } + + if (!found) { + return undefined; + } + return { + input: totalInput, + output: totalOutput, + ...(totalCached > 0 ? { cached: totalCached } : {}), + ...(totalReasoning > 0 ? { reasoning: totalReasoning } : {}), + }; +} + function stringField(record: Record, key: string): string | undefined { const value = record[key]; return typeof value === 'string' ? value : undefined; @@ -443,6 +632,7 @@ function formatError(error: unknown): string { } export const _internal = { - ensureRpcMode, + hasModeFlag, + hasRpcAgentEnd, parseRpcOutput, }; diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index efda8028c..4573c97f7 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -2246,6 +2246,7 @@ function resolvePiRpcConfig( allowLiteral: true, optionalEnv: true, }); + const piRpcSubprovider = normalizePiCliSubprovider(subprovider, baseUrl); const tools = resolveOptionalString(toolsSource, env, `${target.name} pi-rpc tools`, { allowLiteral: true, optionalEnv: true, @@ -2270,7 +2271,7 @@ function resolvePiRpcConfig( return { command, - subprovider, + subprovider: piRpcSubprovider, model, apiKey, baseUrl, diff --git a/packages/core/test/evaluation/providers/pi-runtime.test.ts b/packages/core/test/evaluation/providers/pi-runtime.test.ts index 51e7598c2..55dcbfdbe 100644 --- a/packages/core/test/evaluation/providers/pi-runtime.test.ts +++ b/packages/core/test/evaluation/providers/pi-runtime.test.ts @@ -79,23 +79,58 @@ describe('Pi coding-agent runtime providers', () => { captured = options; const request = JSON.parse(options.stdin?.trim() ?? '{}') as { id: string }; return { - stdout: `${JSON.stringify({ type: 'event', event: { kind: 'start' } })}\n${JSON.stringify({ - jsonrpc: '2.0', - id: request.id, - result: { - output: [{ role: 'assistant', content: 'rpc ok' }], - token_usage: { input: 2, output: 3 }, + stdout: `${JSON.stringify({ type: 'extension_ui_request', id: 'widget-1' })}\n${JSON.stringify( + { + type: 'response', + id: request.id, + command: 'prompt', + success: true, }, + )}\n${JSON.stringify({ + type: 'agent_end', + messages: [ + { role: 'user', content: [{ type: 'text', text: 'hello rpc' }] }, + { + role: 'assistant', + content: [{ type: 'text', text: 'rpc ok' }], + usage: { input: 2, output: 3 }, + }, + ], })}\n`, stderr: '', exitCode: 0, }; }); - const response = await provider.invoke({ question: 'hello rpc', cwd: '/tmp/workspace' }); + const response = await provider.invoke({ + question: 'hello rpc', + systemPrompt: 'case system', + cwd: '/tmp/workspace', + }); - expect(captured?.command).toEqual(['pi', '--mode', 'rpc']); - expect(captured?.stdin).toContain('"method":"run"'); + expect(captured?.command).toEqual([ + 'pi', + '--provider', + 'azure-openai-responses', + '--model', + 'gpt-5-codex', + '--system-prompt', + 'target system', + '--mode', + 'rpc', + '--no-session', + '--tools', + 'read', + '--thinking', + 'low', + ]); + expect(captured?.env.AZURE_OPENAI_API_KEY).toBe('agentv-local'); + expect(captured?.env.AZURE_OPENAI_BASE_URL).toBe('http://127.0.0.1:10531/v1'); + expect(captured?.stdin).toContain('"type":"prompt"'); + expect(captured?.stdin).toContain('case system\\n\\nhello rpc'); + expect(captured?.stdin).not.toContain('"method":"run"'); + expect(captured?.stdinEnd).toBe('manual'); + expect(captured?.completeOnStdout?.('{"type":"agent_end"}\n')).toBe(true); expect(extractLastAssistantContent(response.output)).toBe('rpc ok'); expect(response.tokenUsage).toEqual({ input: 2, output: 3 }); expect(response.targetExecution?.status).toBe('success'); @@ -116,14 +151,16 @@ describe('Pi coding-agent runtime providers', () => { expect(response.targetExecution?.message).toMatch(/malformed protocol/i); }); - it('maps pi-rpc result errors to target task failures', async () => { + it('maps pi-rpc command response errors to target task failures', async () => { const provider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async (options) => { const request = JSON.parse(options.stdin?.trim() ?? '{}') as { id: string }; return { stdout: `${JSON.stringify({ - jsonrpc: '2.0', + type: 'response', id: request.id, - error: { message: 'task failed' }, + command: 'prompt', + success: false, + error: 'task failed', })}\n`, stderr: '', exitCode: 0, @@ -137,6 +174,38 @@ describe('Pi coding-agent runtime providers', () => { expect(response.targetExecution?.message).toBe('task failed'); }); + it('maps pi-rpc assistant stopReason errors to target task failures', async () => { + const provider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async (options) => { + const request = JSON.parse(options.stdin?.trim() ?? '{}') as { id: string }; + return { + stdout: `${JSON.stringify({ + type: 'response', + id: request.id, + command: 'prompt', + success: true, + })}\n${JSON.stringify({ + type: 'agent_end', + messages: [ + { + role: 'assistant', + content: [], + stopReason: 'error', + errorMessage: 'No API key for provider: openai-codex', + }, + ], + })}\n`, + stderr: '', + exitCode: 0, + }; + }); + + const response = await provider.invoke({ question: 'hello rpc', cwd: '/tmp/workspace' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('target_task_failure'); + expect(response.targetExecution?.message).toBe('No API key for provider: openai-codex'); + }); + it('maps pi-rpc timeout and crash failures to target envelopes', async () => { const timeoutProvider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async () => ({ stdout: 'partial', @@ -180,9 +249,14 @@ describe('Pi coding-agent runtime providers', () => { name: 'pi-rpc-id', provider: 'pi-rpc', runtime: 'host', - config: { command: ['pi'] }, + config: { + command: ['pi'], + subprovider: 'openai', + base_url: '{{ env.OPENAI_BASE_URL }}', + api_key: '{{ env.OPENAI_API_KEY }}', + }, } as never, - {}, + { OPENAI_BASE_URL: 'http://127.0.0.1:10531/v1', OPENAI_API_KEY: 'local-key' }, ); const llm = resolveTargetDefinition( { name: 'mock-llm', provider: 'mock', response: 'still works' }, @@ -197,15 +271,30 @@ describe('Pi coding-agent runtime providers', () => { expect(piRpc.kind).toBe('pi-rpc'); if (piRpc.kind !== 'pi-rpc') throw new Error('expected pi-rpc'); expect(piRpc.config.command).toEqual(['pi']); + expect(piRpc.config.subprovider).toBe('azure'); + expect(piRpc.config.baseUrl).toBe('http://127.0.0.1:10531/v1'); + expect(piRpc.config.apiKey).toBe('local-key'); const provider = createProvider(llm); const response = await provider.invoke({ question: 'hello' }); expect(extractLastAssistantContent(response.output)).toBe('still works'); }); - it('does not append duplicate RPC mode flags', () => { - expect(_internal.ensureRpcMode(['pi', '--mode', 'rpc'])).toEqual(['pi', '--mode', 'rpc']); - expect(_internal.ensureRpcMode(['pi', '--mode=rpc'])).toEqual(['pi', '--mode=rpc']); + it('detects pi-rpc completion from current protocol output', () => { + expect(_internal.hasRpcAgentEnd('{"type":"agent_end"}\n', 'req-1')).toBe(true); + expect( + _internal.hasRpcAgentEnd( + '{"id":"req-1","type":"response","command":"prompt","success":false}\n', + 'req-1', + ), + ).toBe(true); + expect(_internal.hasRpcAgentEnd('{"type":"response","success":true}\n', 'req-1')).toBe(false); + }); + + it('detects existing RPC mode flags in custom pi-rpc commands', () => { + expect(_internal.hasModeFlag(['pi', '--mode', 'rpc'])).toBe(true); + expect(_internal.hasModeFlag(['pi', '--mode=rpc'])).toBe(true); + expect(_internal.hasModeFlag(['pi'])).toBe(false); }); }); @@ -226,7 +315,13 @@ function baseCliConfig(): PiCliResolvedConfig { function baseRpcConfig(): PiRpcResolvedConfig { return { command: ['pi'], + subprovider: 'azure', model: 'gpt-5-codex', + apiKey: 'agentv-local', + baseUrl: 'http://127.0.0.1:10531/v1', + tools: 'read', + thinking: 'low', + systemPrompt: 'target system', runtime: { mode: 'host' }, timeoutMs: 1_000, };