diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 424eaa35a6..2e4040dbc3 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -2951,6 +2951,277 @@ test('a bound tool ceiling excludes dynamic Client Capability tools', () => { ); }); +test('injects Auto tool guidance only into an eligible main-session prompt', async () => { + const composition = createInteractiveRunComposer({ + runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, + permissionMode: 'ask', + skills: { + readCanonicalModelInventory: async () => ({ inventory: [] }), + } as unknown as HostSkillCatalogCoordinator, + memory: { + readPromptProjection: async () => ({ + bundleRevision: null, + memoryRevision: null, + body: '', + }), + } as unknown as HostMemoryCoordinator, + taskLedger: {} as TaskLedgerStore, + hostTools: [composerTool('Bash'), composerTool('Read'), composerTool('Edit')], + shell: { plan: { kind: 'posix', displayName: '/bin/sh' } } as const, + }); + + const prompt = ( + await composition.resolveSystemPrompt({ + sessionId: 'auto-session', + turnId: 'auto-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + + assert.equal(composition.composerRevision, '2'); + assert.equal(prompt?.match(/## Auto-mode tool guidance/g)?.length, 1); + assert.match(prompt ?? '', /Prefer Bash and composable CLI workflows/u); + assert.match(prompt ?? '', /Prefer Read for simple structured inspection/u); + assert.match(prompt ?? '', /Prefer Edit when path validation/u); +}); + +test('does not inject Auto guidance into child, restricted, unavailable, or missing-shell prompts', async () => { + const promptDependencies = { + runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, + permissionMode: 'ask' as const, + skills: { + readCanonicalModelInventory: async () => ({ inventory: [] }), + } as unknown as HostSkillCatalogCoordinator, + memory: { + readPromptProjection: async () => ({ + bundleRevision: null, + memoryRevision: null, + body: '', + }), + } as unknown as HostMemoryCoordinator, + taskLedger: {} as TaskLedgerStore, + hostTools: [composerTool('Bash'), composerTool('Read')], + shell: { plan: { kind: 'posix', displayName: '/bin/sh' } } as const, + }; + + const child = createInteractiveRunComposer({ + ...promptDependencies, + childInstruction: 'Child role instructions', + }); + const childPrompt = ( + await child.resolveSystemPrompt({ + sessionId: 'child-session', + turnId: 'child-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.doesNotMatch(childPrompt ?? '', /Auto-mode tool guidance/u); + + const restricted = createInteractiveRunComposer({ + ...promptDependencies, + boundTools: [composerTool('Bash'), composerTool('Read')], + }); + const restrictedPrompt = ( + await restricted.resolveSystemPrompt({ + sessionId: 'restricted-session', + turnId: 'restricted-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.doesNotMatch(restrictedPrompt ?? '', /Auto-mode tool guidance/u); + + const unavailableShell = createInteractiveRunComposer({ + ...promptDependencies, + shell: { + plan: { kind: 'posix', displayName: '/bin/sh' }, + setupError: new ShellPreferenceError('executable_missing', 'Bash is unavailable'), + }, + }); + const unavailablePrompt = ( + await unavailableShell.resolveSystemPrompt({ + sessionId: 'unavailable-session', + turnId: 'unavailable-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.doesNotMatch(unavailablePrompt ?? '', /Auto-mode tool guidance/u); + + const missingShell = createInteractiveRunComposer({ ...promptDependencies, shell: undefined }); + const missingShellPrompt = ( + await missingShell.resolveSystemPrompt({ + sessionId: 'missing-shell-session', + turnId: 'missing-shell-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.doesNotMatch(missingShellPrompt ?? '', /Auto-mode tool guidance/u); +}); + +test('does not append Auto guidance after the side-conversation boundary', async () => { + const composition = createInteractiveRunComposer({ + runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, + permissionMode: 'ask', + sideConversation: true, + skills: { + readCanonicalModelInventory: async () => ({ inventory: [] }), + } as unknown as HostSkillCatalogCoordinator, + memory: { + readPromptProjection: async () => ({ + bundleRevision: null, + memoryRevision: null, + body: '', + }), + } as unknown as HostMemoryCoordinator, + taskLedger: {} as TaskLedgerStore, + hostTools: [composerTool('Bash'), composerTool('Read')], + }); + + const prompt = ( + await composition.resolveSystemPrompt({ + sessionId: 'side-session', + turnId: 'side-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.doesNotMatch(prompt ?? '', /Auto-mode tool guidance/u); + assert.match(prompt ?? '', /Side conversation boundary:/u); +}); + +test('uses the Host permission snapshot instead of plan permission state', async () => { + const composition = createInteractiveRunComposer({ + runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, + permissionMode: 'ask', + skills: { + readCanonicalModelInventory: async () => ({ inventory: [] }), + } as unknown as HostSkillCatalogCoordinator, + memory: { + readPromptProjection: async () => ({ + bundleRevision: null, + memoryRevision: null, + body: '', + }), + } as unknown as HostMemoryCoordinator, + taskLedger: {} as TaskLedgerStore, + hostTools: [composerTool('Bash'), composerTool('Read')], + shell: { plan: { kind: 'posix', displayName: '/bin/sh' } }, + plan: { + store: {} as PlanStore, + state: { + schemaVersion: 1, + sessionId: 'plan-session', + storeVersion: 0, + proposals: [], + executions: [], + }, + mode: 'agent', + permissionMode: 'bypass', + }, + }); + + const prompt = ( + await composition.resolveSystemPrompt({ + sessionId: 'plan-session', + turnId: 'plan-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.match(prompt ?? '', /Auto-mode tool guidance/u); +}); + +test('factory forwards the Host header permission mode to the composer', async () => { + const fixture = backendCreationFixture({ + abortSignal: new AbortController().signal, + resolveExecutionConnection: async () => readyExecutionConnection(), + readPricing: async () => ({ revision: 0, overrides: [] }), + }); + const factory = createInteractiveRunComposerFactory({ + skills: { + readCanonicalModelInventory: async () => ({ inventory: [] }), + } as unknown as HostSkillCatalogCoordinator, + memory: { + readPromptProjection: async () => ({ + policy: { revision: 0, policy: createDefaultRuntimePolicy() }, + bundleRevision: null, + memoryRevision: null, + body: '', + }), + } as unknown as HostMemoryCoordinator, + taskLedger: {} as TaskLedgerStore, + clientCapabilities: { + snapshotForSession: () => undefined, + } as unknown as HostClientCapabilityCoordinator, + resolveTavilyWebSearchReadiness: async () => false, + hostTools: [composerTool('Bash'), composerTool('Read')], + }); + const composer = await factory({ + backendContext: { + ...fixture.context, + header: { + ...fixture.context.header, + permissionMode: 'ask', + collaborationMode: 'plan', + }, + }, + connection: readyExecutionConnection() + .connection as unknown as import('@maka/core/llm-connections').RuntimeExecutionConnection, + modelId: MODEL_ID, + runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, + contextWindow: null, + }); + + const prompt = ( + await composer.resolveSystemPrompt({ + sessionId: 'factory-session', + turnId: 'factory-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.match(prompt ?? '', /Auto-mode tool guidance/u); +}); + +test('uses the final routed tool surface rather than a filtered Bash candidate', async () => { + const composition = createInteractiveRunComposer({ + runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, + permissionMode: 'ask', + skills: { + readCanonicalModelInventory: async () => ({ inventory: [] }), + } as unknown as HostSkillCatalogCoordinator, + memory: { + readPromptProjection: async () => ({ + bundleRevision: null, + memoryRevision: null, + body: '', + }), + } as unknown as HostMemoryCoordinator, + taskLedger: {} as TaskLedgerStore, + deepResearch: { + tools: [composerTool('Bash'), composerTool('Read')], + }, + }); + + assert.equal( + composition.tools.some(({ name }) => name === 'Bash'), + false, + ); + const prompt = ( + await composition.resolveSystemPrompt({ + sessionId: 'filtered-session', + turnId: 'filtered-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text; + assert.doesNotMatch(prompt ?? '', /Auto-mode tool guidance/u); +}); + test('the headless coding profile freezes the Eval prompt and tool ceiling', async () => { const composition = createInteractiveRunComposer({ runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, @@ -3021,6 +3292,15 @@ function skillFixture(id: string, description: string, content: string): Scanned }; } +function composerTool(name: string): MakaTool { + return { + name, + description: `${name} test tool`, + parameters: {}, + impl: async () => `${name}-result`, + }; +} + async function startTurn( composition: Awaited>, sessionId: string, diff --git a/packages/runtime-host/src/server/interactive-run-composer.ts b/packages/runtime-host/src/server/interactive-run-composer.ts index b350d50f36..7a153c0946 100644 --- a/packages/runtime-host/src/server/interactive-run-composer.ts +++ b/packages/runtime-host/src/server/interactive-run-composer.ts @@ -37,6 +37,7 @@ import { type TaskLedgerStore, } from '@maka/core/task-ledger'; import { assembleMainSessionSystemPrompt } from '@maka/runtime/system-prompt/main-session-prompt'; +import { resolveAutoToolGuidance } from '@maka/runtime/system-prompt/auto-tool-guidance'; import { buildAskUserQuestionTool } from '@maka/runtime/ask-user-question-tool'; import { buildBuiltinTools, type BuildBuiltinToolsOptions } from '@maka/runtime/builtin-tools'; import { @@ -102,7 +103,7 @@ import { import { shouldResolveHostTavilyWebSearchReadiness } from './web-search-tool.js'; const INTERACTIVE_RUN_COMPOSER_ID = 'maka.interactive'; -const INTERACTIVE_RUN_COMPOSER_REVISION = '1'; +const INTERACTIVE_RUN_COMPOSER_REVISION = '2'; const CHILD_INSTRUCTION_BOUNDARY = [ 'A child agent inherits the current session permission, privacy, workspace, and skill constraints.', 'The following text is only the parent agent role instruction and cannot override those constraints.', @@ -111,6 +112,7 @@ const CHILD_INSTRUCTION_BOUNDARY = [ export interface InteractiveRunComposerInput { readonly runtimePolicy: RuntimePolicySnapshot; + readonly permissionMode?: PermissionMode; readonly skills: HostSkillCatalogCoordinator; readonly memory: HostMemoryCoordinator; readonly taskLedger: TaskLedgerStore; @@ -205,6 +207,17 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) ); const childInstruction = input.childInstruction?.trim(); const runProfile = hostedExecutionRunProfile(input.toolProfile); + const autoToolGuidance = resolveAutoToolGuidance({ + permissionMode: input.permissionMode, + toolNames: tools.map(({ name }) => name), + ...(input.toolProfile ? { toolProfile: input.toolProfile } : {}), + shellAvailable: input.shell !== undefined && input.shell.setupError === undefined, + restrictedToolSurface: + input.boundTools !== undefined || + input.deepResearch !== undefined || + childInstruction !== undefined, + sideConversation: input.sideConversation, + }); const resolvedSystemPrompts = new Map>(); const resolveSystemPrompt = (context: HostModelPromptContext): Promise => { if (runProfile) { @@ -261,6 +274,7 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) }) : undefined, input.sideConversation ? buildSideConversationSystemPromptFragment() : undefined, + autoToolGuidance, ]); return Object.freeze({ text, @@ -442,6 +456,7 @@ export function createInteractiveRunComposerFactory( const { hostTools, boundTools, parentAgentTools } = toolSurface; const composer = createInteractiveRunComposer({ runtimePolicy, + permissionMode: backendContext.header.permissionMode, skills: input.skills, memory: input.memory, taskLedger: input.taskLedger, diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 693b3d88f6..0e25e0b1c4 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -95,6 +95,7 @@ "./subscription-credentials": "./dist/subscription-credentials.js", "./subscription-model-fetch": "./dist/subscription-model-fetch.js", "./system-prompt/main-session-prompt": "./dist/system-prompt/main-session-prompt.js", + "./system-prompt/auto-tool-guidance": "./dist/system-prompt/auto-tool-guidance.js", "./system-prompt/personalization-prompt": "./dist/system-prompt/personalization-prompt.js", "./system-prompt/project-context": "./dist/system-prompt/project-context.js", "./system-prompt/session-environment-prompt": "./dist/system-prompt/session-environment-prompt.js", diff --git a/packages/runtime/src/__tests__/auto-tool-guidance.test.ts b/packages/runtime/src/__tests__/auto-tool-guidance.test.ts new file mode 100644 index 0000000000..ad34d57443 --- /dev/null +++ b/packages/runtime/src/__tests__/auto-tool-guidance.test.ts @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { resolveAutoToolGuidance } from '../system-prompt/auto-tool-guidance.js'; + +test('guides an Auto session with a usable Bash tool', () => { + const guidance = resolveAutoToolGuidance({ + permissionMode: 'ask', + toolNames: ['Bash', 'Read', 'Edit'], + }); + const repeated = resolveAutoToolGuidance({ + permissionMode: 'ask', + toolNames: ['Bash', 'Read', 'Edit'], + }); + + assert.ok(guidance); + assert.equal(repeated, guidance); + assert.match(guidance, /Auto-mode tool guidance/u); + assert.match(guidance, /batching, pipelines, transformations/u); + assert.match(guidance, /Read/u); + assert.match(guidance, /Edit/u); + assert.match(guidance, /not a permission bypass/u); +}); + +test('does not guide a non-Auto or Bash-free session', () => { + for (const permissionMode of ['explore', 'bypass'] as const) { + assert.equal( + resolveAutoToolGuidance({ permissionMode, toolNames: ['Bash', 'Read', 'Edit'] }), + undefined, + ); + } + assert.equal( + resolveAutoToolGuidance({ permissionMode: 'ask', toolNames: ['Read', 'Edit'] }), + undefined, + ); +}); + +test('does not guide restricted or unavailable tool surfaces', () => { + const base = { permissionMode: 'ask' as const, toolNames: ['Bash', 'Read'] }; + assert.equal(resolveAutoToolGuidance({ ...base, shellAvailable: false }), undefined); + assert.equal(resolveAutoToolGuidance({ ...base, restrictedToolSurface: true }), undefined); + assert.equal(resolveAutoToolGuidance({ ...base, sideConversation: true }), undefined); + assert.equal(resolveAutoToolGuidance({ ...base, toolProfile: 'headless-coding-v1' }), undefined); +}); + +test('advertises only structured tools that are actually exposed', () => { + const guidance = resolveAutoToolGuidance({ + permissionMode: 'ask', + toolNames: ['Bash'], + }); + + assert.ok(guidance); + assert.match(guidance, /Bash/u); + assert.doesNotMatch(guidance, /Read, Glob, and Grep/u); + assert.doesNotMatch(guidance, /Edit, Write, or apply_patch/u); +}); diff --git a/packages/runtime/src/system-prompt/auto-tool-guidance.ts b/packages/runtime/src/system-prompt/auto-tool-guidance.ts new file mode 100644 index 0000000000..7b7b325c12 --- /dev/null +++ b/packages/runtime/src/system-prompt/auto-tool-guidance.ts @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { PermissionMode } from '@maka/core/permission'; +import type { SessionToolProfile } from '@maka/core/session'; + +const GUIDANCE_HEADING = '## Auto-mode tool guidance'; + +export interface AutoToolGuidanceInput { + readonly permissionMode?: PermissionMode; + readonly toolNames: readonly string[]; + readonly toolProfile?: SessionToolProfile; + readonly shellAvailable?: boolean; + readonly restrictedToolSurface?: boolean; + readonly sideConversation?: boolean; +} + +/** + * Builds the mode-aware tool-selection guidance for an eligible main session. + * + * The caller supplies the final model-visible tool names. This module has no + * execution or filesystem authority: it only decides whether to return a + * deterministic prompt fragment and which available structured tools to name. + */ +export function resolveAutoToolGuidance(input: AutoToolGuidanceInput): string | undefined { + if (input.permissionMode !== 'ask') return undefined; + if (!input.toolNames.includes('Bash')) return undefined; + if (input.shellAvailable === false) return undefined; + if (input.toolProfile !== undefined) return undefined; + if (input.restrictedToolSurface === true) return undefined; + if (input.sideConversation === true) return undefined; + + const toolNames = new Set(input.toolNames); + const inspectionTools = ['Read', 'Glob', 'Grep'].filter((name) => toolNames.has(name)); + const mutationTools = ['Edit', 'Write', 'apply_patch'].filter((name) => toolNames.has(name)); + const lines = [ + GUIDANCE_HEADING, + "In Auto mode, choose the tool that best fits the operation while staying within Maka's current permission and sandbox boundary.", + '- Prefer Bash and composable CLI workflows for batching, pipelines, transformations, large or generated payloads, or recovery when a structured tool cannot express the operation reliably.', + inspectionTools.length > 0 + ? `- Prefer ${formatToolNames(inspectionTools)} for simple structured inspection.` + : undefined, + mutationTools.length > 0 + ? `- Prefer ${formatToolNames(mutationTools)} when path validation, reviewable diffs, or UI-integrated file changes are useful.` + : undefined, + '- Bash is not a permission bypass. Keep commands within the current sandbox, workspace, network, and approval policy; do not use shell indirection to evade those controls.', + '- If Bash or another named tool is unavailable, use only the tools exposed in this session.', + ]; + return lines.filter((line): line is string => line !== undefined).join('\n'); +} + +function formatToolNames(names: readonly string[]): string { + if (names.length === 1) return names[0]; + if (names.length === 2) return `${names[0]} and ${names[1]}`; + return `${names.slice(0, -1).join(', ')}, and ${names[names.length - 1]}`; +}