diff --git a/Makefile b/Makefile index ec02458c..1fb3897d 100644 --- a/Makefile +++ b/Makefile @@ -444,7 +444,7 @@ python-coverage: ensure-uv ## Run Python tests with coverage enforcement. @$(UV) run coverage report --fail-under="$(PYTHON_COVERAGE_MIN)" @$(UV) run coverage xml -o "$(GO_COVERAGE_DIR)/coverage-python.xml" -check: check-local-artifacts test check-tool-configs check-gemini-prompts check-agent-skills go-test go-e2e-test ## Run the repo's current verification gate. +check: check-local-artifacts test check-tool-configs check-gemini-prompts check-agent-skills check-provider-matrix go-test go-e2e-test ## Run the repo's current verification gate. check-local-artifacts: ## Fail if local build artifacts escaped managed output dirs. @$(call print_step,Checking for unmanaged local build artifacts) @@ -574,7 +574,19 @@ check-agent-skills: ensure-hook-runtime ## Fail if provider skill surfaces are o @"$(GO_TOOLS_BIN_DIR)/coding-ethos-policy" \ check-agent-skills --ethos-root "$(LOCAL_REPO_ROOT)" $(AGENT_SKILL_FLAGS) -build: sync-tool-configs sync-consumer-tool-configs sync-gemini-prompts _sync-agent-skills _sync-consumer-agent-skills go-tools-install repair-repo-ignores _sync-git-hooks _sync-agent-hooks _sync-consumer-agent-hooks managed-toolchain-install go-hook-runner-install policy-bundle-install _sync-parent-hook-runtime ## Build checkout-local hook runtime artifacts. +sync-provider-matrix: ensure-go go-tools-install ## Generate the provider capability matrix. + @$(call print_step,Syncing provider capability matrix) + @$(call print_info,repo: $(LOCAL_REPO_ROOT)) + @"$(GO_TOOLS_BIN_DIR)/coding-ethos-agent-hooks" \ + sync-provider-matrix --root "$(LOCAL_REPO_ROOT)" + +check-provider-matrix: ensure-go go-tools-install ## Fail if the provider capability matrix is out of sync. + @$(call print_step,Checking provider capability matrix) + @$(call print_info,repo: $(LOCAL_REPO_ROOT)) + @"$(GO_TOOLS_BIN_DIR)/coding-ethos-agent-hooks" \ + check-provider-matrix --root "$(LOCAL_REPO_ROOT)" + +build: sync-tool-configs sync-consumer-tool-configs sync-gemini-prompts _sync-agent-skills _sync-consumer-agent-skills sync-provider-matrix go-tools-install repair-repo-ignores _sync-git-hooks _sync-agent-hooks _sync-consumer-agent-hooks managed-toolchain-install go-hook-runner-install policy-bundle-install _sync-parent-hook-runtime ## Build checkout-local hook runtime artifacts. sandbox-runtime-validate: ensure-go go-tools-install ## Validate required sandbox runtime. @$(call print_step,Validating native sandbox runtime) diff --git a/README.md b/README.md index e6ad4564..dd698559 100644 --- a/README.md +++ b/README.md @@ -1555,13 +1555,10 @@ bin/coding-ethos-run agent-hooks verify ``` Agent hook generation is all-or-nothing. `sync` writes every supported -repo-local surface: - -| Provider | Native file | Coverage | -| --- | --- | --- | -| Claude | `.claude/settings.local.json`, `.mcp.json` | full runtime hook set plus MCP stdio server | -| Codex | `.codex/config.toml` | native supported hook events plus MCP stdio server | -| Gemini CLI | `.gemini/settings.json` | native supported hook events plus MCP stdio server | +repo-local surface. Provider support levels, native settings files, hook events, +MCP setup, generated targets, memory behavior, response shapes, and unsupported +surfaces are generated from the registry into +[Provider Capability Matrix](docs/PROVIDER_CAPABILITY_MATRIX.md). Codex runs one native command hook per supported event so current Codex sessions enter the same policy runtime without depending on unstable tool @@ -1664,12 +1661,8 @@ and carries the denial details in the JSON result instead of duplicating a second compact denial line on stderr. Provider output uses the strongest native shape each agent supports: - -| Provider | Block shape | Context/advice shape | -| --- | --- | --- | -| Claude | `hookSpecificOutput.permissionDecision = deny` | full `hookSpecificOutput`, including `updatedInput` | -| Codex | `decision: "block"` plus `permissionDecision: "deny"` for `PreToolUse`; JSON-mode block details on stdout with empty stderr | compact native `additionalContext` for supported lifecycle/post-tool advice; compact `systemMessage` only where Codex exposes no `additionalContext` | -| Gemini | `decision: "deny"` plus `systemMessage` | `additionalContext` on supported lifecycle hooks | +the generated [Provider Capability Matrix](docs/PROVIDER_CAPABILITY_MATRIX.md) +is the source of truth for block response and context/advice shapes. ### Agent-Hook Scope diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index dafb6ac3..ea145f61 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -7,6 +7,11 @@ work: local Git hooks, AI coding assistants, MCP clients, GitHub Actions, GitLab CI, SARIF consumers, and managed static-analysis tools. +The generated [Provider Capability Matrix](PROVIDER_CAPABILITY_MATRIX.md) is the +source of truth for provider ids, supported hook events, block and advice shapes, +MCP setup, generated targets, memory behavior, caveats, and verification +fixtures. + ## Codex Generated Codex surfaces include: diff --git a/docs/PROVIDER_CAPABILITY_MATRIX.md b/docs/PROVIDER_CAPABILITY_MATRIX.md new file mode 100644 index 00000000..f86deb2b --- /dev/null +++ b/docs/PROVIDER_CAPABILITY_MATRIX.md @@ -0,0 +1,257 @@ + + + + +# Provider Capability Matrix + +This report is generated from the provider capability registry. +It lists supported, partially supported, and unsupported adapter surfaces. + +## Coverage Summary + +| Provider | Display name | Coverage | +| --- | --- | --- | +| `claude` | Claude Code | full | +| `codex` | Codex | partial | +| `gemini` | Gemini CLI | partial | +| `generic` | Generic fallback | unsupported | + +## Provider Details + +### Claude Code + +- Provider id: `claude` +- Coverage: full +- Settings target: .claude/settings.local.json +- MCP setup: project .mcp.json stdio server +- Block response shape: hookSpecificOutput.permissionDecision = deny +- Context/advice shape: hookSpecificOutput additionalContext and updatedInput +- Memory interception: provider memory imports into .coding-ethos/memories +- Memory fallback: central memory guidance when writes target managed paths +- Verification: `TestSyncAndVerifySettingsRunsProviderSmokePayloads` + +Native settings: + +- .claude/settings.local.json +- .mcp.json + +Hook events: + +- PreToolUse +- PostToolUse +- PostToolBatch +- PreCompact +- SessionStart +- UserPromptSubmit +- Stop +- SessionEnd +- SubagentStart +- SubagentStop + +Generated targets: + +- CLAUDE.md +- .claude/skills/*/SKILL.md +- .claude/ethos/MEMORY.md +- .mcp.json + +Supported surfaces: + +- PreToolUse block +- PreToolUse updatedInput rewrite +- PostToolUse additionalContext +- PostToolUse edit verification advice +- PostToolBatch additionalContext +- PreCompact capture +- SessionStart additionalContext +- UserPromptSubmit additionalContext +- Stop additionalContext +- SessionEnd additionalContext +- SubagentStart additionalContext +- SubagentStop additionalContext +- MCP stdio server + +Partially supported surfaces: + +- none + +Unsupported surfaces: + +- none + +Safety caveats: + +- none + +### Codex + +- Provider id: `codex` +- Coverage: partial +- Settings target: .codex/config.toml +- MCP setup: .codex/config.toml mcp_servers.coding-ethos stdio server +- Block response shape: decision = block plus permissionDecision = deny for PreToolUse +- Context/advice shape: additionalContext where native; compact systemMessage otherwise +- Memory interception: provider memory imports into .coding-ethos/memories +- Memory fallback: memory.centralized denial points at the central memory file +- Verification: `TestSyncAndVerifySettingsRunsProviderSmokePayloads` + +Native settings: + +- .codex/config.toml + +Hook events: + +- PreToolUse +- PostToolUse +- SessionStart +- UserPromptSubmit +- Stop + +Generated targets: + +- AGENTS.md +- .codex/skills/*/SKILL.md +- .codex/config.toml + +Supported surfaces: + +- PreToolUse block +- PreToolUse native command hook +- PreToolUse apply_patch/edit policy hook +- PostToolUse compact additionalContext +- PostToolUse edit verification advice +- SessionStart additionalContext +- UserPromptSubmit additionalContext +- Stop compact systemMessage +- MCP stdio server + +Partially supported surfaces: + +- lifecycle context is compacted because Codex flattens multiline allowed context + +Unsupported surfaces: + +- PreToolUse updatedInput rewrite +- PostToolBatch additionalContext +- SessionEnd additionalContext +- SubagentStart additionalContext +- SubagentStop additionalContext + +Safety caveats: + +- none + +### Gemini CLI + +- Provider id: `gemini` +- Coverage: partial +- Settings target: .gemini/settings.json +- MCP setup: .gemini/settings.json mcpServers.coding-ethos stdio server +- Block response shape: decision = deny plus systemMessage +- Context/advice shape: additionalContext on supported lifecycle hooks +- Memory interception: provider memory imports into .coding-ethos/memories +- Memory fallback: memory.centralized denial points at the central memory file +- Verification: `TestSyncAndVerifySettingsRunsProviderSmokePayloads` + +Native settings: + +- .gemini/settings.json + +Hook events: + +- BeforeTool +- AfterTool +- BeforeAgent +- AfterAgent +- SessionStart +- SessionEnd + +Generated targets: + +- GEMINI.md +- .gemini/extensions/coding-ethos/gemini-extension.json +- .gemini/extensions/coding-ethos/skills/*/SKILL.md +- .coding-ethos/gemini/prompt-pack.json +- .gemini/settings.json + +Supported surfaces: + +- BeforeTool deny +- BeforeTool systemMessage +- PreToolUse updatedInput rewrite +- AfterTool additionalContext +- AfterTool edit verification advice +- BeforeAgent additionalContext +- AfterAgent additionalContext +- SessionStart additionalContext +- SessionEnd additionalContext +- MCP stdio server + +Partially supported surfaces: + +- BeforeTool maps to PreToolUse for run_shell_command, write_file, replace, and MultiEdit +- AfterTool maps to PostToolUse for run_shell_command, write_file, replace, and MultiEdit + +Unsupported surfaces: + +- PostToolBatch additionalContext +- PreCompact capture +- SubagentStart additionalContext +- SubagentStop additionalContext + +Safety caveats: + +- none + +### Generic fallback + +- Provider id: `generic` +- Coverage: unsupported +- Settings target: none +- MCP setup: manual stdio MCP client configuration +- Block response shape: none; no provider-native hook decision shape +- Context/advice shape: portable Markdown and MCP responses only +- Memory interception: none; providers must write central memory directly +- Memory fallback: read and write .coding-ethos/memories/MEMORY.md +- Verification: `TestProviderCapabilityMatrixSyncAndCheckDetectDrift` + +Native settings: + +- none + +Hook events: + +- none + +Generated targets: + +- AGENTS.md +- ETHOS.md +- .agents/ethos/*.md +- .agents/skills/*/SKILL.md + +Supported surfaces: + +- portable root guidance +- portable ETHOS.md guidance +- portable skill surfaces +- manual MCP stdio server configuration + +Partially supported surfaces: + +- none + +Unsupported surfaces: + +- native hook settings generation +- provider-native block response +- provider-native context injection +- provider-native updatedInput rewrite +- automatic memory write interception + +Safety caveats: + +- generic fallback providers must route policy checks through MCP or explicit CLI commands +- generic fallback providers do not receive automatic provider-native hook enforcement diff --git a/docs/index.md b/docs/index.md index 190d97b2..68b76154 100644 --- a/docs/index.md +++ b/docs/index.md @@ -71,6 +71,9 @@ Code, Gemini CLI, and human contributors need the same enforceable rules. - [Integrations](INTEGRATIONS.md): setup notes for Codex, Claude Code, Gemini CLI, MCP clients, GitHub Actions, GitLab CI, SARIF consumers, and managed tools. +- [Provider capability matrix](PROVIDER_CAPABILITY_MATRIX.md): generated + adapter support, partial-support, unsupported-surface, MCP, memory, and + verification coverage by provider. - [Runtime sandboxing](RUNTIME_SANDBOXING.md): native namespaces, cgroups, seccomp, network isolation, and least-privilege tool capabilities. - [Red-team suite](RED_TEAM_SUITE.md): adversarial coverage for hook bypass, diff --git a/go/internal/agenthooks/provider_capabilities.go b/go/internal/agenthooks/provider_capabilities.go new file mode 100644 index 00000000..2baa9dc2 --- /dev/null +++ b/go/internal/agenthooks/provider_capabilities.go @@ -0,0 +1,418 @@ +// SPDX-FileCopyrightText: 2026 Blackcat Informatics Inc. +// SPDX-License-Identifier: AGPL-3.0-only + +package agenthooks + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + providerCapabilityMatrixDirMode = 0o755 + providerCapabilityMatrixFileMode = 0o644 + + // ProviderCapabilityMatrixRelativePath is the generated provider report path. + ProviderCapabilityMatrixRelativePath = "docs/PROVIDER_CAPABILITY_MATRIX.md" +) + +// ProviderCapabilities returns the provider adapter capability registry. +func ProviderCapabilities() []ProviderCapability { + return []ProviderCapability{ + claudeProviderCapability(), + codexProviderCapability(), + geminiProviderCapability(), + genericProviderCapability(), + } +} + +func claudeProviderCapability() ProviderCapability { + return ProviderCapability{ + Provider: string(ProviderClaude), + DisplayName: "Claude Code", + Coverage: "full", + NativeFiles: []string{".claude/settings.local.json", ".mcp.json"}, + HookEvents: []string{ + "PreToolUse", + "PostToolUse", + "PostToolBatch", + "PreCompact", + "SessionStart", + "UserPromptSubmit", + "Stop", + "SessionEnd", + "SubagentStart", + "SubagentStop", + }, + BlockResponseShape: "hookSpecificOutput.permissionDecision = deny", + ContextAdviceShape: "hookSpecificOutput additionalContext and updatedInput", + MCPSetup: "project .mcp.json stdio server", + SettingsTarget: ".claude/settings.local.json", + GeneratedTargets: []string{ + "CLAUDE.md", + ".claude/skills/*/SKILL.md", + ".claude/ethos/MEMORY.md", + ".mcp.json", + }, + MemoryInterception: "provider memory imports into .coding-ethos/memories", + MemoryFallback: "central memory guidance when writes target managed paths", + Supported: []string{ + "PreToolUse block", + "PreToolUse updatedInput rewrite", + "PostToolUse additionalContext", + "PostToolUse edit verification advice", + "PostToolBatch additionalContext", + "PreCompact capture", + "SessionStart additionalContext", + "UserPromptSubmit additionalContext", + "Stop additionalContext", + "SessionEnd additionalContext", + "SubagentStart additionalContext", + "SubagentStop additionalContext", + "MCP stdio server", + }, + VerificationFixture: "TestSyncAndVerifySettingsRunsProviderSmokePayloads", + } +} + +func codexProviderCapability() ProviderCapability { + return ProviderCapability{ + Provider: string(ProviderCodex), + DisplayName: "Codex", + Coverage: "partial", + NativeFiles: []string{".codex/config.toml"}, + HookEvents: []string{ + "PreToolUse", + "PostToolUse", + "SessionStart", + "UserPromptSubmit", + "Stop", + }, + SettingsTarget: ".codex/config.toml", + BlockResponseShape: "decision = block plus permissionDecision = deny " + + "for PreToolUse", + ContextAdviceShape: "additionalContext where native; compact " + + "systemMessage otherwise", + MCPSetup: ".codex/config.toml mcp_servers.coding-ethos stdio server", + GeneratedTargets: []string{ + "AGENTS.md", + ".codex/skills/*/SKILL.md", + ".codex/config.toml", + }, + MemoryInterception: "provider memory imports into .coding-ethos/memories", + MemoryFallback: "memory.centralized denial points at the central memory file", + Supported: []string{ + "PreToolUse block", + "PreToolUse native command hook", + "PreToolUse apply_patch/edit policy hook", + "PostToolUse compact additionalContext", + "PostToolUse edit verification advice", + "SessionStart additionalContext", + "UserPromptSubmit additionalContext", + "Stop compact systemMessage", + "MCP stdio server", + }, + ProviderLimited: []string{ + "lifecycle context is compacted because Codex flattens multiline allowed context", + }, + Unsupported: []string{ + "PreToolUse updatedInput rewrite", + "PostToolBatch additionalContext", + "SessionEnd additionalContext", + "SubagentStart additionalContext", + "SubagentStop additionalContext", + }, + VerificationFixture: "TestSyncAndVerifySettingsRunsProviderSmokePayloads", + } +} + +func geminiProviderCapability() ProviderCapability { + return ProviderCapability{ + Provider: string(ProviderGemini), + DisplayName: "Gemini CLI", + Coverage: "partial", + NativeFiles: []string{".gemini/settings.json"}, + HookEvents: []string{ + "BeforeTool", + "AfterTool", + "BeforeAgent", + "AfterAgent", + "SessionStart", + "SessionEnd", + }, + SettingsTarget: ".gemini/settings.json", + BlockResponseShape: "decision = deny plus systemMessage", + ContextAdviceShape: "additionalContext on supported lifecycle hooks", + MCPSetup: ".gemini/settings.json mcpServers.coding-ethos stdio server", + GeneratedTargets: []string{ + "GEMINI.md", + ".gemini/extensions/coding-ethos/gemini-extension.json", + ".gemini/extensions/coding-ethos/skills/*/SKILL.md", + ".coding-ethos/gemini/prompt-pack.json", + ".gemini/settings.json", + }, + MemoryInterception: "provider memory imports into .coding-ethos/memories", + MemoryFallback: "memory.centralized denial points at the central memory file", + Supported: []string{ + "BeforeTool deny", + "BeforeTool systemMessage", + "PreToolUse updatedInput rewrite", + "AfterTool additionalContext", + "AfterTool edit verification advice", + "BeforeAgent additionalContext", + "AfterAgent additionalContext", + "SessionStart additionalContext", + "SessionEnd additionalContext", + "MCP stdio server", + }, + ProviderLimited: []string{ + providerLimitedToolMapping("BeforeTool", "PreToolUse"), + providerLimitedToolMapping("AfterTool", "PostToolUse"), + }, + Unsupported: []string{ + "PostToolBatch additionalContext", + "PreCompact capture", + "SubagentStart additionalContext", + "SubagentStop additionalContext", + }, + VerificationFixture: "TestSyncAndVerifySettingsRunsProviderSmokePayloads", + } +} + +func genericProviderCapability() ProviderCapability { + return ProviderCapability{ + Provider: string(ProviderGeneric), + DisplayName: "Generic fallback", + Coverage: "unsupported", + NativeFiles: []string{}, + HookEvents: []string{}, + BlockResponseShape: "none; no provider-native hook decision shape", + ContextAdviceShape: "portable Markdown and MCP responses only", + MCPSetup: "manual stdio MCP client configuration", + SettingsTarget: "none", + GeneratedTargets: []string{ + "AGENTS.md", + "ETHOS.md", + ".agents/ethos/*.md", + ".agents/skills/*/SKILL.md", + }, + MemoryInterception: "none; providers must write central memory directly", + MemoryFallback: "read and write .coding-ethos/memories/MEMORY.md", + Supported: []string{ + "portable root guidance", + "portable ETHOS.md guidance", + "portable skill surfaces", + "manual MCP stdio server configuration", + }, + Unsupported: []string{ + "native hook settings generation", + "provider-native block response", + "provider-native context injection", + "provider-native updatedInput rewrite", + "automatic memory write interception", + }, + SafetyCaveats: []string{ + "generic fallback providers must route policy checks through MCP " + + "or explicit CLI commands", + "generic fallback providers do not receive automatic " + + "provider-native hook enforcement", + }, + VerificationFixture: "TestProviderCapabilityMatrixSyncAndCheckDetectDrift", + } +} + +// ProviderCapabilityMatrixMarkdown renders the generated provider report. +func ProviderCapabilityMatrixMarkdown() string { + var builder strings.Builder + + builder.WriteString("\n") + builder.WriteString("\n\n") + builder.WriteString("\n") + builder.WriteString("# Provider Capability Matrix\n\n") + builder.WriteString( + "This report is generated from the provider capability registry.\n", + ) + builder.WriteString("It lists supported, partially supported, and unsupported " + + "adapter surfaces.\n\n") + builder.WriteString("## Coverage Summary\n\n") + builder.WriteString("| Provider | Display name | Coverage |\n") + builder.WriteString("| --- | --- | --- |\n") + + for _, capability := range ProviderCapabilities() { + appendCapabilitySummaryRow(&builder, capability) + } + + builder.WriteString("\n## Provider Details\n") + + for _, capability := range ProviderCapabilities() { + appendCapabilityDetail(&builder, capability) + } + + return builder.String() +} + +func appendCapabilitySummaryRow( + builder *strings.Builder, + capability ProviderCapability, +) { + builder.WriteString("| `") + builder.WriteString(markdownCell(capability.Provider)) + builder.WriteString("` | ") + builder.WriteString(markdownCell(capability.DisplayName)) + builder.WriteString(" | ") + builder.WriteString(markdownCell(capability.Coverage)) + builder.WriteString(" |\n") +} + +// SyncProviderCapabilityMatrix writes the generated provider report to disk. +func SyncProviderCapabilityMatrix(root string) ([]string, error) { + path := providerCapabilityMatrixPath(root) + + err := os.MkdirAll(filepath.Dir(path), providerCapabilityMatrixDirMode) + if err != nil { + return nil, fmt.Errorf("create provider matrix dir: %w", err) + } + + err = os.WriteFile( + path, + []byte(ProviderCapabilityMatrixMarkdown()), + providerCapabilityMatrixFileMode, + ) + if err != nil { + return nil, fmt.Errorf("write provider capability matrix: %w", err) + } + + return []string{path}, nil +} + +// CheckProviderCapabilityMatrix reports generated provider report drift. +func CheckProviderCapabilityMatrix(root string) ([]string, error) { + path := providerCapabilityMatrixPath(root) + + current, err := os.ReadFile(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("read provider capability matrix: %w", err) + } + + return []string{path}, nil + } + + if string(current) != ProviderCapabilityMatrixMarkdown() { + return []string{path}, nil + } + + return []string{}, nil +} + +func providerCapabilityMatrixPath(root string) string { + return filepath.Join( + root, + filepath.FromSlash(ProviderCapabilityMatrixRelativePath), + ) +} + +func appendCapabilityDetail( + builder *strings.Builder, + capability ProviderCapability, +) { + builder.WriteString("\n### " + capability.DisplayName + "\n\n") + appendKeyValue(builder, "Provider id", "`"+markdownCell(capability.Provider)+"`") + appendKeyValue(builder, "Coverage", markdownCell(capability.Coverage)) + appendKeyValue( + builder, + "Settings target", + markdownCell(capability.SettingsTarget), + ) + appendKeyValue(builder, "MCP setup", markdownCell(capability.MCPSetup)) + appendKeyValue( + builder, + "Block response shape", + markdownCell(capability.BlockResponseShape), + ) + appendKeyValue( + builder, + "Context/advice shape", + markdownCell(capability.ContextAdviceShape), + ) + appendKeyValue( + builder, + "Memory interception", + markdownCell(capability.MemoryInterception), + ) + appendKeyValue( + builder, + "Memory fallback", + markdownCell(capability.MemoryFallback), + ) + appendKeyValue( + builder, + "Verification", + "`"+markdownCell(capability.VerificationFixture)+"`", + ) + + builder.WriteString("\nNative settings:\n") + appendBulletList(builder, capability.NativeFiles) + + builder.WriteString("\nHook events:\n") + appendBulletList(builder, capability.HookEvents) + + builder.WriteString("\nGenerated targets:\n") + appendBulletList(builder, capability.GeneratedTargets) + + builder.WriteString("\nSupported surfaces:\n") + appendBulletList(builder, capability.Supported) + + builder.WriteString("\nPartially supported surfaces:\n") + appendBulletList(builder, capability.ProviderLimited) + + builder.WriteString("\nUnsupported surfaces:\n") + appendBulletList(builder, capability.Unsupported) + + builder.WriteString("\nSafety caveats:\n") + appendBulletList(builder, capability.SafetyCaveats) +} + +func appendKeyValue( + builder *strings.Builder, + label string, + value string, +) { + builder.WriteString("- " + label + ": " + value + "\n") +} + +func appendBulletList(builder *strings.Builder, values []string) { + if len(values) == 0 { + builder.WriteString("\n- none\n") + + return + } + + builder.WriteString("\n") + + for _, value := range values { + builder.WriteString("- " + markdownCell(value) + "\n") + } +} + +func markdownCell(value string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "none" + } + + escaped := strings.ReplaceAll(trimmed, "|", "\\|") + + return escaped +} + +func providerLimitedToolMapping(event, target string) string { + return event + " maps to " + target + " for run_shell_command, " + + "write_file, replace, and MultiEdit" +} diff --git a/go/internal/agenthooks/settings.go b/go/internal/agenthooks/settings.go index 94f19605..e473de24 100644 --- a/go/internal/agenthooks/settings.go +++ b/go/internal/agenthooks/settings.go @@ -90,12 +90,23 @@ type claudeSettings struct { } type ProviderCapability struct { - Provider string `json:"provider"` - Coverage string `json:"coverage"` - NativeFiles []string `json:"native_files"` - Supported []string `json:"supported"` - ProviderLimited []string `json:"provider_limited,omitempty"` - Unsupported []string `json:"unsupported,omitempty"` + Provider string `json:"provider"` + DisplayName string `json:"display_name"` + Coverage string `json:"coverage"` + NativeFiles []string `json:"native_files"` + HookEvents []string `json:"hook_events"` + BlockResponseShape string `json:"block_response_shape"` + ContextAdviceShape string `json:"context_advice_shape"` + MCPSetup string `json:"mcp_setup"` + SettingsTarget string `json:"settings_target"` + GeneratedTargets []string `json:"generated_targets"` + MemoryInterception string `json:"memory_interception"` + MemoryFallback string `json:"memory_fallback"` + Supported []string `json:"supported"` + ProviderLimited []string `json:"provider_limited,omitempty"` + Unsupported []string `json:"unsupported,omitempty"` + VerificationFixture string `json:"verification_fixture"` + SafetyCaveats []string `json:"safety_caveats,omitempty"` } type allSettings struct { @@ -320,17 +331,7 @@ func syncSettingsFile(path string, merge func(map[string]any)) error { return err } - err = os.MkdirAll(filepath.Dir(path), settingsDirMode) - if err != nil { - return fmt.Errorf("create settings directory: %w", err) - } - - err = os.WriteFile(path, []byte(content), settingsFileMode) - if err != nil { - return fmt.Errorf("write settings: %w", err) - } - - return nil + return writeSettingsFile(path, content) } func renderSettingsFileContent( @@ -364,7 +365,11 @@ func syncTextSettingsFile(path string, mutate func(string) string) error { return err } - err = os.MkdirAll(filepath.Dir(path), settingsDirMode) + return writeSettingsFile(path, content) +} + +func writeSettingsFile(path, content string) error { + err := os.MkdirAll(filepath.Dir(path), settingsDirMode) if err != nil { return fmt.Errorf("create settings directory: %w", err) } @@ -989,7 +994,7 @@ func skillSurfacePaths(root, skillID string) []skillSurfacePath { return []skillSurfacePath{ { - provider: "portable", + provider: string(ProviderGeneric), path: filepath.Join(root, ".agents", "skills", entrypoint), }, { @@ -1089,89 +1094,6 @@ func buildAllSettings(hookCommand string) (allSettings, error) { }, nil } -func ProviderCapabilities() []ProviderCapability { - return []ProviderCapability{ - { - Provider: string(ProviderClaude), - Coverage: "full", - NativeFiles: []string{".claude/settings.local.json", ".mcp.json"}, - Supported: []string{ - "PreToolUse block", - "PreToolUse updatedInput rewrite", - "PostToolUse additionalContext", - "PostToolUse edit verification advice", - "PostToolBatch additionalContext", - "PreCompact capture", - "SessionStart additionalContext", - "UserPromptSubmit additionalContext", - "Stop additionalContext", - "SessionEnd additionalContext", - "SubagentStart additionalContext", - "SubagentStop additionalContext", - "MCP stdio server", - }, - }, - { - Provider: string(ProviderCodex), - Coverage: "partial", - NativeFiles: []string{".codex/config.toml"}, - Supported: []string{ - "PreToolUse block", - "PreToolUse native command hook", - "PreToolUse apply_patch/edit policy hook", - "PostToolUse compact additionalContext", - "PostToolUse edit verification advice", - "SessionStart additionalContext", - "UserPromptSubmit additionalContext", - "Stop compact systemMessage", - "MCP stdio server", - }, - ProviderLimited: []string{ - "lifecycle context is compacted because Codex flattens multiline allowed context", - }, - Unsupported: []string{ - "PreToolUse updatedInput rewrite", - "PostToolBatch additionalContext", - "SessionEnd additionalContext", - "SubagentStart additionalContext", - "SubagentStop additionalContext", - }, - }, - { - Provider: string(ProviderGemini), - Coverage: "partial", - NativeFiles: []string{".gemini/settings.json"}, - Supported: []string{ - "BeforeTool deny", - "BeforeTool systemMessage", - "PreToolUse updatedInput rewrite", - "AfterTool additionalContext", - "AfterTool edit verification advice", - "BeforeAgent additionalContext", - "AfterAgent additionalContext", - "SessionStart additionalContext", - "SessionEnd additionalContext", - "MCP stdio server", - }, - ProviderLimited: []string{ - providerLimitedToolMapping("BeforeTool", "PreToolUse"), - providerLimitedToolMapping("AfterTool", "PostToolUse"), - }, - Unsupported: []string{ - "PostToolBatch additionalContext", - "PreCompact capture", - "SubagentStart additionalContext", - "SubagentStop additionalContext", - }, - }, - } -} - -func providerLimitedToolMapping(event, target string) string { - return event + " maps to " + target + " for run_shell_command, " + - "write_file, replace, and MultiEdit" -} - func buildClaudeSettings(specs []HookSpec, hookCommand string) claudeSettings { hooks := make(map[string][]matcherHook) for _, spec := range specs { diff --git a/go/internal/agenthooks/settings_test.go b/go/internal/agenthooks/settings_test.go index d566a3dc..1c9da63d 100644 --- a/go/internal/agenthooks/settings_test.go +++ b/go/internal/agenthooks/settings_test.go @@ -38,8 +38,14 @@ func TestWriteSettingsIncludesAllProviders(t *testing.T) { `"codex": {`, `"gemini": {`, `"capabilities": [`, + `"display_name": "Claude Code"`, + `"provider": "generic"`, `"coverage": "full"`, `"coverage": "partial"`, + `"coverage": "unsupported"`, + `"block_response_shape"`, + `"context_advice_shape"`, + `"verification_fixture"`, `"provider_limited"`, `"unsupported"`, `"PreToolUse"`, @@ -71,7 +77,7 @@ func TestProviderCapabilitiesDocumentProviderLimits(t *testing.T) { t.Parallel() capabilities := agenthooks.ProviderCapabilities() - if len(capabilities) != 3 { + if len(capabilities) != 4 { t.Fatalf("capability count mismatch: %#v", capabilities) } @@ -85,8 +91,24 @@ func TestProviderCapabilitiesDocumentProviderLimits(t *testing.T) { ) } - assertUnsupported(t, capabilities, "codex", "PreToolUse updatedInput rewrite") - assertUnsupported(t, capabilities, "gemini", "PostToolBatch additionalContext") + assertUnsupported( + t, + capabilities, + string(agenthooks.ProviderCodex), + "PreToolUse updatedInput rewrite", + ) + assertUnsupported( + t, + capabilities, + string(agenthooks.ProviderGemini), + "PostToolBatch additionalContext", + ) + assertUnsupported( + t, + capabilities, + string(agenthooks.ProviderGeneric), + "native hook settings generation", + ) } type providerCapabilityExpectation struct { @@ -97,23 +119,102 @@ type providerCapabilityExpectation struct { func providerCapabilityExpectations() []providerCapabilityExpectation { return []providerCapabilityExpectation{ - {"claude", "full", "PreToolUse updatedInput rewrite"}, - {"claude", "full", "UserPromptSubmit additionalContext"}, - {"claude", "full", "MCP stdio server"}, - {"codex", "partial", "PreToolUse native command hook"}, - {"codex", "partial", "PreToolUse apply_patch/edit policy hook"}, - {"codex", "partial", "PostToolUse compact additionalContext"}, - {"codex", "partial", "PostToolUse edit verification advice"}, - {"codex", "partial", "SessionStart additionalContext"}, - {"codex", "partial", "UserPromptSubmit additionalContext"}, - {"codex", "partial", "Stop compact systemMessage"}, - {"codex", "partial", "MCP stdio server"}, - {"gemini", "partial", "BeforeTool deny"}, - {"gemini", "partial", "PreToolUse updatedInput rewrite"}, - {"gemini", "partial", "AfterTool additionalContext"}, - {"gemini", "partial", "BeforeAgent additionalContext"}, - {"gemini", "partial", "SessionEnd additionalContext"}, - {"gemini", "partial", "MCP stdio server"}, + {string(agenthooks.ProviderClaude), "full", "PreToolUse updatedInput rewrite"}, + {string(agenthooks.ProviderClaude), "full", "UserPromptSubmit additionalContext"}, + {string(agenthooks.ProviderClaude), "full", "MCP stdio server"}, + {string(agenthooks.ProviderCodex), "partial", "PreToolUse native command hook"}, + { + string(agenthooks.ProviderCodex), + "partial", + "PreToolUse apply_patch/edit policy hook", + }, + { + string(agenthooks.ProviderCodex), + "partial", + "PostToolUse compact additionalContext", + }, + {string(agenthooks.ProviderCodex), "partial", "PostToolUse edit verification advice"}, + {string(agenthooks.ProviderCodex), "partial", "SessionStart additionalContext"}, + {string(agenthooks.ProviderCodex), "partial", "UserPromptSubmit additionalContext"}, + {string(agenthooks.ProviderCodex), "partial", "Stop compact systemMessage"}, + {string(agenthooks.ProviderCodex), "partial", "MCP stdio server"}, + {string(agenthooks.ProviderGemini), "partial", "BeforeTool deny"}, + {string(agenthooks.ProviderGemini), "partial", "PreToolUse updatedInput rewrite"}, + {string(agenthooks.ProviderGemini), "partial", "AfterTool additionalContext"}, + {string(agenthooks.ProviderGemini), "partial", "BeforeAgent additionalContext"}, + {string(agenthooks.ProviderGemini), "partial", "SessionEnd additionalContext"}, + {string(agenthooks.ProviderGemini), "partial", "MCP stdio server"}, + {string(agenthooks.ProviderGeneric), "unsupported", "portable skill surfaces"}, + } +} + +func TestProviderCapabilityMatrixSyncAndCheckDetectDrift(t *testing.T) { + t.Parallel() + + root := t.TempDir() + + written, err := agenthooks.SyncProviderCapabilityMatrix(root) + if err != nil { + t.Fatalf("sync provider matrix: %v", err) + } + + if len(written) != 1 || + filepath.Base(written[0]) != "PROVIDER_CAPABILITY_MATRIX.md" { + t.Fatalf("written provider matrix paths = %#v", written) + } + + info, err := os.Stat(written[0]) + if err != nil { + t.Fatalf("stat provider matrix: %v", err) + } + if info.Mode().Perm() != 0o644 { + t.Fatalf("provider matrix mode = %s, want -rw-r--r--", info.Mode()) + } + + mismatched, err := agenthooks.CheckProviderCapabilityMatrix(root) + if err != nil { + t.Fatalf("check provider matrix: %v", err) + } + + if len(mismatched) != 0 { + t.Fatalf("provider matrix drift after sync = %#v", mismatched) + } + + err = os.WriteFile(written[0], []byte("drift\n"), 0o600) + if err != nil { + t.Fatalf("write provider matrix drift: %v", err) + } + + mismatched, err = agenthooks.CheckProviderCapabilityMatrix(root) + if err != nil { + t.Fatalf("check provider matrix after drift: %v", err) + } + + if len(mismatched) != 1 || mismatched[0] != written[0] { + t.Fatalf("provider matrix drift = %#v, want %s", mismatched, written[0]) + } +} + +func TestProviderCapabilityMatrixDocsStayInSync(t *testing.T) { + t.Parallel() + + path := filepath.Join( + "..", + "..", + "..", + filepath.FromSlash(agenthooks.ProviderCapabilityMatrixRelativePath), + ) + + current, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read provider matrix doc: %v", err) + } + + expected := agenthooks.ProviderCapabilityMatrixMarkdown() + if string(current) != expected { + t.Fatalf( + "provider matrix doc drifted; run make sync-provider-matrix", + ) } } @@ -251,12 +352,7 @@ func TestGeminiSettingsDoNotClaimUnsupportedPostToolUse(t *testing.T) { output := buffer.String() - geminiIndex := strings.Index(output, `"gemini": {`) - if geminiIndex == -1 { - t.Fatalf("missing gemini settings:\n%s", output) - } - - geminiSettings := output[geminiIndex:] + geminiSettings := providerSettingsSection(t, output, "gemini", "capabilities") if strings.Contains(geminiSettings, `"PostToolUse"`) { t.Fatalf("Gemini must not claim unsupported PostToolUse:\n%s", output) } @@ -488,7 +584,7 @@ func providerSettingsSection( t.Fatalf("missing %s settings:\n%s", provider, output) } - end := strings.Index(output[start:], `"`+nextProvider+`": {`) + end := strings.Index(output[start:], `"`+nextProvider+`":`) if end == -1 { t.Fatalf("missing %s settings after %s:\n%s", nextProvider, provider, output) } @@ -647,7 +743,12 @@ func TestSyncAndVerifySettingsRunsProviderSmokePayloads(t *testing.T) { t.Fatalf("check count = %d, want 15: %#v", len(report.Checks), report.Checks) } + knownProviders := providerIDsByRegistry() for _, check := range report.Checks { + if !knownProviders[check.Provider] { + t.Fatalf("check uses unregistered provider %q: %#v", check.Provider, check) + } + if check.Status != "pass" { t.Fatalf("failed check: %#v", check) } @@ -696,7 +797,7 @@ func TestVerifySettingsRejectsInvalidPortableSkillSurface(t *testing.T) { for _, check := range report.Checks { if check.Event == "skill-surface" && - check.Provider == "portable" && + check.Provider == string(agenthooks.ProviderGeneric) && check.Tool == "managed-toolchain" && check.Status == "fail" && strings.Contains(check.Detail, "missing YAML frontmatter") { @@ -705,7 +806,7 @@ func TestVerifySettingsRejectsInvalidPortableSkillSurface(t *testing.T) { } if !found { - t.Fatalf("missing failed portable skill-surface check: %#v", report.Checks) + t.Fatalf("missing failed generic skill-surface check: %#v", report.Checks) } } @@ -1079,9 +1180,19 @@ func providersWithCapability( return providers } +func providerIDsByRegistry() map[string]bool { + providers := map[string]bool{} + + for _, capability := range agenthooks.ProviderCapabilities() { + providers[capability.Provider] = true + } + + return providers +} + func capabilityProbePayload(provider, cwd string) string { switch provider { - case "claude": + case string(agenthooks.ProviderClaude): return fmt.Sprintf(`{ "provider": "claude", "hook_event_name": "PreToolUse", @@ -1089,7 +1200,7 @@ func capabilityProbePayload(provider, cwd string) string { "tool_name": "Bash", "tool_input": {"command": "git add file.txt"} }`, cwd) - case "codex": + case string(agenthooks.ProviderCodex): return fmt.Sprintf(`{ "provider": "codex", "event": "PreToolUse", @@ -1097,7 +1208,7 @@ func capabilityProbePayload(provider, cwd string) string { "tool": "exec_command", "input": {"command": "git add file.txt"} }`, cwd) - case "gemini": + case string(agenthooks.ProviderGemini): return fmt.Sprintf(`{ "provider": "gemini-cli", "hookEventName": "BeforeTool", diff --git a/go/internal/agenthooks/spec.go b/go/internal/agenthooks/spec.go index f75dfb65..52acab5f 100644 --- a/go/internal/agenthooks/spec.go +++ b/go/internal/agenthooks/spec.go @@ -13,6 +13,8 @@ const ( ProviderCodex Provider = "codex" // ProviderGemini renders a Gemini-owned coding-ethos hook manifest. ProviderGemini Provider = "gemini" + // ProviderGeneric identifies portable agent surfaces with no native hooks. + ProviderGeneric Provider = "generic" ) // HookSpec describes the provider-neutral hook surface the runtime protects. diff --git a/go/internal/agenthookscli/main.go b/go/internal/agenthookscli/main.go index 4826b4da..85b4d3c6 100644 --- a/go/internal/agenthookscli/main.go +++ b/go/internal/agenthookscli/main.go @@ -21,7 +21,12 @@ const ( commandArgsOffset = 2 ) -var errUnknownCommand = apperror.StaticError("unknown agent-hooks command") +var ( + errProviderMatrixDrift = apperror.StaticError( + "provider capability matrix out of sync", + ) + errUnknownCommand = apperror.StaticError("unknown agent-hooks command") +) func runCLI(args []string) int { if len(args) == 0 { @@ -41,6 +46,10 @@ func runCLI(args []string) int { err = doctorSettings(args[1:]) case "verify": err = verifySettings(args[1:]) + case "sync-provider-matrix": + err = syncProviderMatrix(args[1:]) + case "check-provider-matrix": + err = checkProviderMatrix(args[1:]) default: usage() @@ -236,6 +245,48 @@ func verifySettings(args []string) error { return nil } +func syncProviderMatrix(args []string) error { + flags := flag.NewFlagSet("sync-provider-matrix", flag.ContinueOnError) + root := flags.String("root", ".", "Repository root for generated docs") + + err := flags.Parse(args) + if err != nil { + return fmt.Errorf("parse sync-provider-matrix flags: %w", err) + } + + _, err = agenthooks.SyncProviderCapabilityMatrix(*root) + if err != nil { + return fmt.Errorf("sync provider capability matrix: %w", err) + } + + return nil +} + +func checkProviderMatrix(args []string) error { + flags := flag.NewFlagSet("check-provider-matrix", flag.ContinueOnError) + root := flags.String("root", ".", "Repository root for generated docs") + + err := flags.Parse(args) + if err != nil { + return fmt.Errorf("parse check-provider-matrix flags: %w", err) + } + + mismatched, err := agenthooks.CheckProviderCapabilityMatrix(*root) + if err != nil { + return fmt.Errorf("check provider capability matrix: %w", err) + } + + if len(mismatched) != 0 { + return fmt.Errorf( + "%w: %s", + errProviderMatrixDrift, + strings.Join(mismatched, ", "), + ) + } + + return nil +} + func defaultHookCommand(hookCommand string) string { if strings.TrimSpace(hookCommand) != "" { return hookCommand @@ -272,7 +323,8 @@ func usage() { } func usageTo(writer io.Writer) { - const text = "Usage: coding-ethos-agent-hooks " + + const text = "Usage: coding-ethos-agent-hooks " + + " " + "[flags]; sync supports --dry-run --format json|toon" feedback.Emit( diff --git a/go/internal/agenthookscli/main_internal_test.go b/go/internal/agenthookscli/main_internal_test.go index 9a3412be..05d0600a 100644 --- a/go/internal/agenthookscli/main_internal_test.go +++ b/go/internal/agenthookscli/main_internal_test.go @@ -186,6 +186,18 @@ func TestRunCLIDispatchesAgentHookCommands(t *testing.T) { ); code != 1 { t.Fatalf("verify exit code = %d, want 1 for missing executable probe", code) } + + if code := runCLI( + []string{"sync-provider-matrix", "--root", root}, + ); code != 0 { + t.Fatalf("sync-provider-matrix exit code = %d, want 0", code) + } + + if code := runCLI( + []string{"check-provider-matrix", "--root", root}, + ); code != 0 { + t.Fatalf("check-provider-matrix exit code = %d, want 0", code) + } } func writeAgentHooksCLITestFile(t *testing.T, path, content string) { @@ -257,6 +269,10 @@ func TestRunCLIReturnsUsageAndCommandErrors(t *testing.T) { if code := runCLI([]string{"sync", "--root"}); code != 1 { t.Fatalf("invalid flags exit code = %d, want 1", code) } + + if code := runCLI([]string{"check-provider-matrix", "--root"}); code != 1 { + t.Fatalf("invalid provider matrix flags exit code = %d, want 1", code) + } } func TestUsageWritesCommandSummary(t *testing.T) { diff --git a/go/internal/celexpr/helpers.go b/go/internal/celexpr/helpers.go index 0a8054db..2bb8ae12 100644 --- a/go/internal/celexpr/helpers.go +++ b/go/internal/celexpr/helpers.go @@ -496,6 +496,8 @@ func normalizedAdvertisingText(text string) string { ".codex\\", ".claude/", ".claude\\", + ".coding-ethos/gemini/", + ".coding-ethos\\gemini\\", ".gemini/", ".gemini\\", } { diff --git a/go/internal/codeintelcli/main_internal_test.go b/go/internal/codeintelcli/main_internal_test.go index 1d761726..8c0cc9f0 100644 --- a/go/internal/codeintelcli/main_internal_test.go +++ b/go/internal/codeintelcli/main_internal_test.go @@ -494,14 +494,20 @@ func TestGitSignalsCommandRefreshesRealRepository(t *testing.T) { runCodeIntelCLIGit(t, ctx, root, "commit", "-m", "test(repo): add app") dbPath := filepath.Join(root, ".coding-ethos", "code-intel.duckdb") - err := run(ctx, []string{ - "git-signals", - "--root", root, - "--db", dbPath, - "--path", "pkg/app.go", + var runErr error + output := captureStdout(t, func() { + runErr = run(ctx, []string{ + "git-signals", + "--root", root, + "--db", dbPath, + "--path", "pkg/app.go", + }) }) - if err != nil { - t.Fatalf("git-signals command returned error: %v", err) + if runErr != nil { + t.Fatalf("git-signals command returned error: %v", runErr) + } + if !strings.Contains(output, `"kind": "git_signals"`) { + t.Fatalf("git-signals output missing kind:\n%s", output) } } diff --git a/go/internal/evaluators/cel_expression_test.go b/go/internal/evaluators/cel_expression_test.go index 3a852c97..4e80c011 100644 --- a/go/internal/evaluators/cel_expression_test.go +++ b/go/internal/evaluators/cel_expression_test.go @@ -318,6 +318,12 @@ func TestEvaluateCELExpressionAllowsAgentConfigPathReferences(t *testing.T) { `"Update .gemini/extensions/coding-ethos/gemini-extension.json"`, provider: "gemini", }, + { + name: "gemini prompt pack", + command: `gh pr edit 66 --body ` + + `"Update .coding-ethos/gemini/prompt-pack.json"`, + provider: "gemini", + }, } for _, testCase := range testCases { diff --git a/tests/TESTS.md b/tests/TESTS.md index 45a2a88d..a0568483 100644 --- a/tests/TESTS.md +++ b/tests/TESTS.md @@ -64,9 +64,9 @@ For the full current repo gate: make check ``` -`make check` runs tests plus generated tool-config and Gemini prompt-pack drift -checks, including real Go end-to-end workflow tests that prepare the runtime -through `make build`. +`make check` runs tests plus generated tool-config, Gemini prompt-pack, agent +skill, and provider-matrix drift checks, including real Go end-to-end workflow +tests that prepare the runtime through `make build`. When hook runtime changes are involved, also run: