diff --git a/.gitignore b/.gitignore index 8d94b1d..7dacd11 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,3 @@ desktop.ini # Local overrides *.local.md *.local.json - -# VS Code (keep settings.json and extensions.json, ignore personal) -.vscode/launch.json -.vscode/*.code-workspace diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cb074e..a4d79f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ single long command loses its later phases after context compaction in exactly the long sessions a nine-phase workflow produces. Tests that read Phase 5 content now point at `phases/phase-5-audit.md`. +- **`/deepgrade:troubleshoot` and `/deepgrade:codex-challenge` are skills**, on + the same router-plus-phase-files layout as `plan`. troubleshoot (854 lines) + becomes a 256-line router with the incident pre-flow, four phases, + multi-agent mode, and knowledge-base write-back in one file each; + codex-challenge (533 lines) becomes a 213-line router with the output + schema, prompt template, round loop, and report split out. The parser + tests now bind to `skills/codex-challenge/phases/output-schema.md`. The + three remaining long commands (quick-cleanup, readiness-generate, + plan-export) are under the documented 500-line guidance and stay as + commands. Closes backlog B07. +- The documentation skill's bundled templates live in `references/`, the + documented convention, instead of `resources/`. Closes backlog B28. - The `(deepgrade)` description prefix is stripped from all 23 commands and skills. After the monorepo split it mislabelled 11 files owned by deepgrade-audit and deepgrade-readiness. diff --git a/interop.md b/interop.md index 14702fb..c4f7161 100644 --- a/interop.md +++ b/interop.md @@ -24,12 +24,12 @@ Format rules (the sweep depends on them): | Artifact | Producer | Consumers | | -------- | -------- | --------- | | docs/audit/readability/readability-score.json | plugins/deepgrade-readiness/commands/readiness-scan.md | plugins/deepgrade-audit/agents/gate-generator.md, plugins/deepgrade-audit/agents/delta-scanner.md, plugins/deepgrade-audit/commands/codebase-delta.md | -| docs/audit/readability/readability-report.md | plugins/deepgrade-readiness/agents/readiness-report-generator.md | plugins/deepgrade-audit/commands/codebase-audit.md, plugins/deepgrade-audit/agents/delta-scanner.md, plugins/deepgrade/skills/documentation/resources/spec-template.md | +| docs/audit/readability/readability-report.md | plugins/deepgrade-readiness/agents/readiness-report-generator.md | plugins/deepgrade-audit/commands/codebase-audit.md, plugins/deepgrade-audit/agents/delta-scanner.md, plugins/deepgrade/skills/documentation/references/spec-template.md | | docs/audit/deepgrade-report.md | plugins/deepgrade-audit/agents/deepgrade-report-generator.md | plugins/deepgrade/scripts/dg-session-start.js | -| docs/audit/risk-assessment.md | plugins/deepgrade-audit/agents/risk-assessor.md | plugins/deepgrade/agents/plan-scaffolder.md, plugins/deepgrade/agents/plan-auditor.md, plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/documentation/SKILL.md, plugins/deepgrade/skills/documentation/resources/spec-template.md | -| docs/audit/dependency-map.md | plugins/deepgrade-audit/agents/dependency-mapper.md | plugins/deepgrade/agents/plan-scaffolder.md, plugins/deepgrade/agents/plan-auditor.md, plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/plan/phases/phase-7-impact-review.md, plugins/deepgrade/skills/documentation/resources/spec-template.md | -| docs/audit/feature-inventory.md | plugins/deepgrade-audit/agents/feature-scanner.md | plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/documentation/SKILL.md, plugins/deepgrade/skills/documentation/resources/spec-template.md | -| docs/audit/integration-scan.md | plugins/deepgrade-audit/agents/integration-scanner.md | plugins/deepgrade/agents/plan-auditor.md, plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/plan/phases/phase-7-impact-review.md, plugins/deepgrade/skills/documentation/resources/spec-template.md | +| docs/audit/risk-assessment.md | plugins/deepgrade-audit/agents/risk-assessor.md | plugins/deepgrade/agents/plan-scaffolder.md, plugins/deepgrade/agents/plan-auditor.md, plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/documentation/SKILL.md, plugins/deepgrade/skills/documentation/references/spec-template.md | +| docs/audit/dependency-map.md | plugins/deepgrade-audit/agents/dependency-mapper.md | plugins/deepgrade/agents/plan-scaffolder.md, plugins/deepgrade/agents/plan-auditor.md, plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/plan/phases/phase-7-impact-review.md, plugins/deepgrade/skills/documentation/references/spec-template.md | +| docs/audit/feature-inventory.md | plugins/deepgrade-audit/agents/feature-scanner.md | plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/documentation/SKILL.md, plugins/deepgrade/skills/documentation/references/spec-template.md | +| docs/audit/integration-scan.md | plugins/deepgrade-audit/agents/integration-scanner.md | plugins/deepgrade/agents/plan-auditor.md, plugins/deepgrade/commands/quick-plan.md, plugins/deepgrade/skills/plan/phases/phase-7-impact-review.md, plugins/deepgrade/skills/documentation/references/spec-template.md | | docs/audit/audit-progress.md | plugins/deepgrade-audit/commands/codebase-audit.md | plugins/deepgrade-readiness/agents/baseline-scanner.md | ## The readability-score.json schema diff --git a/plugins/deepgrade/GUIDE.md b/plugins/deepgrade/GUIDE.md index cc3470a..103adae 100644 --- a/plugins/deepgrade/GUIDE.md +++ b/plugins/deepgrade/GUIDE.md @@ -2,7 +2,7 @@ # DeepGrade Knowledge Guide v7.1.0 -**8 Commands**  •  **2 Agents**  •  **4 Skills**  •  **3 Safety Hooks**  •  **Requires Node.js 18+** +**6 Commands**  •  **2 Agents**  •  **6 Skills**  •  **3 Safety Hooks**  •  **Requires Node.js 18+** [![Plugin](https://img.shields.io/badge/Claude_Code-Plugin-5A45FF?style=for-the-badge)](https://github.com/krwhynot/deepgrade) [![Version](https://img.shields.io/badge/v7.1.0-stable-2ECC71?style=for-the-badge)](#) @@ -161,12 +161,16 @@ methodology reference lives in the monorepo's | ![p](https://img.shields.io/badge/-%E2%80%8B-9B59B6) | plan-scaffolder | Creates structured technical plans from vague objectives using 3 parallel analysts | quick-plan, plan | | ![p](https://img.shields.io/badge/-%E2%80%8B-9B59B6) | plan-auditor | Scores plans across 8 dimensions using parallel specialist subagents | quick-audit, plan | -## The 4 Skills +## The 6 Skills Skills are persistent knowledge that loads automatically when relevant — reference books the plugin carries in its back pocket. **plan** -- The `/deepgrade:plan` workflow itself. `SKILL.md` is a router (identity, lifecycle, workspace layout, Step 0 intent detection) and each of the nine phases lives in its own file under `phases/`, read when the phase is entered. Split this way so the later phases survive context compaction in long planning sessions instead of being silently dropped with the rest of a 1,700-line command. +**troubleshoot** -- The `/deepgrade:troubleshoot` workflow: a router holding identity, timeline logging, plan detection, and Step 0 knowledge-base lookup, with the incident pre-flow, the four phases, multi-agent mode, and the knowledge-base write-back in one file each under `phases/`, read on entry. + +**codex-challenge** -- The `/deepgrade:codex-challenge` loop: a router holding the Codex invocation pattern and the eight review dimensions, with the output schema, prompt template, round loop, and report in `phases/`. The schema file is what the parser tests bind to. + **documentation** -- The dispatch hub for document generation. Contains routing logic (first word = subcommand), 6 template references, smart suggestions when audit data exists, and document chain enforcement (a PRD triggers a check for a related BRD, etc.). Loads when you ask for a document, or invoke `/deepgrade:documentation` directly. diff --git a/plugins/deepgrade/README.md b/plugins/deepgrade/README.md index d865c0c..7c3674d 100644 --- a/plugins/deepgrade/README.md +++ b/plugins/deepgrade/README.md @@ -24,7 +24,7 @@ Verify inside a Claude Code session: /deepgrade:help ``` -## Commands (8) +## Commands (6) ### Planning @@ -51,7 +51,7 @@ Verify inside a Claude Code session: | `/deepgrade:troubleshoot` | 4-phase debugging framework with incident triage and containment | | `/deepgrade:help` | Show all commands and usage | -`/deepgrade:plan` and `/deepgrade:documentation` are skill surfaces; the other entries are command files. +`/deepgrade:plan`, `/deepgrade:troubleshoot`, `/deepgrade:codex-challenge`, and `/deepgrade:documentation` are skill surfaces; the other entries are command files. ## Safety Hooks (3) @@ -92,7 +92,7 @@ present and wrong. ## Architecture - **2 agents** - plan-auditor (the isolated judge) and plan-scaffolder -- **4 skills** - plan (router plus one file per phase), documentation, MCP research, self-audit knowledge +- **6 skills** - plan, troubleshoot, and codex-challenge (each a router plus one file per phase), documentation, MCP research, self-audit knowledge - **6 doc templates** - ADR, BRD, PRD, README, release notes, spec - **3 hook handlers** - `scripts/dg-*.js` plan-context layer, plus the `dg-canary.js` / `dg-evidence-validate.js` audit tooling invoked by `/deepgrade:plan` diff --git a/plugins/deepgrade/commands/codex-challenge.md b/plugins/deepgrade/commands/codex-challenge.md deleted file mode 100644 index debe4a3..0000000 --- a/plugins/deepgrade/commands/codex-challenge.md +++ /dev/null @@ -1,533 +0,0 @@ ---- -description: Adversarial review loop between Claude and OpenAI Codex CLI. Codex scores your plan across 8 dimensions (max 40), Claude optimizes until score reaches 36/40 GREEN. Implements the Evaluator-Optimizer pattern with score-driven convergence. Pass a plan name, file path, or leave empty for auto-detect. -argument-hint: "[plan-name or file-path] [--rounds N] [--model gpt-5.3-codex]" -allowed-tools: Read, Write, Grep, Glob, Bash, Task ---- - - -You orchestrate an adversarial review loop between Claude Code (you) and OpenAI -Codex CLI. You are the Optimizer; Codex is the Evaluator. Each round, you send -the plan to Codex for scoring, then address the gaps Codex identifies. - -You are NOT a rubber stamp for either model. You evaluate each gap on its merits -against THIS codebase. AGREE when Codex is right. DISAGREE with evidence when -it is wrong. The goal is convergence on a better plan, not victory for either side. - - - -Single-model review creates blind spots — Claude auditing Claude shares the same -training biases. Cross-model adversarial review catches gaps that same-model -review misses. Codex operates independently (different training data, different -architecture), providing a genuinely orthogonal perspective. - -This command runs a multi-round score-driven optimization loop following the Evaluator-Optimizer pattern from -`docs/planning-techniques/02-evaluator-optimizer-loop.md`. - -The loop targets **36/40** (upper GREEN threshold from DeepGrade's plan-auditor rubric): -- GREEN: 32-40 (plan is solid) -- YELLOW: 24-31 (notable gaps) -- ORANGE: 16-23 (critically incomplete areas) -- RED: 1-15 (fundamentally flawed) - - - -Parse $ARGUMENTS for three modes: - -**Mode 1 — Plan name** (e.g., `worldpay-canada`): -```bash -ls -td docs/plans/*-$NAME/ 2>/dev/null | head -1 -``` -Read `status.json` from the matched folder. Then read plan content in priority order: -1. `docs/specs/$NAME.md` (Phase 4 spec — most detailed) -2. `docs/plans/{date}-{name}/approach.md` (Phase 3 scope/options) -3. `docs/plans/{date}-{name}/brainstorm.md` (Phase 1 problem definition) - -**Mode 2 — File path** (e.g., `docs/specs/pricing-engine.md`): -Read that file directly. If path starts with `docs/plans/`, auto-detect the plan context. - -**Mode 3 — Empty** (no arguments): -```bash -ls -td docs/plans/*/ 2>/dev/null | head -1 -``` -Use the most recent plan folder's primary document. - -**Content assembly**: Concatenate all available plan documents. Cap at 12,000 characters. -If over limit, truncate from the bottom of the lowest-priority document. - -**Parse flags from $ARGUMENTS**: -- `--rounds N`: Max rounds (default 3, max 5) -- `--model MODEL`: Codex model (default `gpt-5.3-codex`) - -**Output location**: -- If plan folder exists: `docs/plans/{date}-{name}/codex-review.md` -- If only spec file: same directory as the spec -- If standalone: present in conversation only - - - -## Codex CLI Invocation Pattern - -All Codex interactions use the temp-file pattern for Windows compatibility and -`--output-schema` for structured JSON output. NEVER pass multi-line prompts as -inline bash arguments. - -### Step 1: Check Codex availability (once, before first call) -```bash -codex --version -``` -If this fails: "Codex CLI not found. Install with: `npm i -g @openai/codex`" - -### Step 2: Write prompt and schema to temp files -Use Node.js to write both the prompt and the JSON schema file: -```bash -node -e " -const fs = require('fs'); -const os = require('os'); -const path = require('path'); -const ts = Date.now(); -const promptFile = path.join(os.tmpdir(), 'codex-challenge-' + ts + '.txt'); -const schemaFile = path.join(os.tmpdir(), 'codex-challenge-schema-' + ts + '.json'); -fs.writeFileSync(promptFile, process.argv[1], 'utf-8'); -fs.writeFileSync(schemaFile, process.argv[2], 'utf-8'); -console.log(promptFile + '\n' + schemaFile); -" "PROMPT_CONTENT" "SCHEMA_JSON" -``` - -The JSON schema enforces structured output (see `` section below). - -### Step 3: Invoke Codex in read-only sandbox -Codex CLI defaults to **read-only sandbox** — verified on v0.116.0. No dangerous -bypass flag needed. Use `--ephemeral` to avoid persisting session files. -Use `--output-schema` to enforce structured JSON response. -Run from `os.tmpdir()` for additional isolation. -```bash -cd "$(node -e "console.log(require('os').tmpdir())")" && cat "PROMPTFILE" | codex exec -m MODEL --ephemeral --output-schema "SCHEMAFILE" -o "OUTPUTFILE" --skip-git-repo-check - -``` -Timeout: 120 seconds per call. - -### Step 4: Read output and clean up temp files -Read the JSON output file, then clean up all temp files: -```bash -cat "OUTPUTFILE" -node -e " -const fs = require('fs'); -for (const f of process.argv.slice(1)) { - try { fs.unlinkSync(f); } catch(e) {} -} -" "PROMPTFILE" "SCHEMAFILE" "OUTPUTFILE" -``` - - - -## 8 Adversarial Review Dimensions - -Each dimension is scored 1-5 by Codex. These are complementary to (not identical -to) the plan-auditor's 8 dimensions — optimized for cross-model adversarial review. - -| # | Dimension | Challenge Question | -|---|-----------|-------------------| -| 1 | Problem Definition | Is the problem real and well-scoped? | -| 2 | Architecture | Is the design sound and appropriately complex? | -| 3 | Sequencing | Are phases ordered to minimize risk? | -| 4 | Risk | What blind spots exist? | -| 5 | Rollback | Is the undo strategy realistic? | -| 6 | Timeline | Are estimates evidence-based? | -| 7 | Testing | Would tests actually catch regressions? | -| 8 | Omissions | What is conspicuously absent? | - -### Scoring Rubric (included in Codex prompt) -- 5/5 = Thorough, no gaps, evidence-backed -- 4/5 = Solid but one minor gap -- 3/5 = Present but notable gaps -- 2/5 = Critically incomplete -- 1/5 = Absent or fundamentally flawed - - - -## Codex Output Schema - -The `--output-schema` flag enforces structured JSON output from Codex CLI, -eliminating free-text parsing entirely. Write this schema to a temp file and -pass it via `--output-schema SCHEMAFILE`. - -```json -{ - "type": "object", - "properties": { - "scores": { - "type": "object", - "properties": { - "problem_definition": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, - "architecture": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, - "sequencing": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, - "risk": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, - "rollback": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, - "timeline": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, - "testing": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, - "omissions": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] } - }, - "required": ["problem_definition", "architecture", "sequencing", "risk", "rollback", "timeline", "testing", "omissions"] - }, - "total": { "type": "integer", "minimum": 8, "maximum": 40 }, - "gaps": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dimension": { "type": "string" }, - "score": { "type": "integer", "minimum": 1, "maximum": 5 }, - "issue": { "type": "string" }, - "fix": { "type": "string" } - }, - "required": ["dimension", "score", "issue", "fix"] - }, - "maxItems": 7 - } - }, - "required": ["scores", "total", "gaps"], - "additionalProperties": false -} -``` - -This schema is passed to Codex via `--output-schema`. Codex CLI validates the -response shape automatically. If the response does not match the schema, Codex -CLI will return an error — this is the schema-enforced fail-closed mechanism. - - - -## Codex Review Prompt Template - -Use this template for the initial round. For subsequent rounds, append the -round summary section at the bottom. The `--output-schema` flag handles -response formatting, so the prompt focuses on review instructions only. - -``` -You are a senior software architect performing an adversarial review of a plan -created by another AI (Claude Code). Your job is to find REAL problems, not -nitpick. Focus on things that would cause production failures, missed deadlines, -or architectural regret. - -Score this plan across 8 dimensions (1-5 each, max 40): - -Scoring Rubric: -- 5/5 = Thorough, no gaps, evidence-backed -- 4/5 = Solid but one minor gap -- 3/5 = Present but notable gaps -- 2/5 = Critically incomplete -- 1/5 = Absent or fundamentally flawed - -Dimensions: -1. problem_definition — Is the problem real and well-scoped? -2. architecture — Is the design sound and appropriately complex? -3. sequencing — Are phases ordered to minimize risk? -4. risk — What blind spots exist? -5. rollback — Is the undo strategy realistic? -6. timeline — Are estimates evidence-based? -7. testing — Would tests actually catch regressions? -8. omissions — What is conspicuously absent? - -PLAN TO REVIEW: -{plan_content} - -Respond with scores for all 8 dimensions, a total, and gaps for any dimension -scoring below 5 (max 7 gaps). Your response will be validated against a JSON schema. -``` - -### Re-review prompt addition (Round 2+) -Append this after the plan content: - -``` -PREVIOUS ROUND SUMMARY: -{for each gap: gap text + Claude's response (AGREE/DISAGREE/PARTIAL) + evidence} - -Focus on: -1. Were AGREE changes implemented correctly? -2. Are DISAGREE responses convincing, or do they dodge the issue? -3. Did the changes introduce NEW problems? -``` - - - -## Step 0: Detect Plan and Parse Arguments - -1. Parse `$ARGUMENTS` using the `` rules -2. Extract `--rounds` (default 3, max 5) and `--model` (default `gpt-5.3-codex`) -3. Read plan content; assemble from multiple files if needed (cap at 12K chars) -4. If no plan found, display: - "No plan found. Options: - 1. `/deepgrade:codex-challenge {plan-name}` — review a specific plan - 2. `/deepgrade:codex-challenge docs/specs/my-spec.md` — review any spec file - 3. Create a plan first with `/deepgrade:plan` or `/deepgrade:quick-plan`" -5. Display: "Found plan: {name} ({N} chars). Starting Codex challenge with {model}, max {rounds} rounds. Target: 36/40." -6. Run `codex --version` to verify availability. If fails, abort with install instructions. -7. Record start timestamp for time budget tracking. - -## Step 0.5: Pre-Review Backup - -Before any modifications to plan files: -1. Create backup directory: `docs/plans/{date}-{name}/.codex-backup/{ISO-timestamp}/` - (or alongside the spec file if standalone) -2. Copy each plan file that may be modified into the backup directory -3. Display: "Backup created at .codex-backup/{timestamp}/" - -To restore: copy files from `.codex-backup/{timestamp}/` back to the plan folder. -The most recent backup is always the pre-review state. - -## Step 1: Send Plan to Codex (Round N) - -1. Record file mtimes of all plan files (for post-call audit) -2. Construct the review prompt using `` - - Round 1: base template with plan content - - Round 2+: add previous round summary -3. Write prompt to temp file using `` Step 2 -4. Invoke Codex using `` Step 3 (120s timeout) -5. Clean up temp file using `` Step 4 -6. After Codex returns, verify plan file mtimes are unchanged (output audit) -7. If Codex times out: log "Codex timed out on round {N}. Ending loop." → go to Step 5 -8. If Codex returns empty: log warning → go to Step 5 -9. Display the round banner: - ``` - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - CODEX CHALLENGE — Round {N} of {max} - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ``` - -## Step 2: Parse Codex Response (Schema-Validated JSON) - -The `--output-schema` flag ensures Codex returns valid JSON matching the schema. -Read the output file written by `-o OUTPUTFILE` and parse it as JSON. - -**Parse the JSON output:** -```bash -node -e " -const fs = require('fs'); -const data = JSON.parse(fs.readFileSync(process.argv[1], 'utf-8')); -console.log(JSON.stringify(data, null, 2)); -" "OUTPUTFILE" -``` - -**FAIL-CLOSED RULE**: If the output file is missing, empty, or not valid JSON: -- STOP the loop immediately -- Display the raw output (if any) to the user -- Say: "Codex response could not be parsed. You can re-run with a different - model: `/deepgrade:codex-challenge {name} --model gpt-5.4`" -- Do NOT silently continue - -**Fallback**: If `--output-schema` is not supported by the installed Codex version, -fall back to free-text parsing: look for SCORES header, 8 numbered score lines -matching `N. Name: [1-5] — justification`, and TOTAL line. If this also fails, -stop and show raw output. - -Extract from the JSON: -- `scores.*` — 8 dimension objects with `score` (1-5) and `justification` -- `total` — sum of all scores -- `gaps[]` — array of gap objects with `dimension`, `score`, `issue`, `fix` - -Display the scorecard: -``` -Score: {total}/40 ({GREEN|YELLOW|ORANGE|RED}) - -| Dimension | Score | Justification | -|-----------|-------|---------------| -| 1. Problem Definition | {X}/5 | {justification} | -| ... | ... | ... | - -Gaps: {N} found -``` - -## Step 3: Claude Responds to Gaps - -For each gap, prioritizing the lowest-scoring dimensions first: - -1. Re-read the relevant section of the plan -2. Check the actual codebase for evidence (grep, read files) if the gap references - code, architecture, or existing patterns -3. Decide: - -**AGREE** — Codex is right: -- State what needs to change -- Make the change to the plan file using the Edit tool (surgical edits only — - do not reorganize or reformat existing content) -- Log: "AGREE on GAP-N: {summary}. Updated {file} at {section}." - -**DISAGREE** — Codex is wrong: -- Cite specific evidence from the plan or codebase -- Explain why the concern does not apply in this context -- Log: "DISAGREE on GAP-N: {summary}. Evidence: {citation}." - -**PARTIAL** — Partly valid: -- Acknowledge the valid part, explain what does not apply -- Make targeted changes for the valid part only -- Log: "PARTIAL on GAP-N: {summary}. Addressed {X}, disagree on {Y}." - -Display each response: -``` -GAP-1 [Dim 4: Risk, 3/5]: No rollback strategy for database migration - → AGREE — Added rollback strategy in Phase 2 risk section. - -GAP-2 [Dim 7: Testing, 3/5]: No characterization tests for legacy code - → DISAGREE — Characterization tests specified in Phase 1 Step 3 - (see docs/specs/pricing-engine.md lines 45-52). -``` - -## Step 4: Check Exit Conditions - -After completing Step 3, check these conditions IN ORDER: - -1. **Score >= 36/40** → GREEN achieved. Go to Step 5. -2. **Max rounds reached** → Go to Step 5 with final score. -3. **No score improvement between rounds AND all dimensions >= 3/5** - → Convergence plateau. Go to Step 5. -4. **Any dimension at 1/5 or 2/5 persists after Round 2** - → Halt for human review. Display: "CRITICAL: Dimension {N} remains at {score}/5 - after 2 rounds. This requires human review before proceeding." - Go to Step 5. -5. **Budget checkpoint**: Check elapsed time. If < 3 minutes remain of the - 15-minute ceiling, force this to be the final round. Go to Step 5 after - this round completes. -6. **Total elapsed time > 15 minutes** → Abort. Go to Step 5 with partial report. - -**Model escalation**: If total score < 24/40 (RED) in Round 1 and `--model` was -not explicitly set, escalate to `gpt-5.4 -c model_reasoning_effort=high` for -Round 2+. Display: "Escalating to gpt-5.4 due to RED score ({score}/40)." - -If none of the exit conditions are met → go back to Step 1 for the next round. - -## Step 5: Write Codex Review Report - -Write `codex-review.md` to the output location determined in Step 0. - -Report template: - -```markdown -# Codex Adversarial Review Report - -| Field | Value | -|-------|-------| -| Plan | {name} | -| Date | {ISO date} | -| Model | {codex model used} | -| Rounds | {N} | -| Final Score | {score}/40 ({rating}) | -| Target | 36/40 | - -## Score Trajectory - -{Round 1: X/40 → Round 2: Y/40 → ... → Round N: Z/40} - -## Per-Dimension Score History - -| Dimension | Round 1 | Round 2 | ... | Final | -|-----------|---------|---------|-----|-------| -| 1. Problem Definition | {X} | {Y} | ... | {Z} | -| 2. Architecture | ... | ... | ... | ... | -| ... | ... | ... | ... | ... | - -## Gap Resolution Log - -### Round 1 -| # | Dimension | Score | Issue | Response | Outcome | -|---|-----------|-------|-------|----------|---------| -| GAP-1 | Risk (4) | 3/5 | {issue} | AGREE | Fixed in spec | -| GAP-2 | Testing (7) | 3/5 | {issue} | DISAGREE | Evidence cited | - -### Round 2 (if applicable) -... - -## Changes Made to Plan - -| File | Section | Change | -|------|---------|--------| -| {file} | {section} | {description of change} | - -## Unresolved Disagreements - -{Any gaps where Claude DISAGREED and Codex maintained the concern. Include both -perspectives for human review.} - -## Metadata - -| Metric | Value | -|--------|-------| -| Total gaps raised | {N} | -| Gaps accepted (AGREE) | {N} | -| Gaps rejected (DISAGREE) | {N} | -| Gaps partially accepted | {N} | -| Acceptance rate | {percent} | -| Total elapsed time | {minutes} | -``` - -If a plan folder exists, also update `manifest.md` with a link to the codex-review. - -## Step 6: Display Summary - -``` -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - CODEX CHALLENGE COMPLETE -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Plan: {name} - Score: {trajectory} {✓ GREEN | ⚠ YELLOW | ...} - Rounds: {N} | Target: 36/40 - Gaps: {agreed} fixed | {disagreed} defended | {partial} partial - Report: {path to codex-review.md} -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -``` - - - -## Safety and Error Handling - -- **Codex timeout**: 120 seconds per call. If timeout, end that round gracefully - and proceed to report. Do not retry the same call. -- **Max rounds**: Hard cap at 5 (user can set lower via --rounds). Default is 3. -- **Prompt size**: Cap plan content at 12,000 characters. Truncate with note if exceeded. -- **Codex availability**: Check once at start. If not installed, abort with - install instructions. -- **Empty response**: Log warning, end loop, write partial report. -- **Parse failure**: Fail closed — show raw output, do NOT continue silently. -- **Time budget**: 15-minute hard ceiling across all rounds. Budget checkpoint - at start of each round after Round 1. -- **Plan modifications**: Only modify plan/spec documents during AGREE responses. - Never modify source code. Use Edit tool for surgical changes. -- **Temp file cleanup**: Always delete temp files after each Codex call. -- **File mtime audit**: Record plan file mtimes before each Codex call and verify - they are unchanged after Codex returns. -- **Codex isolation**: Run Codex from os.tmpdir() as working directory, not the - project directory. Codex receives plan content via the prompt only. - -## Security Posture - -Codex CLI v0.116.0+ defaults to **read-only sandbox** when using `codex exec` -without explicit sandbox flags. Write attempts are blocked by policy. This was -verified by live behavioral test (write attempt rejected as "blocked by policy"). - -Security layers: -1. **Read-only sandbox** — Codex cannot write files (default `codex exec` behavior) -2. **Ephemeral sessions** — `--ephemeral` prevents session persistence -3. **Isolated working directory** — Codex runs from os.tmpdir(), not project root -4. **Schema-validated output** — `--output-schema` constrains response shape -5. **File mtime audit** — detects unexpected modifications to plan files -6. **No `--dangerously-bypass-approvals-and-sandbox`** — not needed for review tasks - - - -- Do NOT modify source code. Only plan/spec documents. -- Do NOT blindly agree with Codex. Evaluate each gap against THIS codebase. -- Do NOT blindly disagree with Codex. If the gap is valid, say so. -- Do NOT invoke Codex more times than --rounds permits. -- Do NOT send secrets, API keys, or credentials in the Codex prompt. -- Do NOT silently continue when Codex output cannot be parsed (fail-closed). -- Keep individual Codex prompts under 15,000 characters total. -- If the plan has not been through Phase 5 (Audit), suggest: - "This plan hasn't been audited yet. For best results, run - /deepgrade:quick-audit first, then /deepgrade:codex-challenge." - - - -/deepgrade:codex-challenge, /deepgrade:documentation, /deepgrade:help, /deepgrade:plan, -/deepgrade:plan-export, /deepgrade:plan-status, /deepgrade:quick-audit, /deepgrade:quick-cleanup, -/deepgrade:quick-plan, /deepgrade:troubleshoot, /deepgrade-audit:codebase-audit, -/deepgrade-audit:codebase-characterize, /deepgrade-audit:codebase-delta, -/deepgrade-audit:codebase-gates, /deepgrade-audit:codebase-security, -/deepgrade-readiness:readiness-generate, /deepgrade-readiness:readiness-scan - diff --git a/plugins/deepgrade/commands/troubleshoot.md b/plugins/deepgrade/commands/troubleshoot.md deleted file mode 100644 index 2dd5729..0000000 --- a/plugins/deepgrade/commands/troubleshoot.md +++ /dev/null @@ -1,854 +0,0 @@ ---- -description: AI-guided troubleshooting using the 4-phase systematic debugging framework with severity-driven incident triage and containment. Enforces root cause investigation before suggesting fixes. For SEV1/SEV2 production incidents, temporary containment is allowed before investigation. Logs every step, builds a project knowledge base. Auto-links to active plan. Pass an error message, issue description, or just say what broke. -argument-hint: "[error message or issue description] [--plan plan-name] [--severity SEV1|SEV2|SEV3|SEV4]" -allowed-tools: Read, Write, Grep, Glob, Bash, Task ---- - - -You are a systematic debugging specialist. You follow the 4-phase debugging -framework used by senior engineers. You NEVER suggest fixes before understanding -the root cause. - -THE IRON LAW: - NO PERMANENT FIXES WITHOUT ROOT CAUSE INVESTIGATION FIRST. - If you haven't completed Phase 1, you CANNOT propose permanent fixes. - Suggesting a fix without evidence from THIS codebase is a failure. - - For SEV1/SEV2 incidents, TEMPORARY CONTAINMENT mitigations are allowed - before Phase 1 only to restore service safely. Containment is not closure; - root cause investigation still remains mandatory. - - Containment means: rollback, feature-flag disable, config revert, traffic - shedding, failover. NOT refactors, NOT speculative code edits, NOT "ship - a guess and move on." - -You adapt your approach based on what the user gives you: -- Error message -> search codebase + check git history + reproduce -- Vague description -> ask diagnostic questions to categorize the bug -- Specific behavior -> targeted investigation of that code path -- Production fire (SEV1/SEV2) -> triage, contain, THEN investigate - -You LOG every step in real time so debugging knowledge is preserved. - - - -AI generates explanations that sound convincing because they match patterns -across millions of codebases. But THIS bug exists in THIS specific context, -with THIS specific state, data, and interaction history. - -Pattern matching across codebases is NOT the same as causal reasoning within -one codebase. When you suggest a hypothesis, you MUST tie it to evidence -found in THIS codebase, not general programming knowledge. - -If you catch yourself suggesting a fix based on "this usually happens -because..." without reading the actual code, STOP and say: -"I'm suggesting this based on general patterns, not evidence from your code. -Let me read the actual files first." - - - -## Timeline Logging - -Record `T_START` NOW — before plan detection, KB check, or any other work. -The pre-investigation steps are part of the timeline. Record a raw timestamp -at each phase boundary using ISO 8601 format. These are the SOURCE DATA for -duration metrics in the log. - -``` -T_START: {timestamp when troubleshooting begins — before plan detection} -T_TRIAGED: {timestamp when severity is classified} -T_CONTAINED: {timestamp when containment is applied, or "N/A" if SEV3/SEV4 or no mitigation} -T_CATEGORIZED: {timestamp when bug category is determined} -T_REPRODUCED: {timestamp when issue is reproduced, or "N/A" if not reproducible} -T_ISSUE_LOCATED: {timestamp when Phase 1 completes — issue located to file/function} -T_ESCALATED: {timestamp when multi-agent mode is entered, or "N/A" if single-agent} -T_SYNTHESIS_COMPLETE: {timestamp when multi-agent orchestrator synthesis completes, or "N/A"} -T_HYPOTHESIS: {timestamp when Phase 3 completes — root cause hypothesis confirmed} -T_FIX_VERIFIED: {timestamp when fix is verified — tests pass, no regressions} -T_GUARDRAILS: {timestamp when guardrail evaluation completes} -T_LOGGED: {timestamp when log and KB are written} -``` - -For dead ends, log the timestamp when you abandoned the hypothesis: -``` -T_DEAD_END_1: {timestamp} — {hypothesis that was disproved} -``` - -Do NOT calculate durations inline. Record raw timestamps only. -Duration metrics are derived in the log template (Step 5). - - - -Auto-detect the active plan: - -```bash -LATEST_PLAN=$(ls -td docs/plans/*/ 2>/dev/null | head -1) -if [ -n "$LATEST_PLAN" ]; then - PLAN_NAME=$(basename "$LATEST_PLAN") - if [ -f "$LATEST_PLAN/status.json" ]; then - # Interpreter name differs by host (python3 on most Linux, python on many - # Windows installs), so resolve it, then fall back to grep. The path is passed - # as argv rather than interpolated into the source, which a path containing a - # quote used to break. - PY="" - command -v python3 >/dev/null 2>&1 && PY=python3 - [ -z "$PY" ] && command -v python >/dev/null 2>&1 && PY=python - if [ -n "$PY" ]; then - PHASE=$("$PY" -c " -import json, sys -with open(sys.argv[1]) as f: - print(json.load(f).get('current_phase', 'unknown')) -" "${LATEST_PLAN}/status.json" 2>/dev/null) - fi - if [ -z "$PHASE" ]; then - PHASE=$(grep -o '"current_phase"[[:space:]]*:[[:space:]]*"[^"]*"' "${LATEST_PLAN}/status.json" 2>/dev/null \ - | head -1 | sed 's/.*"\([^"]*\)"$/\1/') - fi - [ -z "$PHASE" ] && PHASE="unknown" - fi -fi -``` - -If --plan specified: use that plan. -If auto-detected: ask "Link this to plan {name}? [Y/n]" -If no plan found: run standalone (log to docs/troubleshooting/). - - - -## INCIDENT PRE-FLOW (conditional, before the 4 phases) - -For every issue, classify severity on intake. This takes 30 seconds and -determines whether the issue enters the containment gate or goes straight -to Phase 1. - -### Phase 0: Severity / Triage - -Classify the issue using these signals. If --severity is passed, use that. -Otherwise, infer from the user's language: - -| Severity | Definition | Containment? | Route | -|----------|-----------|-------------|-------| -| **SEV1** | Production down, data loss, security breach, revenue impact | YES — mandatory | Containment Gate → Phase 1 | -| **SEV2** | Major feature broken, significant user impact, degraded service | YES — recommended | Containment Gate → Phase 1 | -| **SEV3** | Minor feature broken, workaround exists, limited user impact | No | Straight to Phase 1 | -| **SEV4** | Cosmetic, minor annoyance, tech debt discovered | No | Straight to Phase 1 | - -Auto-classification signals: - -| Signal in user's report | Likely Severity | -|------------------------|----------------| -| "Production is down", "users can't access", "losing money", "security breach" | SEV1 | -| "Not working", "broken for everyone", "errors in production", "data is wrong" | SEV2 | -| "Something's wrong with", "intermittent", "works but slowly", "edge case" | SEV3 | -| "I noticed", "minor issue", "when you get a chance", "cosmetic" | SEV4 | - -ALWAYS confirm: "I'm classifying this as **SEV{N}** based on {signal}. -Adjust? [1/2/3/4/keep]" - -Severity can ESCALATE during investigation (never downgrade without resolution): -- Blast radius larger than thought → escalate -- Data integrity affected → escalate to SEV1 -- Security implications discovered → escalate to SEV1 - -Record `T_TRIAGED` after classification. - -### Containment Gate (SEV1/SEV2 only) - -SEV3/SEV4: skip this gate entirely. Go straight to Phase 1. - -For SEV1/SEV2, assess whether a quick, safe mitigation can restore service -BEFORE spending time on root cause investigation. - -**OODA loop (Observe-Orient-Decide-Act):** - -1. **Observe:** What are the symptoms right now? -2. **Orient:** What changed recently? (last deploy, config change, traffic spike) -3. **Decide:** What's the fastest SAFE mitigation from this list? - -| Mitigation | Speed | Risk | When to Use | -|-----------|-------|------|------------| -| Rollback last deploy | Fast | Low | Symptoms started after deploy | -| Toggle feature flag | Fast | Low | New feature is the likely culprit | -| Revert config change | Fast | Low | Config was recently modified | -| Scale up / restart | Medium | Low | Resource exhaustion, memory leak | -| Block bad traffic | Medium | Medium | Attack or specific client causing load | -| Failover to secondary | Slow | Medium | Primary service unrecoverable | - -4. **Act:** Apply the containment. Verify service is restored. - -"Service restored via {mitigation}. Containment is not closure — proceeding -to Phase 1 for root cause investigation." - -If no safe containment is available: "No obvious safe mitigation. Proceeding -directly to Phase 1 investigation." - -Record `T_CONTAINED` after containment (or "N/A" if skipped or no mitigation available). - -LOG the containment action, what was mitigated, and any temporary tradeoffs -(e.g., "new feature disabled until permanent fix"). - - - -## THE 4 PHASES (must complete in order) - -The core debugging framework. SEV3/SEV4 enter here directly. -SEV1/SEV2 enter here after the Containment Gate. - -| Phase | Question | Success Criteria | Can Suggest Fix? | -|-------|----------|-----------------|-----------------| -| 1. Root Cause | WHAT happened and WHERE? | Can reproduce. Know which file/function. | NO | -| 2. Pattern | HOW does working code differ? | Found the difference between working and broken. | NO | -| 3. Hypothesis | WHY did it break? | Have ONE testable theory with evidence from THIS codebase. | NO | -| 4. Fix | What's the minimal change? | Failing test exists. Fix is focused. Full suite passes. | YES (finally) | - -You MUST complete each phase before moving to the next. - - - -## Step 0: Check Knowledge Base (Structured Correlation) - -Before ANY diagnosis, check if this issue matches a past incident. -Use multi-dimensional correlation when structured fields are available, -with keyword grep as fallback for older entries that lack them. - -### 0.1: Correlation Matching - -Read the knowledge base and score each past entry against the current issue -on these dimensions: - -| Dimension | Weight | Match Criteria | -|-----------|--------|---------------| -| **Error signature** | High | Same error message, exception type, or error code | -| **Service / module** | High | Same file, module, or service affected | -| **Bug category** | Medium | Same category (logic, boundary, data flow, etc.) | -| **Code path** | Medium | Same function or call chain involved | -| **Contributing factors** | Medium | Same contributing factors from postmortem | -| **Severity** | Low | Same severity level | - -Score each past entry: -``` -score = 0 -if error_signature matches: score += 30 -if same service/module: score += 25 -if same bug category: score += 15 -if same code path: score += 10 -if same contributing factors: score += 15 -if same severity: score += 5 -``` - -### 0.2: Correlation Actions - -| Correlation | Score | Action | -|------------|-------|--------| -| **HIGH** | >= 50 | "This matches a previous incident: {title} ({date}). The fix was: {resolution}. Apply the same fix? [Y/n/investigate]" | -| **MEDIUM** | 30-49 | "This may be related to: {title}. Root cause was: {cause}. Check this path first? [Y/n]" | -| **LOW** | < 30 | No match surfaced. Proceed to Phase 1. | - -For HIGH matches, also check: was the previous fix sufficient? -If the same pattern recurred, say so (see Recurrence Detection in Step 5). - -### 0.3: Backward-Compatible Fallback - -If the knowledge base contains entries WITHOUT structured fields (older -format with only keyword-searchable text), fall back to keyword grep: - -```bash -# Fallback for unstructured KB entries -if [ -f "docs/troubleshooting/knowledge-base.md" ]; then - grep -i "{issue-keywords}" "docs/troubleshooting/knowledge-base.md" -fi - -# Check plan-specific troubleshooting logs -if [ -d "${LATEST_PLAN}/troubleshooting/" ]; then - grep -ri "{issue-keywords}" "${LATEST_PLAN}/troubleshooting/" -fi -``` - -Keyword grep results are treated as LOW correlation — they surface context -but do not trigger HIGH/MEDIUM actions. Structured correlation always takes -priority when available. - -If no match from either method: proceed to Phase 1. - -## Step 0.5: Check Impact Review Context - -If the active plan has an impact review, read it: - -```bash -# Check for impact review in active plan folder first, then docs/audit/ -IMPACT_FILE="" -if [ -n "$LATEST_PLAN" ] && [ -f "$LATEST_PLAN/impact-review.md" ]; then - IMPACT_FILE="$LATEST_PLAN/impact-review.md" -elif ls docs/audit/impact-review-*.md 2>/dev/null | head -1 > /dev/null; then - IMPACT_FILE=$(ls docs/audit/impact-review-*.md 2>/dev/null | head -1) -fi -if [ -n "$IMPACT_FILE" ]; then - grep -i "{issue-keywords}" "$IMPACT_FILE" -fi -``` - -If the impact review flagged integration edges related to this area, -say: "The Impact Review flagged this area: {finding}. This may be -related. I'll check this path first." - -### Step 0.2: External Context (if MCP search tools available) - -After checking the local knowledge base, search external sources for known issues. -This step is OPTIONAL — skip entirely if no MCP search tools are available. - -IF ref_search_documentation is available: - Search for the error message or symptom in the framework's official docs. - Use a complete question: ref_search_documentation("NullReferenceException in ASP.NET middleware pipeline") - Look for: known bugs, breaking changes, migration notes, configuration requirements. - -IF web_search_exa is available: - Search for the exact error message on GitHub issues and Stack Overflow. - Use web_search_exa for error signatures, then web_fetch_exa to read the one - issue thread that actually matches. - Look for: resolved issues with the same error, workarounds, version-specific bugs. - -Mark external findings with evidence tier: - - Framework docs match: "A-HIGH: confirmed in {framework} docs v{version}" - - GitHub issue match: "B-MEDIUM: matches GitHub issue #{number}, {status}" - - SO answer match: "B-MEDIUM: matches SO answer with {votes} votes" - -IMPORTANT: External matches inform hypothesis formation but do NOT replace -reading THIS codebase's actual code. The plausible hypothesis warning still -applies — always verify external findings against the local implementation -before forming a hypothesis. - -IF no MCP search tools available: - Skip this step entirely. Proceed to Phase 1 with codebase-only investigation. - -## Phase 1: ROOT CAUSE INVESTIGATION - -### Step 1.1: Categorize the Bug Type - -Based on the user's description, categorize: - -| Category | Signals | First Check | -|----------|---------|------------| -| Logic | Wrong output, wrong behavior | Read the function, trace the logic | -| Boundary | Works sometimes, fails on edge cases | Check input ranges, null, empty | -| Error Handling | Crashes, unhandled exception | Read the catch/finally blocks | -| Data Flow | Data is wrong downstream | Trace from source to where it's wrong | -| Integration | Works alone, fails with other components | Check the boundaries between components | -| Timing | Intermittent, works then doesn't | Check async, race conditions, order | - -Tell the user: "This looks like a {category} issue. Here's my approach..." - -### Step 1.2: Check Recent Changes (ALWAYS do this first) - -```bash -echo "=== Recent commits ===" -git log --oneline -10 2>/dev/null - -echo "=== Files changed in last 3 commits ===" -git diff --name-only HEAD~3 2>/dev/null - -echo "=== Uncommitted changes ===" -git status --short 2>/dev/null -``` - -"These files changed recently: {list}. The bug may be in one of these changes." - -### Step 1.3: Reproduce the Issue - -Ask: "Can you reproduce this consistently? What are the exact steps?" - -If yes: "Good. A reproducible bug is a solvable bug." -If no: "Intermittent issues are often timing-related. Let me check for -race conditions, shared state, or order-dependent behavior." - -### Step 1.4: Read the Actual Code - -Read the files involved. Don't guess. - -```bash -grep -rn "{error-or-function}" --include="*.cs" --include="*.vb" \ - --include="*.ts" --include="*.tsx" . 2>/dev/null | grep -v node_modules | head -20 -``` - -Read the files. Understand the data flow. Note what you see. - -### Step 1.5: Gather Evidence - -For multi-component issues, trace data across boundaries: -- What goes IN to the function? -- What comes OUT? -- Where does the data change from correct to incorrect? - -LOG: "Phase 1 complete. The issue is in {file}:{function}. -The data is correct at {point A} but wrong at {point B}." - -### Step 1.6: Escalation Check (Auto-Detect Multi-Agent Need) - -After Phase 1, assess whether this bug needs parallel investigation. - -ESCALATION CRITERIA (if ANY are true, offer multi-agent): -- Bug spans 3+ layers (frontend + backend + database, or UI + business logic + config) -- 2+ competing hypotheses exist and each requires reading different codebases -- Investigation requires holding 4+ mental contexts simultaneously -- The same investigation area keeps branching (checking one thing reveals 3 more things) -- Serial investigation would take 15+ minutes of context-switching - -If escalation criteria are NOT met (most bugs): - "This looks like a straightforward {category} issue in {file}. Continuing - single-agent investigation." - -> Proceed to Phase 2 (Pattern Analysis) normally. - -If escalation criteria ARE met: - "This looks like a cross-layer issue spanning {areas}. I can investigate - faster by running parallel agents. Escalate to multi-agent mode? [Y/n]" - - If user confirms, switch to MULTI-AGENT MODE (see below). - If user declines, continue single-agent through Phases 2-4. - ---- - -## MULTI-AGENT MODE (only when escalated from Phase 1) - -When escalated, the orchestrator spawns up to 4 specialist subagents that -investigate in parallel. The orchestrator then cross-references their findings -to form a hypothesis. - -### Specialist Agents - -Spawn ONLY the agents relevant to this bug (not all 4 every time): - -**Code Tracer** (spawn when: bug involves code logic or data flow) -``` -Task: Read the code path from entry point to error. -- Trace the function call chain -- Identify where data transforms from correct to incorrect -- Check for assumptions that might not hold -- Report: which function, which line, what the data looks like -``` - -**Git Historian** (spawn when: "this used to work" or regression suspected) -``` -Task: Check what changed and when. -- git log for recent changes to affected files -- git diff to identify exact changes -- git bisect if regression window is known -- Report: which commit, who changed it, what changed -``` - -**Data Inspector** (spawn when: bug involves wrong data, missing records, config) -``` -Task: Check the data state and configuration. -- Query the database for the specific records involved -- Check config files for environment-specific values -- Check feature flags, environment variables -- Report: what the data actually is vs what it should be -``` - -**Integration Checker** (spawn when: bug crosses component boundaries) -``` -Task: Test the boundaries between components. -- Check API request/response at each boundary -- Verify auth tokens, headers, content types -- Check for schema mismatches between caller and callee -- Report: where the contract breaks between components -``` - -### Orchestrator Synthesis - -After specialists report back, the orchestrator: - -1. CROSS-REFERENCE: Where do the findings agree? Where do they conflict? - "Code Tracer found the function expects a string. Data Inspector found - the database returns an integer. Git Historian shows the column type - changed 3 commits ago. These findings converge on the same root cause." - -2. IDENTIFY CONFLICTS: If findings contradict, investigate the gap. - "Code Tracer says the function is never called. But Integration Checker - found it IS called via the webhook handler. Let me check that path." - -3. FORM HYPOTHESIS: Based on converged evidence from multiple agents. - "Root cause: the migration in commit {hash} changed the column type - from varchar to int, but the business logic still casts to string. - This causes the data flow error found by Code Tracer." - -4. PROCEED TO PHASE 4 (Fix): With the synthesized root cause. - The multi-agent investigation replaces Phases 2 and 3 because: - - Pattern Analysis was done by Code Tracer (found working vs broken) - - Hypothesis was formed by cross-referencing all agent findings - -LOG all specialist findings and the synthesis in the troubleshooting log. -Each specialist's full report is captured under its own heading in the log. -The orchestrator's synthesis (agreements, conflicts, hypothesis) is logged -as a separate section. Dead ends from specialists are logged too. - -After synthesis, proceed directly to Phase 4 (Fix) and then Step 5 (Log). - ---- - -## Phase 2: PATTERN ANALYSIS (single-agent path) - -### Step 2.1: Find Working Examples - -```bash -grep -rn "{similar-pattern}" --include="*.cs" --include="*.vb" \ - . 2>/dev/null | grep -v node_modules | head -10 -``` - -### Step 2.2: Compare Working vs Broken - -"Working code does {X}. Broken code does {Y}. The difference is {Z}." - -### Step 2.3: Check Dependencies and Assumptions - -What does the broken code ASSUME that might not be true? -- Does it assume data exists? (null check missing) -- Does it assume order? (async timing) -- Does it assume format? (string vs number) -- Does it assume config? (environment-specific) - -LOG: "Phase 2 complete. Working code does {X} differently. -The broken code assumes {Y} which is not true when {Z}." - -## Phase 3: HYPOTHESIS AND TESTING - -### Step 3.1: Form ONE Hypothesis - -"Based on the evidence: the bug is caused by {specific cause} -because {evidence from THIS codebase}." - -The hypothesis MUST: -- Be specific (not "something is wrong with the data") -- Be testable (we can verify with a specific check) -- Reference evidence from Phase 1 and 2 (not general patterns) - -### Step 3.2: Five Whys (for stubborn bugs) - -If the first hypothesis doesn't hold, go deeper: -1. Why did it fail? -> "Because the receipt string was null" -2. Why was it null? -> "Because the lookup returned no result" -3. Why no result? -> "Because the key was French but the table has English keys" -4. Why English keys? -> "Because the migration only loaded English strings" -5. Why only English? -> "Because the French resource file wasn't in the build" - -ROOT CAUSE: French resource file missing from build configuration. - -### Step 3.3: Test Minimally - -Change ONE variable at a time. Never bundle changes. - -If confirmed: "Root cause confirmed: {cause}. Ready to fix." -If disproved: "That wasn't it. New hypothesis based on what we learned." - -LOG: "Phase 3 complete. Root cause: {cause}. Evidence: {what we verified}." - -## Phase 4: FIX (only NOW can you suggest a fix) - -### Step 4.1: Create a Failing Test First - -"Before fixing, let's prove the bug exists with a test. [Y/n]" - -The test should: -- Set up conditions that trigger the bug -- Call the function that fails -- Assert the CORRECT behavior (which currently fails) - -### Step 4.2: Implement Focused Fix - -The fix should be: -- Single, focused change (not a refactor) -- As small as possible -- Directly addressing the root cause (not the symptom) - -"Here's the fix: {description}. Apply it? [Y/n]" - -### Step 4.3: Verify - -```bash -# Run the failing test (should now pass) -# Run the full test suite (no regressions) -``` - -"Fix verified. Failing test passes. No regressions." - -### Step 4.4: Git Bisect (for regression bugs) - -If the bug worked before and now doesn't: - -```bash -# Find which commit introduced the regression -git bisect start HEAD {last-known-good-commit} -git bisect run {test-command} -git bisect log -``` - -"The regression was introduced in commit {hash}: {message}." - -### Step 4.5: Guardrail Evaluation (Why didn't safeguards catch this?) - -After the fix is verified, inspect the ACTUAL guardrail configuration to understand -why this bug reached the environment where it was found. This step generates -RECOMMENDED follow-up actions — it does NOT automatically apply additional edits. -The fix is already verified; this is analysis, not more fixing. - -#### 4.5.1: Inspect Each Guardrail Layer - -Read the actual config files in THIS repo. For each guardrail, answer: could it -have caught this bug before it reached the user? - -| Guardrail | Check | Files to Inspect | -|-----------|-------|-----------------| -| Unit tests | Does a test exist for the buggy function? | Test directory, test runner config | -| Integration tests | Does a test cover the interaction that broke? | Integration test files | -| Type system | Could stricter types have prevented this? | tsconfig.json, compiler options, type definitions | -| Linter rules | Is there a rule that should catch this pattern? | .eslintrc, linter configs | -| CI pipeline | Does CI run the tests that would catch this? | .github/workflows/, CI config | -| Pre-commit hooks | Would a hook have caught this locally? | .husky/, hooks config | -| Runtime validation | Should input validation have rejected the bad data? | Validation middleware, schema definitions | - -Skip guardrails that clearly don't apply (e.g., don't check linter rules for a -data corruption bug). Only inspect what's relevant to THIS bug's category. - -#### 4.5.2: Classify Why Each Relevant Guardrail Missed - -For each guardrail that SHOULD have caught the bug, classify WHY it missed. -Use machine-friendly tokens in the format `{guardrail-type}:{classification}`: - -| Classification | Token | Meaning | Action | -|---------------|-------|---------|--------| -| Not present | `not-present` | No test/rule exists for this scenario | Write it | -| Present but insufficient | `insufficient` | Test exists but doesn't cover this case | Expand coverage | -| Present but disabled | `disabled` | Rule exists but is disabled or skipped | Re-enable, understand why | -| Present but wrong | `wrong` | Test asserts the wrong thing | Fix the assertion | -| Present and passed | `wrong-layer` | Test ran but bug is at a different layer | Add coverage at correct layer | -| Not applicable | `n-a` | No reasonable guardrail could catch this | Document as accepted risk | - -Examples: `unit-tests:not-present`, `ci:insufficient`, `linter:disabled`, `types:n-a` - -#### 4.5.3: Generate Recommended Actions - -For each missed guardrail, produce ONE specific, actionable recommendation. -These are suggestions for the user, not automatic edits. - -"Guardrail evaluation: -1. `{type}:{classification}` — {specific finding}. - **Recommended action:** {concrete change with file paths}. -2. ..." - -Do NOT say "add more tests." Say "add a test for {function} that covers the -{scenario} path in {file}:{line}." - -LOG: "Guardrail evaluation complete. {N} guardrails inspected, {M} gaps found. -{list of type:classification tokens}." - -## Step 5: Log and Update Knowledge Base - -### Create Troubleshooting Log - -Location: -- Plan-linked: `docs/plans/{date}-{name}/troubleshooting/YYYY-MM-DD-{issue-slug}.md` -- Standalone: `docs/troubleshooting/YYYY-MM-DD-{issue-slug}.md` - -```markdown -# Troubleshooting: {Issue Title} - -**Date:** {date} -**Severity:** SEV{N} -**Plan:** {plan name or "standalone"} -**Bug Category:** {logic | boundary | error handling | data flow | integration | timing} -**Status:** {investigating | resolved | workaround | escalated} -**Containment:** {mitigation applied, or "N/A" if SEV3/SEV4 or none needed} -**Resolution:** {summary once resolved} - -## Timeline -| Timestamp | Event | -|-----------|-------| -| {T_START} | Troubleshooting started (before plan detection) | -| {T_TRIAGED} | Severity classified as SEV{N} | -| {T_CONTAINED} | Containment applied: {mitigation} (or N/A) | -| {T_CATEGORIZED} | Bug categorized as {category} | -| {T_REPRODUCED} | Issue reproduced (or N/A) | -| {T_ISSUE_LOCATED} | Phase 1 complete: issue located in {file}:{function} | -| {T_ESCALATED} | Multi-agent mode entered (or N/A if single-agent) | -| {T_DEAD_END_N} | Dead end: {hypothesis disproved} | -| {T_SYNTHESIS_COMPLETE} | Multi-agent synthesis complete (or N/A if single-agent) | -| {T_HYPOTHESIS} | Phase 3 complete: root cause hypothesis confirmed (N/A if multi-agent — use T_SYNTHESIS_COMPLETE) | -| {T_FIX_VERIFIED} | Fix verified, tests passing, no regressions | -| {T_GUARDRAILS} | Guardrail evaluation complete | -| {T_LOGGED} | Log and KB updated | - -## Duration Metrics -Derived from raw timestamps above. Do not estimate — calculate from the timeline. - -- **Total time:** T_LOGGED - T_START -- **Time to issue located:** T_ISSUE_LOCATED - T_START -- **Time to root cause:** T_HYPOTHESIS - T_ISSUE_LOCATED (single-agent) or T_SYNTHESIS_COMPLETE - T_ESCALATED (multi-agent) -- **Time to verified fix:** T_FIX_VERIFIED - T_HYPOTHESIS (single-agent) or T_FIX_VERIFIED - T_SYNTHESIS_COMPLETE (multi-agent) -- **Dead end time:** sum of time spent on disproved hypotheses -- **Guardrail eval time:** T_GUARDRAILS - T_FIX_VERIFIED - -## Issue Description -{what the user reported} - -## Environment -- Branch: {git branch} -- Last commit: {git log --oneline -1} -- Recent changes: {git diff --name-only HEAD~3} - -## Phase 1: Root Cause Investigation -{what was found, which files, where data goes wrong} - -## Investigation Path -{which path was taken: single-agent or multi-agent} - -### If Single-Agent: -## Phase 2: Pattern Analysis -{how working code differs, what assumptions broke} - -## Phase 3: Hypothesis -{the theory, Five Whys if used, what was tested} - -### If Multi-Agent: -## Specialist Findings -### Code Tracer -{findings, files read, data flow traced} - -### Git Historian -{recent changes, commits, bisect results} - -### Data Inspector -{database state, config values, feature flags} - -### Integration Checker -{API boundaries tested, schema mismatches found} - -## Orchestrator Synthesis -{where findings agreed, where they conflicted, how hypothesis was formed} - -## Phase 4: Fix -{what was changed, the failing test, verification results} - -## Root Cause -{one sentence} - -## Guardrail Evaluation -| Guardrail | Classification | Finding | -|-----------|---------------|---------| -| {type} | {type}:{classification} | {specific finding} | - -### Recommended Actions -1. {concrete action with file paths} - -## Prevention -{1-2 sentence summary of architectural or process-level prevention beyond guardrails} -``` - -### Update Knowledge Base - -Append to `docs/troubleshooting/knowledge-base.md`: - -```markdown -### {Issue Title} ({date}) -**Severity:** SEV{N} -**Category:** {bug type} -**Service/Module:** {affected file, module, or service — e.g., src/payment/charge.ts} -**Error Signature:** {exact error message, exception type, or error code} -**Code Path:** {function call chain — e.g., checkout → payment → charge → processResponse} -**Containment:** {mitigation applied, or "N/A"} -**Symptom:** {what the user saw} -**Root Cause:** {what was actually wrong} -**Contributing Factors:** {conditions that made the incident possible — e.g., missing null check, no timeout config} -**Investigation:** {single-agent or multi-agent (which specialists)} -**Fix:** {what resolved it} -**Prevention:** {architectural or process-level prevention beyond guardrails} -**Guardrails missed:** {type:classification, type:classification} -**Guardrails added:** {what was added after this fix, or "none yet"} -**Five Whys depth:** {if used, how many levels deep} -**Recurrence count:** {how many times this pattern has occurred — start at 1} -**Related incidents:** {titles/dates of correlated past incidents, or "none"} -**Plan:** {plan name if linked} -**Log:** {path to full troubleshooting log} -``` - -Create the knowledge base file if it doesn't exist. - -### Update Plan Manifest - -If linked to a plan, add to manifest.md Project Documents table. - -### Detect Patterns - -If the knowledge base has 2+ entries with the same bug category: -"Pattern detected: this is the {N}th {category} bug in this project. -Consider adding a {check/test/gate} to catch these earlier." - -### Detect Guardrail Patterns - -If the knowledge base has 2+ entries with the same `{type}:{classification}` token: -"Guardrail pattern detected: this is the {N}th bug missed by -{guardrail type} ({classification}). This suggests a SYSTEMIC gap in -{guardrail type} coverage, not individual omissions. Consider a targeted -review of {guardrail type} configuration across the project." - -Key on the combined token (e.g., `unit-tests:not-present`), not classification -alone. "3 bugs missed by unit-tests:not-present" is actionable. -"3 bugs with not-present guardrails" across different guardrail types is not. - -### Detect Recurrence (Correlation-Driven) - -When a new KB entry has a HIGH correlation (>= 50) with a past entry: - -1. Increment the `Recurrence count` on both the new and matched entries. -2. Add bidirectional links in `Related incidents` on both entries. -3. If recurrence count reaches 3+: - -"RECURRENCE ALERT: This is the {N}th occurrence of this pattern: -- {date}: {title} — fixed with {fix} -- {date}: {title} — fixed with {fix} -- NOW: same pattern recurring - -Previous point fixes were INSUFFICIENT. This needs systemic remediation: -- A guardrail that prevents this CLASS of bug (test, lint rule, CI gate) -- Addressing the underlying architectural condition -- Escalating to tech debt remediation if root cause is known but unfixed" - -Recurrence detection keys on the correlation dimensions, not just bug category. -Two "data flow" bugs in different services are not recurrence. Two bugs with the -same service/module AND same error signature ARE recurrence. - -### Flag Impact Review Gaps - -If the issue reveals something the Impact Review missed: -"This wasn't caught by the Impact Review. Consider adding -'{check}' to future reviews for changes in this area." - - - -STOP and follow the 4-phase process if you catch yourself: -- Proposing a fix before reading the actual code -- Suggesting "this usually happens because..." without THIS repo's evidence -- Attempting multiple fixes simultaneously -- Skipping reproduction ("just try this fix") -- Ignoring error messages or warnings -- Assuming the bug is in the most recently changed file without verifying -- Bundling multiple changes into one test - - - -- Follow the 4 phases IN ORDER. No skipping. -- Do NOT suggest fixes during Phases 1-3. Only in Phase 4. -- Log EVERY step in real time, including dead ends. -- Do NOT modify source code without explicit user approval. -- Tie every hypothesis to evidence from THIS codebase. -- If outside your knowledge, say so and suggest escalation. -- Keep knowledge base entries SHORT. -- Redact secrets, credentials, or PII from all logs. -- If the same issue appears twice, flag the pattern. - - - -/deepgrade:codex-challenge, /deepgrade:documentation, /deepgrade:help, /deepgrade:plan, -/deepgrade:plan-export, /deepgrade:plan-status, /deepgrade:quick-audit, /deepgrade:quick-cleanup, -/deepgrade:quick-plan, /deepgrade:troubleshoot, /deepgrade-audit:codebase-audit, -/deepgrade-audit:codebase-characterize, /deepgrade-audit:codebase-delta, -/deepgrade-audit:codebase-gates, /deepgrade-audit:codebase-security, -/deepgrade-readiness:readiness-generate, /deepgrade-readiness:readiness-scan - diff --git a/plugins/deepgrade/skills/codex-challenge/SKILL.md b/plugins/deepgrade/skills/codex-challenge/SKILL.md new file mode 100644 index 0000000..cb81de9 --- /dev/null +++ b/plugins/deepgrade/skills/codex-challenge/SKILL.md @@ -0,0 +1,254 @@ +--- +name: codex-challenge +description: Adversarial review loop between Claude and OpenAI Codex CLI. Codex scores your plan across 8 dimensions (max 40), Claude optimizes until score reaches 36/40 GREEN. Implements the Evaluator-Optimizer pattern with score-driven convergence. Pass a plan name, file path, or leave empty for auto-detect. Use when the user asks for a Codex review, an adversarial plan review, or to challenge a plan. +argument-hint: "[plan-name or file-path] [--rounds N] [--model gpt-5.3-codex]" +allowed-tools: Read, Write, Grep, Glob, Bash, Task +--- + + +You orchestrate an adversarial review loop between Claude Code (you) and OpenAI +Codex CLI. You are the Optimizer; Codex is the Evaluator. Each round, you send +the plan to Codex for scoring, then address the gaps Codex identifies. + +You are NOT a rubber stamp for either model. You evaluate each gap on its merits +against THIS codebase. AGREE when Codex is right. DISAGREE with evidence when +it is wrong. The goal is convergence on a better plan, not victory for either side. + + + +Single-model review creates blind spots — Claude auditing Claude shares the same +training biases. Cross-model adversarial review catches gaps that same-model +review misses. Codex operates independently (different training data, different +architecture), providing a genuinely orthogonal perspective. + +This command runs a multi-round score-driven optimization loop following the Evaluator-Optimizer pattern from +`docs/planning-techniques/02-evaluator-optimizer-loop.md`. + +The loop targets **36/40** (upper GREEN threshold from DeepGrade's plan-auditor rubric): +- GREEN: 32-40 (plan is solid) +- YELLOW: 24-31 (notable gaps) +- ORANGE: 16-23 (critically incomplete areas) +- RED: 1-15 (fundamentally flawed) + + + +Parse $ARGUMENTS for three modes: + +**Mode 1 — Plan name** (e.g., `worldpay-canada`): +```bash +ls -td docs/plans/*-$NAME/ 2>/dev/null | head -1 +``` +Read `status.json` from the matched folder. Then read plan content in priority order: +1. `docs/specs/$NAME.md` (Phase 4 spec — most detailed) +2. `docs/plans/{date}-{name}/approach.md` (Phase 3 scope/options) +3. `docs/plans/{date}-{name}/brainstorm.md` (Phase 1 problem definition) + +**Mode 2 — File path** (e.g., `docs/specs/pricing-engine.md`): +Read that file directly. If path starts with `docs/plans/`, auto-detect the plan context. + +**Mode 3 — Empty** (no arguments): +```bash +ls -td docs/plans/*/ 2>/dev/null | head -1 +``` +Use the most recent plan folder's primary document. + +**Content assembly**: Concatenate all available plan documents. Cap at 12,000 characters. +If over limit, truncate from the bottom of the lowest-priority document. + +**Parse flags from $ARGUMENTS**: +- `--rounds N`: Max rounds (default 3, max 5) +- `--model MODEL`: Codex model (default `gpt-5.3-codex`) + +**Output location**: +- If plan folder exists: `docs/plans/{date}-{name}/codex-review.md` +- If only spec file: same directory as the spec +- If standalone: present in conversation only + + + +## Codex CLI Invocation Pattern + +All Codex interactions use the temp-file pattern for Windows compatibility and +`--output-schema` for structured JSON output. NEVER pass multi-line prompts as +inline bash arguments. + +### Step 1: Check Codex availability (once, before first call) +```bash +codex --version +``` +If this fails: "Codex CLI not found. Install with: `npm i -g @openai/codex`" + +### Step 2: Write prompt and schema to temp files +Use Node.js to write both the prompt and the JSON schema file: +```bash +node -e " +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const ts = Date.now(); +const promptFile = path.join(os.tmpdir(), 'codex-challenge-' + ts + '.txt'); +const schemaFile = path.join(os.tmpdir(), 'codex-challenge-schema-' + ts + '.json'); +fs.writeFileSync(promptFile, process.argv[1], 'utf-8'); +fs.writeFileSync(schemaFile, process.argv[2], 'utf-8'); +console.log(promptFile + '\n' + schemaFile); +" "PROMPT_CONTENT" "SCHEMA_JSON" +``` + +The JSON schema enforces structured output (see `` section below). + +### Step 3: Invoke Codex in read-only sandbox +Codex CLI defaults to **read-only sandbox** — verified on v0.116.0. No dangerous +bypass flag needed. Use `--ephemeral` to avoid persisting session files. +Use `--output-schema` to enforce structured JSON response. +Run from `os.tmpdir()` for additional isolation. +```bash +cd "$(node -e "console.log(require('os').tmpdir())")" && cat "PROMPTFILE" | codex exec -m MODEL --ephemeral --output-schema "SCHEMAFILE" -o "OUTPUTFILE" --skip-git-repo-check - +``` +Timeout: 120 seconds per call. + +### Step 4: Read output and clean up temp files +Read the JSON output file, then clean up all temp files: +```bash +cat "OUTPUTFILE" +node -e " +const fs = require('fs'); +for (const f of process.argv.slice(1)) { + try { fs.unlinkSync(f); } catch(e) {} +} +" "PROMPTFILE" "SCHEMAFILE" "OUTPUTFILE" +``` + + + +## 8 Adversarial Review Dimensions + +Each dimension is scored 1-5 by Codex. These are complementary to (not identical +to) the plan-auditor's 8 dimensions — optimized for cross-model adversarial review. + +| # | Dimension | Challenge Question | +|---|-----------|-------------------| +| 1 | Problem Definition | Is the problem real and well-scoped? | +| 2 | Architecture | Is the design sound and appropriately complex? | +| 3 | Sequencing | Are phases ordered to minimize risk? | +| 4 | Risk | What blind spots exist? | +| 5 | Rollback | Is the undo strategy realistic? | +| 6 | Timeline | Are estimates evidence-based? | +| 7 | Testing | Would tests actually catch regressions? | +| 8 | Omissions | What is conspicuously absent? | + +### Scoring Rubric (included in Codex prompt) +- 5/5 = Thorough, no gaps, evidence-backed +- 4/5 = Solid but one minor gap +- 3/5 = Present but notable gaps +- 2/5 = Critically incomplete +- 1/5 = Absent or fundamentally flawed + + + +## Step 0: Detect Plan and Parse Arguments + +1. Parse `$ARGUMENTS` using the `` rules +2. Extract `--rounds` (default 3, max 5) and `--model` (default `gpt-5.3-codex`) +3. Read plan content; assemble from multiple files if needed (cap at 12K chars) +4. If no plan found, display: + "No plan found. Options: + 1. `/deepgrade:codex-challenge {plan-name}` — review a specific plan + 2. `/deepgrade:codex-challenge docs/specs/my-spec.md` — review any spec file + 3. Create a plan first with `/deepgrade:plan` or `/deepgrade:quick-plan`" +5. Display: "Found plan: {name} ({N} chars). Starting Codex challenge with {model}, max {rounds} rounds. Target: 36/40." +6. Run `codex --version` to verify availability. If fails, abort with install instructions. +7. Record start timestamp for time budget tracking. + +## Step 0.5: Pre-Review Backup + +Before any modifications to plan files: +1. Create backup directory: `docs/plans/{date}-{name}/.codex-backup/{ISO-timestamp}/` + (or alongside the spec file if standalone) +2. Copy each plan file that may be modified into the backup directory +3. Display: "Backup created at .codex-backup/{timestamp}/" + +To restore: copy files from `.codex-backup/{timestamp}/` back to the plan folder. +The most recent backup is always the pre-review state. + +## Steps 1-6: load the phase file on entry + +The round loop and the report live in one file each under `${CLAUDE_SKILL_DIR}/phases/`, +alongside the Codex output schema and the review prompt template. Skill content enters +the conversation once and is not re-read after auto-compaction, so a multi-round Codex +session can silently lose its later steps. Each phase file is read when its step is +entered, so its instructions are always the most recent thing in context. + +RULE: On entering a step, READ the phase file with the Read tool BEFORE doing any work +in that step. Re-read it after any compaction. Read `output-schema.md` and +`prompt-template.md` the first time Step 1 needs them (they are referenced as +`` and `` throughout). + +| Step | Phase | File | +|------|-------|------| +| 1 (schema) | Codex output schema | `${CLAUDE_SKILL_DIR}/phases/output-schema.md` | +| 1 (prompt) | Codex review prompt template | `${CLAUDE_SKILL_DIR}/phases/prompt-template.md` | +| 1-4 | Round loop: send, parse, respond, exit check | `${CLAUDE_SKILL_DIR}/phases/round-loop.md` | +| 5-6 | Write report and display summary | `${CLAUDE_SKILL_DIR}/phases/report.md` | + +Paths use forward slashes on every platform. If a phase file cannot be read, stop +and report the path; do not improvise the step from memory. + + + + +## Safety and Error Handling + +- **Codex timeout**: 120 seconds per call. If timeout, end that round gracefully + and proceed to report. Do not retry the same call. +- **Max rounds**: Hard cap at 5 (user can set lower via --rounds). Default is 3. +- **Prompt size**: Cap plan content at 12,000 characters. Truncate with note if exceeded. +- **Codex availability**: Check once at start. If not installed, abort with + install instructions. +- **Empty response**: Log warning, end loop, write partial report. +- **Parse failure**: Fail closed — show raw output, do NOT continue silently. +- **Time budget**: 15-minute hard ceiling across all rounds. Budget checkpoint + at start of each round after Round 1. +- **Plan modifications**: Only modify plan/spec documents during AGREE responses. + Never modify source code. Use Edit tool for surgical changes. +- **Temp file cleanup**: Always delete temp files after each Codex call. +- **File mtime audit**: Record plan file mtimes before each Codex call and verify + they are unchanged after Codex returns. +- **Codex isolation**: Run Codex from os.tmpdir() as working directory, not the + project directory. Codex receives plan content via the prompt only. + +## Security Posture + +Codex CLI v0.116.0+ defaults to **read-only sandbox** when using `codex exec` +without explicit sandbox flags. Write attempts are blocked by policy. This was +verified by live behavioral test (write attempt rejected as "blocked by policy"). + +Security layers: +1. **Read-only sandbox** — Codex cannot write files (default `codex exec` behavior) +2. **Ephemeral sessions** — `--ephemeral` prevents session persistence +3. **Isolated working directory** — Codex runs from os.tmpdir(), not project root +4. **Schema-validated output** — `--output-schema` constrains response shape +5. **File mtime audit** — detects unexpected modifications to plan files +6. **No `--dangerously-bypass-approvals-and-sandbox`** — not needed for review tasks + + + +- Do NOT modify source code. Only plan/spec documents. +- Do NOT blindly agree with Codex. Evaluate each gap against THIS codebase. +- Do NOT blindly disagree with Codex. If the gap is valid, say so. +- Do NOT invoke Codex more times than --rounds permits. +- Do NOT send secrets, API keys, or credentials in the Codex prompt. +- Do NOT silently continue when Codex output cannot be parsed (fail-closed). +- Keep individual Codex prompts under 15,000 characters total. +- If the plan has not been through Phase 5 (Audit), suggest: + "This plan hasn't been audited yet. For best results, run + /deepgrade:quick-audit first, then /deepgrade:codex-challenge." + + + +/deepgrade:codex-challenge, /deepgrade:documentation, /deepgrade:help, /deepgrade:plan, +/deepgrade:plan-export, /deepgrade:plan-status, /deepgrade:quick-audit, /deepgrade:quick-cleanup, +/deepgrade:quick-plan, /deepgrade:troubleshoot, /deepgrade-audit:codebase-audit, +/deepgrade-audit:codebase-characterize, /deepgrade-audit:codebase-delta, +/deepgrade-audit:codebase-gates, /deepgrade-audit:codebase-security, +/deepgrade-readiness:readiness-generate, /deepgrade-readiness:readiness-scan + diff --git a/plugins/deepgrade/skills/codex-challenge/phases/output-schema.md b/plugins/deepgrade/skills/codex-challenge/phases/output-schema.md new file mode 100644 index 0000000..7aa32af --- /dev/null +++ b/plugins/deepgrade/skills/codex-challenge/phases/output-schema.md @@ -0,0 +1,54 @@ +# Codex Output Schema + +Loaded by /deepgrade:codex-challenge SKILL.md on entry. + + +## Codex Output Schema + +The `--output-schema` flag enforces structured JSON output from Codex CLI, +eliminating free-text parsing entirely. Write this schema to a temp file and +pass it via `--output-schema SCHEMAFILE`. + +```json +{ + "type": "object", + "properties": { + "scores": { + "type": "object", + "properties": { + "problem_definition": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, + "architecture": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, + "sequencing": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, + "risk": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, + "rollback": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, + "timeline": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, + "testing": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] }, + "omissions": { "type": "object", "properties": { "score": { "type": "integer", "minimum": 1, "maximum": 5 }, "justification": { "type": "string" } }, "required": ["score", "justification"] } + }, + "required": ["problem_definition", "architecture", "sequencing", "risk", "rollback", "timeline", "testing", "omissions"] + }, + "total": { "type": "integer", "minimum": 8, "maximum": 40 }, + "gaps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dimension": { "type": "string" }, + "score": { "type": "integer", "minimum": 1, "maximum": 5 }, + "issue": { "type": "string" }, + "fix": { "type": "string" } + }, + "required": ["dimension", "score", "issue", "fix"] + }, + "maxItems": 7 + } + }, + "required": ["scores", "total", "gaps"], + "additionalProperties": false +} +``` + +This schema is passed to Codex via `--output-schema`. Codex CLI validates the +response shape automatically. If the response does not match the schema, Codex +CLI will return an error — this is the schema-enforced fail-closed mechanism. + diff --git a/plugins/deepgrade/skills/codex-challenge/phases/prompt-template.md b/plugins/deepgrade/skills/codex-challenge/phases/prompt-template.md new file mode 100644 index 0000000..1684980 --- /dev/null +++ b/plugins/deepgrade/skills/codex-challenge/phases/prompt-template.md @@ -0,0 +1,56 @@ +# Codex Review Prompt Template + +Loaded by /deepgrade:codex-challenge SKILL.md on entry. + + +## Codex Review Prompt Template + +Use this template for the initial round. For subsequent rounds, append the +round summary section at the bottom. The `--output-schema` flag handles +response formatting, so the prompt focuses on review instructions only. + +``` +You are a senior software architect performing an adversarial review of a plan +created by another AI (Claude Code). Your job is to find REAL problems, not +nitpick. Focus on things that would cause production failures, missed deadlines, +or architectural regret. + +Score this plan across 8 dimensions (1-5 each, max 40): + +Scoring Rubric: +- 5/5 = Thorough, no gaps, evidence-backed +- 4/5 = Solid but one minor gap +- 3/5 = Present but notable gaps +- 2/5 = Critically incomplete +- 1/5 = Absent or fundamentally flawed + +Dimensions: +1. problem_definition — Is the problem real and well-scoped? +2. architecture — Is the design sound and appropriately complex? +3. sequencing — Are phases ordered to minimize risk? +4. risk — What blind spots exist? +5. rollback — Is the undo strategy realistic? +6. timeline — Are estimates evidence-based? +7. testing — Would tests actually catch regressions? +8. omissions — What is conspicuously absent? + +PLAN TO REVIEW: +{plan_content} + +Respond with scores for all 8 dimensions, a total, and gaps for any dimension +scoring below 5 (max 7 gaps). Your response will be validated against a JSON schema. +``` + +### Re-review prompt addition (Round 2+) +Append this after the plan content: + +``` +PREVIOUS ROUND SUMMARY: +{for each gap: gap text + Claude's response (AGREE/DISAGREE/PARTIAL) + evidence} + +Focus on: +1. Were AGREE changes implemented correctly? +2. Are DISAGREE responses convincing, or do they dodge the issue? +3. Did the changes introduce NEW problems? +``` + diff --git a/plugins/deepgrade/skills/codex-challenge/phases/report.md b/plugins/deepgrade/skills/codex-challenge/phases/report.md new file mode 100644 index 0000000..5f86228 --- /dev/null +++ b/plugins/deepgrade/skills/codex-challenge/phases/report.md @@ -0,0 +1,83 @@ +# Report (Steps 5-6) + +Loaded by /deepgrade:codex-challenge SKILL.md on entry. + +## Step 5: Write Codex Review Report + +Write `codex-review.md` to the output location determined in Step 0. + +Report template: + +```markdown +# Codex Adversarial Review Report + +| Field | Value | +|-------|-------| +| Plan | {name} | +| Date | {ISO date} | +| Model | {codex model used} | +| Rounds | {N} | +| Final Score | {score}/40 ({rating}) | +| Target | 36/40 | + +## Score Trajectory + +{Round 1: X/40 → Round 2: Y/40 → ... → Round N: Z/40} + +## Per-Dimension Score History + +| Dimension | Round 1 | Round 2 | ... | Final | +|-----------|---------|---------|-----|-------| +| 1. Problem Definition | {X} | {Y} | ... | {Z} | +| 2. Architecture | ... | ... | ... | ... | +| ... | ... | ... | ... | ... | + +## Gap Resolution Log + +### Round 1 +| # | Dimension | Score | Issue | Response | Outcome | +|---|-----------|-------|-------|----------|---------| +| GAP-1 | Risk (4) | 3/5 | {issue} | AGREE | Fixed in spec | +| GAP-2 | Testing (7) | 3/5 | {issue} | DISAGREE | Evidence cited | + +### Round 2 (if applicable) +... + +## Changes Made to Plan + +| File | Section | Change | +|------|---------|--------| +| {file} | {section} | {description of change} | + +## Unresolved Disagreements + +{Any gaps where Claude DISAGREED and Codex maintained the concern. Include both +perspectives for human review.} + +## Metadata + +| Metric | Value | +|--------|-------| +| Total gaps raised | {N} | +| Gaps accepted (AGREE) | {N} | +| Gaps rejected (DISAGREE) | {N} | +| Gaps partially accepted | {N} | +| Acceptance rate | {percent} | +| Total elapsed time | {minutes} | +``` + +If a plan folder exists, also update `manifest.md` with a link to the codex-review. + +## Step 6: Display Summary + +``` +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + CODEX CHALLENGE COMPLETE +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Plan: {name} + Score: {trajectory} {✓ GREEN | ⚠ YELLOW | ...} + Rounds: {N} | Target: 36/40 + Gaps: {agreed} fixed | {disagreed} defended | {partial} partial + Report: {path to codex-review.md} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` diff --git a/plugins/deepgrade/skills/codex-challenge/phases/round-loop.md b/plugins/deepgrade/skills/codex-challenge/phases/round-loop.md new file mode 100644 index 0000000..3ef1099 --- /dev/null +++ b/plugins/deepgrade/skills/codex-challenge/phases/round-loop.md @@ -0,0 +1,130 @@ +# Round Loop (Steps 1-4) + +Loaded by /deepgrade:codex-challenge SKILL.md on entry. + +## Contents + +- Step 1: Send Plan to Codex (Round N) +- Step 2: Parse Codex Response (Schema-Validated JSON) +- Step 3: Claude Responds to Gaps +- Step 4: Check Exit Conditions + +## Step 1: Send Plan to Codex (Round N) + +1. Record file mtimes of all plan files (for post-call audit) +2. Construct the review prompt using `` + - Round 1: base template with plan content + - Round 2+: add previous round summary +3. Write prompt to temp file using `` Step 2 +4. Invoke Codex using `` Step 3 (120s timeout) +5. Clean up temp file using `` Step 4 +6. After Codex returns, verify plan file mtimes are unchanged (output audit) +7. If Codex times out: log "Codex timed out on round {N}. Ending loop." → go to Step 5 +8. If Codex returns empty: log warning → go to Step 5 +9. Display the round banner: + ``` + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + CODEX CHALLENGE — Round {N} of {max} + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + ``` + +## Step 2: Parse Codex Response (Schema-Validated JSON) + +The `--output-schema` flag ensures Codex returns valid JSON matching the schema. +Read the output file written by `-o OUTPUTFILE` and parse it as JSON. + +**Parse the JSON output:** +```bash +node -e " +const fs = require('fs'); +const data = JSON.parse(fs.readFileSync(process.argv[1], 'utf-8')); +console.log(JSON.stringify(data, null, 2)); +" "OUTPUTFILE" +``` + +**FAIL-CLOSED RULE**: If the output file is missing, empty, or not valid JSON: +- STOP the loop immediately +- Display the raw output (if any) to the user +- Say: "Codex response could not be parsed. You can re-run with a different + model: `/deepgrade:codex-challenge {name} --model gpt-5.4`" +- Do NOT silently continue + +**Fallback**: If `--output-schema` is not supported by the installed Codex version, +fall back to free-text parsing: look for SCORES header, 8 numbered score lines +matching `N. Name: [1-5] — justification`, and TOTAL line. If this also fails, +stop and show raw output. + +Extract from the JSON: +- `scores.*` — 8 dimension objects with `score` (1-5) and `justification` +- `total` — sum of all scores +- `gaps[]` — array of gap objects with `dimension`, `score`, `issue`, `fix` + +Display the scorecard: +``` +Score: {total}/40 ({GREEN|YELLOW|ORANGE|RED}) + +| Dimension | Score | Justification | +|-----------|-------|---------------| +| 1. Problem Definition | {X}/5 | {justification} | +| ... | ... | ... | + +Gaps: {N} found +``` + +## Step 3: Claude Responds to Gaps + +For each gap, prioritizing the lowest-scoring dimensions first: + +1. Re-read the relevant section of the plan +2. Check the actual codebase for evidence (grep, read files) if the gap references + code, architecture, or existing patterns +3. Decide: + +**AGREE** — Codex is right: +- State what needs to change +- Make the change to the plan file using the Edit tool (surgical edits only — + do not reorganize or reformat existing content) +- Log: "AGREE on GAP-N: {summary}. Updated {file} at {section}." + +**DISAGREE** — Codex is wrong: +- Cite specific evidence from the plan or codebase +- Explain why the concern does not apply in this context +- Log: "DISAGREE on GAP-N: {summary}. Evidence: {citation}." + +**PARTIAL** — Partly valid: +- Acknowledge the valid part, explain what does not apply +- Make targeted changes for the valid part only +- Log: "PARTIAL on GAP-N: {summary}. Addressed {X}, disagree on {Y}." + +Display each response: +``` +GAP-1 [Dim 4: Risk, 3/5]: No rollback strategy for database migration + → AGREE — Added rollback strategy in Phase 2 risk section. + +GAP-2 [Dim 7: Testing, 3/5]: No characterization tests for legacy code + → DISAGREE — Characterization tests specified in Phase 1 Step 3 + (see docs/specs/pricing-engine.md lines 45-52). +``` + +## Step 4: Check Exit Conditions + +After completing Step 3, check these conditions IN ORDER: + +1. **Score >= 36/40** → GREEN achieved. Go to Step 5. +2. **Max rounds reached** → Go to Step 5 with final score. +3. **No score improvement between rounds AND all dimensions >= 3/5** + → Convergence plateau. Go to Step 5. +4. **Any dimension at 1/5 or 2/5 persists after Round 2** + → Halt for human review. Display: "CRITICAL: Dimension {N} remains at {score}/5 + after 2 rounds. This requires human review before proceeding." + Go to Step 5. +5. **Budget checkpoint**: Check elapsed time. If < 3 minutes remain of the + 15-minute ceiling, force this to be the final round. Go to Step 5 after + this round completes. +6. **Total elapsed time > 15 minutes** → Abort. Go to Step 5 with partial report. + +**Model escalation**: If total score < 24/40 (RED) in Round 1 and `--model` was +not explicitly set, escalate to `gpt-5.4 -c model_reasoning_effort=high` for +Round 2+. Display: "Escalating to gpt-5.4 due to RED score ({score}/40)." + +If none of the exit conditions are met → go back to Step 1 for the next round. diff --git a/plugins/deepgrade/skills/documentation/SKILL.md b/plugins/deepgrade/skills/documentation/SKILL.md index 4a15eb7..0b828fa 100644 --- a/plugins/deepgrade/skills/documentation/SKILL.md +++ b/plugins/deepgrade/skills/documentation/SKILL.md @@ -21,7 +21,7 @@ With no plan named, use the standard locations only. ## Resolving template paths -Templates live beside this file. Read them via `${CLAUDE_SKILL_DIR}/resources/` +Templates live beside this file. Read them via `${CLAUDE_SKILL_DIR}/references/` — an installed plugin does not sit at a path this file can assume, and a relative read from the session's working directory resolves into the *user's project*, not the plugin. The table below uses repo-relative links for human readability; at @@ -43,12 +43,12 @@ Parse `$ARGUMENTS` to determine the document type and topic: | Subcommand | Template | Description | |------------|----------|-------------| -| `adr` | [resources/adr-template.md](resources/adr-template.md) | Architecture Decision Record | -| `brd` | [resources/brd-template.md](resources/brd-template.md) | Business Requirements Document | -| `prd` | [resources/prd-template.md](resources/prd-template.md) | Product Requirements Document | -| `readme` | [resources/readme-template.md](resources/readme-template.md) | Project README | -| `release-notes` | [resources/release-notes-template.md](resources/release-notes-template.md) | Release Notes / Changelog | -| `spec` | [resources/spec-template.md](resources/spec-template.md) | Technical Specification (extraction, migration, feature, infrastructure) | +| `adr` | [references/adr-template.md](references/adr-template.md) | Architecture Decision Record | +| `brd` | [references/brd-template.md](references/brd-template.md) | Business Requirements Document | +| `prd` | [references/prd-template.md](references/prd-template.md) | Product Requirements Document | +| `readme` | [references/readme-template.md](references/readme-template.md) | Project README | +| `release-notes` | [references/release-notes-template.md](references/release-notes-template.md) | Release Notes / Changelog | +| `spec` | [references/spec-template.md](references/spec-template.md) | Technical Specification (extraction, migration, feature, infrastructure) | ### Dispatch Logic @@ -163,7 +163,7 @@ This ensures documents don't exist in isolation. Every doc links to related docs ### Execution -Read the selected `resources/*.md` template file and follow its instructions exactly, +Read the selected `references/*.md` template file and follow its instructions exactly, treating the remaining arguments as the topic (feature, domain, or project name) wherever the template shows its dollar-one placeholder. ### Command Reference Rule diff --git a/plugins/deepgrade/skills/documentation/resources/adr-template.md b/plugins/deepgrade/skills/documentation/references/adr-template.md similarity index 100% rename from plugins/deepgrade/skills/documentation/resources/adr-template.md rename to plugins/deepgrade/skills/documentation/references/adr-template.md diff --git a/plugins/deepgrade/skills/documentation/resources/brd-template.md b/plugins/deepgrade/skills/documentation/references/brd-template.md similarity index 100% rename from plugins/deepgrade/skills/documentation/resources/brd-template.md rename to plugins/deepgrade/skills/documentation/references/brd-template.md diff --git a/plugins/deepgrade/skills/documentation/resources/prd-template.md b/plugins/deepgrade/skills/documentation/references/prd-template.md similarity index 100% rename from plugins/deepgrade/skills/documentation/resources/prd-template.md rename to plugins/deepgrade/skills/documentation/references/prd-template.md diff --git a/plugins/deepgrade/skills/documentation/resources/readme-template.md b/plugins/deepgrade/skills/documentation/references/readme-template.md similarity index 100% rename from plugins/deepgrade/skills/documentation/resources/readme-template.md rename to plugins/deepgrade/skills/documentation/references/readme-template.md diff --git a/plugins/deepgrade/skills/documentation/resources/release-notes-template.md b/plugins/deepgrade/skills/documentation/references/release-notes-template.md similarity index 100% rename from plugins/deepgrade/skills/documentation/resources/release-notes-template.md rename to plugins/deepgrade/skills/documentation/references/release-notes-template.md diff --git a/plugins/deepgrade/skills/documentation/resources/spec-template.md b/plugins/deepgrade/skills/documentation/references/spec-template.md similarity index 100% rename from plugins/deepgrade/skills/documentation/resources/spec-template.md rename to plugins/deepgrade/skills/documentation/references/spec-template.md diff --git a/plugins/deepgrade/skills/troubleshoot/SKILL.md b/plugins/deepgrade/skills/troubleshoot/SKILL.md new file mode 100644 index 0000000..d5fb62b --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/SKILL.md @@ -0,0 +1,318 @@ +--- +name: troubleshoot +description: AI-guided troubleshooting using the 4-phase systematic debugging framework with severity-driven incident triage and containment. Enforces root cause investigation before suggesting fixes. For SEV1/SEV2 production incidents, temporary containment is allowed before investigation. Logs every step, builds a project knowledge base. Auto-links to active plan. Pass an error message, issue description, or just say what broke. Use when the user reports an error, a broken feature, or a production incident. +argument-hint: "[error message or issue description] [--plan plan-name] [--severity SEV1|SEV2|SEV3|SEV4]" +allowed-tools: Read, Write, Grep, Glob, Bash, Task +--- + + + +You are a systematic debugging specialist. You follow the 4-phase debugging +framework used by senior engineers. You NEVER suggest fixes before understanding +the root cause. + +THE IRON LAW: + NO PERMANENT FIXES WITHOUT ROOT CAUSE INVESTIGATION FIRST. + If you haven't completed Phase 1, you CANNOT propose permanent fixes. + Suggesting a fix without evidence from THIS codebase is a failure. + + For SEV1/SEV2 incidents, TEMPORARY CONTAINMENT mitigations are allowed + before Phase 1 only to restore service safely. Containment is not closure; + root cause investigation still remains mandatory. + + Containment means: rollback, feature-flag disable, config revert, traffic + shedding, failover. NOT refactors, NOT speculative code edits, NOT "ship + a guess and move on." + +You adapt your approach based on what the user gives you: +- Error message -> search codebase + check git history + reproduce +- Vague description -> ask diagnostic questions to categorize the bug +- Specific behavior -> targeted investigation of that code path +- Production fire (SEV1/SEV2) -> triage, contain, THEN investigate + +You LOG every step in real time so debugging knowledge is preserved. + + + +AI generates explanations that sound convincing because they match patterns +across millions of codebases. But THIS bug exists in THIS specific context, +with THIS specific state, data, and interaction history. + +Pattern matching across codebases is NOT the same as causal reasoning within +one codebase. When you suggest a hypothesis, you MUST tie it to evidence +found in THIS codebase, not general programming knowledge. + +If you catch yourself suggesting a fix based on "this usually happens +because..." without reading the actual code, STOP and say: +"I'm suggesting this based on general patterns, not evidence from your code. +Let me read the actual files first." + + + +## Timeline Logging + +Record `T_START` NOW — before plan detection, KB check, or any other work. +The pre-investigation steps are part of the timeline. Record a raw timestamp +at each phase boundary using ISO 8601 format. These are the SOURCE DATA for +duration metrics in the log. + +``` +T_START: {timestamp when troubleshooting begins — before plan detection} +T_TRIAGED: {timestamp when severity is classified} +T_CONTAINED: {timestamp when containment is applied, or "N/A" if SEV3/SEV4 or no mitigation} +T_CATEGORIZED: {timestamp when bug category is determined} +T_REPRODUCED: {timestamp when issue is reproduced, or "N/A" if not reproducible} +T_ISSUE_LOCATED: {timestamp when Phase 1 completes — issue located to file/function} +T_ESCALATED: {timestamp when multi-agent mode is entered, or "N/A" if single-agent} +T_SYNTHESIS_COMPLETE: {timestamp when multi-agent orchestrator synthesis completes, or "N/A"} +T_HYPOTHESIS: {timestamp when Phase 3 completes — root cause hypothesis confirmed} +T_FIX_VERIFIED: {timestamp when fix is verified — tests pass, no regressions} +T_GUARDRAILS: {timestamp when guardrail evaluation completes} +T_LOGGED: {timestamp when log and KB are written} +``` + +For dead ends, log the timestamp when you abandoned the hypothesis: +``` +T_DEAD_END_1: {timestamp} — {hypothesis that was disproved} +``` + +Do NOT calculate durations inline. Record raw timestamps only. +Duration metrics are derived in the log template (Step 5). + + + +Auto-detect the active plan: + +```bash +LATEST_PLAN=$(ls -td docs/plans/*/ 2>/dev/null | head -1) +if [ -n "$LATEST_PLAN" ]; then + PLAN_NAME=$(basename "$LATEST_PLAN") + if [ -f "$LATEST_PLAN/status.json" ]; then + # Interpreter name differs by host (python3 on most Linux, python on many + # Windows installs), so resolve it, then fall back to grep. The path is passed + # as argv rather than interpolated into the source, which a path containing a + # quote used to break. + PY="" + command -v python3 >/dev/null 2>&1 && PY=python3 + [ -z "$PY" ] && command -v python >/dev/null 2>&1 && PY=python + if [ -n "$PY" ]; then + PHASE=$("$PY" -c " +import json, sys +with open(sys.argv[1]) as f: + print(json.load(f).get('current_phase', 'unknown')) +" "${LATEST_PLAN}/status.json" 2>/dev/null) + fi + if [ -z "$PHASE" ]; then + PHASE=$(grep -o '"current_phase"[[:space:]]*:[[:space:]]*"[^"]*"' "${LATEST_PLAN}/status.json" 2>/dev/null \ + | head -1 | sed 's/.*"\([^"]*\)"$/\1/') + fi + [ -z "$PHASE" ] && PHASE="unknown" + fi +fi +``` + +If --plan specified: use that plan. +If auto-detected: ask "Link this to plan {name}? [Y/n]" +If no plan found: run standalone (log to docs/troubleshooting/). + + + +## THE 4 PHASES (must complete in order) + +The core debugging framework. SEV3/SEV4 enter here directly. +SEV1/SEV2 enter here after the Containment Gate. + +| Phase | Question | Success Criteria | Can Suggest Fix? | +|-------|----------|-----------------|-----------------| +| 1. Root Cause | WHAT happened and WHERE? | Can reproduce. Know which file/function. | NO | +| 2. Pattern | HOW does working code differ? | Found the difference between working and broken. | NO | +| 3. Hypothesis | WHY did it break? | Have ONE testable theory with evidence from THIS codebase. | NO | +| 4. Fix | What's the minimal change? | Failing test exists. Fix is focused. Full suite passes. | YES (finally) | + +You MUST complete each phase before moving to the next. + + + +## Section files: load on entry + +The incident pre-flow, Phases 1-4, multi-agent mode, and Step 5 live in one file +each under `${CLAUDE_SKILL_DIR}/phases/`. Skill content enters the conversation +once and is not re-read after auto-compaction, so a single long file silently +loses its later phases in exactly the long sessions a debugging run produces. +The split is the fix: each section is read when it is entered, so its +instructions are always the most recent thing in context. + +RULE: On entering a section, READ its file with the Read tool BEFORE doing any +work in that section. Re-read it after any compaction. Read only the current +section; do not read ahead. Files are one level deep and referenced only from +this table. + +| Order | Section | When | File | +|-------|---------|------|------| +| 0 | Incident pre-flow (severity triage, containment gate) | On intake, right after plan detection and before Step 0 | `${CLAUDE_SKILL_DIR}/phases/incident-preflow.md` | +| 1 | Phase 1: Root Cause Investigation | After Step 0 / 0.5 / 0.2 below | `${CLAUDE_SKILL_DIR}/phases/phase-1-root-cause.md` | +| 1b | Multi-agent mode | Only if the Phase 1 escalation check is confirmed; replaces Phases 2-3 | `${CLAUDE_SKILL_DIR}/phases/multi-agent-mode.md` | +| 2 | Phase 2: Pattern Analysis | Single-agent path after Phase 1 | `${CLAUDE_SKILL_DIR}/phases/phase-2-pattern-analysis.md` | +| 3 | Phase 3: Hypothesis and Testing | Single-agent path after Phase 2 | `${CLAUDE_SKILL_DIR}/phases/phase-3-hypothesis.md` | +| 4 | Phase 4: Fix (incl. guardrail evaluation) | After Phase 3, or after multi-agent synthesis | `${CLAUDE_SKILL_DIR}/phases/phase-4-fix.md` | +| 5 | Step 5: Log and Update Knowledge Base | After Phase 4 | `${CLAUDE_SKILL_DIR}/phases/step-5-knowledge-base.md` | + +Paths use forward slashes on every platform. If a section file cannot be read, +stop and report the path; do not improvise the section from memory. + + +## Step 0: Check Knowledge Base (Structured Correlation) + +Before ANY diagnosis, check if this issue matches a past incident. +Use multi-dimensional correlation when structured fields are available, +with keyword grep as fallback for older entries that lack them. + +### 0.1: Correlation Matching + +Read the knowledge base and score each past entry against the current issue +on these dimensions: + +| Dimension | Weight | Match Criteria | +|-----------|--------|---------------| +| **Error signature** | High | Same error message, exception type, or error code | +| **Service / module** | High | Same file, module, or service affected | +| **Bug category** | Medium | Same category (logic, boundary, data flow, etc.) | +| **Code path** | Medium | Same function or call chain involved | +| **Contributing factors** | Medium | Same contributing factors from postmortem | +| **Severity** | Low | Same severity level | + +Score each past entry: +``` +score = 0 +if error_signature matches: score += 30 +if same service/module: score += 25 +if same bug category: score += 15 +if same code path: score += 10 +if same contributing factors: score += 15 +if same severity: score += 5 +``` + +### 0.2: Correlation Actions + +| Correlation | Score | Action | +|------------|-------|--------| +| **HIGH** | >= 50 | "This matches a previous incident: {title} ({date}). The fix was: {resolution}. Apply the same fix? [Y/n/investigate]" | +| **MEDIUM** | 30-49 | "This may be related to: {title}. Root cause was: {cause}. Check this path first? [Y/n]" | +| **LOW** | < 30 | No match surfaced. Proceed to Phase 1. | + +For HIGH matches, also check: was the previous fix sufficient? +If the same pattern recurred, say so (see Recurrence Detection in Step 5). + +### 0.3: Backward-Compatible Fallback + +If the knowledge base contains entries WITHOUT structured fields (older +format with only keyword-searchable text), fall back to keyword grep: + +```bash +# Fallback for unstructured KB entries +if [ -f "docs/troubleshooting/knowledge-base.md" ]; then + grep -i "{issue-keywords}" "docs/troubleshooting/knowledge-base.md" +fi + +# Check plan-specific troubleshooting logs +if [ -d "${LATEST_PLAN}/troubleshooting/" ]; then + grep -ri "{issue-keywords}" "${LATEST_PLAN}/troubleshooting/" +fi +``` + +Keyword grep results are treated as LOW correlation — they surface context +but do not trigger HIGH/MEDIUM actions. Structured correlation always takes +priority when available. + +If no match from either method: proceed to Phase 1. + +## Step 0.5: Check Impact Review Context + +If the active plan has an impact review, read it: + +```bash +# Check for impact review in active plan folder first, then docs/audit/ +IMPACT_FILE="" +if [ -n "$LATEST_PLAN" ] && [ -f "$LATEST_PLAN/impact-review.md" ]; then + IMPACT_FILE="$LATEST_PLAN/impact-review.md" +elif ls docs/audit/impact-review-*.md 2>/dev/null | head -1 > /dev/null; then + IMPACT_FILE=$(ls docs/audit/impact-review-*.md 2>/dev/null | head -1) +fi +if [ -n "$IMPACT_FILE" ]; then + grep -i "{issue-keywords}" "$IMPACT_FILE" +fi +``` + +If the impact review flagged integration edges related to this area, +say: "The Impact Review flagged this area: {finding}. This may be +related. I'll check this path first." + +### Step 0.2: External Context (if MCP search tools available) + +After checking the local knowledge base, search external sources for known issues. +This step is OPTIONAL — skip entirely if no MCP search tools are available. + +IF ref_search_documentation is available: + Search for the error message or symptom in the framework's official docs. + Use a complete question: ref_search_documentation("NullReferenceException in ASP.NET middleware pipeline") + Look for: known bugs, breaking changes, migration notes, configuration requirements. + +IF web_search_exa is available: + Search for the exact error message on GitHub issues and Stack Overflow. + Use web_search_exa for error signatures, then web_fetch_exa to read the one + issue thread that actually matches. + Look for: resolved issues with the same error, workarounds, version-specific bugs. + +Mark external findings with evidence tier: + - Framework docs match: "A-HIGH: confirmed in {framework} docs v{version}" + - GitHub issue match: "B-MEDIUM: matches GitHub issue #{number}, {status}" + - SO answer match: "B-MEDIUM: matches SO answer with {votes} votes" + +IMPORTANT: External matches inform hypothesis formation but do NOT replace +reading THIS codebase's actual code. The plausible hypothesis warning still +applies — always verify external findings against the local implementation +before forming a hypothesis. + +IF no MCP search tools available: + Skip this step entirely. Proceed to Phase 1 with codebase-only investigation. + + +## Phases 1-4 and Step 5 + +Continue with the section files in the dispatch table above, starting with +`${CLAUDE_SKILL_DIR}/phases/phase-1-root-cause.md`. + + + + + +STOP and follow the 4-phase process if you catch yourself: +- Proposing a fix before reading the actual code +- Suggesting "this usually happens because..." without THIS repo's evidence +- Attempting multiple fixes simultaneously +- Skipping reproduction ("just try this fix") +- Ignoring error messages or warnings +- Assuming the bug is in the most recently changed file without verifying +- Bundling multiple changes into one test + + + +- Follow the 4 phases IN ORDER. No skipping. +- Do NOT suggest fixes during Phases 1-3. Only in Phase 4. +- Log EVERY step in real time, including dead ends. +- Do NOT modify source code without explicit user approval. +- Tie every hypothesis to evidence from THIS codebase. +- If outside your knowledge, say so and suggest escalation. +- Keep knowledge base entries SHORT. +- Redact secrets, credentials, or PII from all logs. +- If the same issue appears twice, flag the pattern. + + + +/deepgrade:codex-challenge, /deepgrade:documentation, /deepgrade:help, /deepgrade:plan, +/deepgrade:plan-export, /deepgrade:plan-status, /deepgrade:quick-audit, /deepgrade:quick-cleanup, +/deepgrade:quick-plan, /deepgrade:troubleshoot, /deepgrade-audit:codebase-audit, +/deepgrade-audit:codebase-characterize, /deepgrade-audit:codebase-delta, +/deepgrade-audit:codebase-gates, /deepgrade-audit:codebase-security, +/deepgrade-readiness:readiness-generate, /deepgrade-readiness:readiness-scan + diff --git a/plugins/deepgrade/skills/troubleshoot/phases/incident-preflow.md b/plugins/deepgrade/skills/troubleshoot/phases/incident-preflow.md new file mode 100644 index 0000000..ec296e7 --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/phases/incident-preflow.md @@ -0,0 +1,77 @@ +# Incident Pre-Flow (Phase 0: Severity / Triage and Containment Gate) + +Phase file for /deepgrade:troubleshoot, loaded by SKILL.md on entry. + + +## INCIDENT PRE-FLOW (conditional, before the 4 phases) + +For every issue, classify severity on intake. This takes 30 seconds and +determines whether the issue enters the containment gate or goes straight +to Phase 1. + +### Phase 0: Severity / Triage + +Classify the issue using these signals. If --severity is passed, use that. +Otherwise, infer from the user's language: + +| Severity | Definition | Containment? | Route | +|----------|-----------|-------------|-------| +| **SEV1** | Production down, data loss, security breach, revenue impact | YES — mandatory | Containment Gate → Phase 1 | +| **SEV2** | Major feature broken, significant user impact, degraded service | YES — recommended | Containment Gate → Phase 1 | +| **SEV3** | Minor feature broken, workaround exists, limited user impact | No | Straight to Phase 1 | +| **SEV4** | Cosmetic, minor annoyance, tech debt discovered | No | Straight to Phase 1 | + +Auto-classification signals: + +| Signal in user's report | Likely Severity | +|------------------------|----------------| +| "Production is down", "users can't access", "losing money", "security breach" | SEV1 | +| "Not working", "broken for everyone", "errors in production", "data is wrong" | SEV2 | +| "Something's wrong with", "intermittent", "works but slowly", "edge case" | SEV3 | +| "I noticed", "minor issue", "when you get a chance", "cosmetic" | SEV4 | + +ALWAYS confirm: "I'm classifying this as **SEV{N}** based on {signal}. +Adjust? [1/2/3/4/keep]" + +Severity can ESCALATE during investigation (never downgrade without resolution): +- Blast radius larger than thought → escalate +- Data integrity affected → escalate to SEV1 +- Security implications discovered → escalate to SEV1 + +Record `T_TRIAGED` after classification. + +### Containment Gate (SEV1/SEV2 only) + +SEV3/SEV4: skip this gate entirely. Go straight to Phase 1. + +For SEV1/SEV2, assess whether a quick, safe mitigation can restore service +BEFORE spending time on root cause investigation. + +**OODA loop (Observe-Orient-Decide-Act):** + +1. **Observe:** What are the symptoms right now? +2. **Orient:** What changed recently? (last deploy, config change, traffic spike) +3. **Decide:** What's the fastest SAFE mitigation from this list? + +| Mitigation | Speed | Risk | When to Use | +|-----------|-------|------|------------| +| Rollback last deploy | Fast | Low | Symptoms started after deploy | +| Toggle feature flag | Fast | Low | New feature is the likely culprit | +| Revert config change | Fast | Low | Config was recently modified | +| Scale up / restart | Medium | Low | Resource exhaustion, memory leak | +| Block bad traffic | Medium | Medium | Attack or specific client causing load | +| Failover to secondary | Slow | Medium | Primary service unrecoverable | + +4. **Act:** Apply the containment. Verify service is restored. + +"Service restored via {mitigation}. Containment is not closure — proceeding +to Phase 1 for root cause investigation." + +If no safe containment is available: "No obvious safe mitigation. Proceeding +directly to Phase 1 investigation." + +Record `T_CONTAINED` after containment (or "N/A" if skipped or no mitigation available). + +LOG the containment action, what was mitigated, and any temporary tradeoffs +(e.g., "new feature disabled until permanent fix"). + diff --git a/plugins/deepgrade/skills/troubleshoot/phases/multi-agent-mode.md b/plugins/deepgrade/skills/troubleshoot/phases/multi-agent-mode.md new file mode 100644 index 0000000..a807c77 --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/phases/multi-agent-mode.md @@ -0,0 +1,82 @@ +# Multi-Agent Mode (only when escalated from Phase 1) + +Phase file for /deepgrade:troubleshoot, loaded by SKILL.md on entry. + +## MULTI-AGENT MODE (only when escalated from Phase 1) + +When escalated, the orchestrator spawns up to 4 specialist subagents that +investigate in parallel. The orchestrator then cross-references their findings +to form a hypothesis. + +### Specialist Agents + +Spawn ONLY the agents relevant to this bug (not all 4 every time): + +**Code Tracer** (spawn when: bug involves code logic or data flow) +``` +Task: Read the code path from entry point to error. +- Trace the function call chain +- Identify where data transforms from correct to incorrect +- Check for assumptions that might not hold +- Report: which function, which line, what the data looks like +``` + +**Git Historian** (spawn when: "this used to work" or regression suspected) +``` +Task: Check what changed and when. +- git log for recent changes to affected files +- git diff to identify exact changes +- git bisect if regression window is known +- Report: which commit, who changed it, what changed +``` + +**Data Inspector** (spawn when: bug involves wrong data, missing records, config) +``` +Task: Check the data state and configuration. +- Query the database for the specific records involved +- Check config files for environment-specific values +- Check feature flags, environment variables +- Report: what the data actually is vs what it should be +``` + +**Integration Checker** (spawn when: bug crosses component boundaries) +``` +Task: Test the boundaries between components. +- Check API request/response at each boundary +- Verify auth tokens, headers, content types +- Check for schema mismatches between caller and callee +- Report: where the contract breaks between components +``` + +### Orchestrator Synthesis + +After specialists report back, the orchestrator: + +1. CROSS-REFERENCE: Where do the findings agree? Where do they conflict? + "Code Tracer found the function expects a string. Data Inspector found + the database returns an integer. Git Historian shows the column type + changed 3 commits ago. These findings converge on the same root cause." + +2. IDENTIFY CONFLICTS: If findings contradict, investigate the gap. + "Code Tracer says the function is never called. But Integration Checker + found it IS called via the webhook handler. Let me check that path." + +3. FORM HYPOTHESIS: Based on converged evidence from multiple agents. + "Root cause: the migration in commit {hash} changed the column type + from varchar to int, but the business logic still casts to string. + This causes the data flow error found by Code Tracer." + +4. PROCEED TO PHASE 4 (Fix): With the synthesized root cause. + The multi-agent investigation replaces Phases 2 and 3 because: + - Pattern Analysis was done by Code Tracer (found working vs broken) + - Hypothesis was formed by cross-referencing all agent findings + +LOG all specialist findings and the synthesis in the troubleshooting log. +Each specialist's full report is captured under its own heading in the log. +The orchestrator's synthesis (agreements, conflicts, hypothesis) is logged +as a separate section. Dead ends from specialists are logged too. + +After synthesis, proceed directly to Phase 4 (Fix) and then Step 5 (Log). + +--- + diff --git a/plugins/deepgrade/skills/troubleshoot/phases/phase-1-root-cause.md b/plugins/deepgrade/skills/troubleshoot/phases/phase-1-root-cause.md new file mode 100644 index 0000000..d00c3a5 --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/phases/phase-1-root-cause.md @@ -0,0 +1,90 @@ +# Phase 1: Root Cause Investigation + +Phase file for /deepgrade:troubleshoot, loaded by SKILL.md on entry. + +## Phase 1: ROOT CAUSE INVESTIGATION + +### Step 1.1: Categorize the Bug Type + +Based on the user's description, categorize: + +| Category | Signals | First Check | +|----------|---------|------------| +| Logic | Wrong output, wrong behavior | Read the function, trace the logic | +| Boundary | Works sometimes, fails on edge cases | Check input ranges, null, empty | +| Error Handling | Crashes, unhandled exception | Read the catch/finally blocks | +| Data Flow | Data is wrong downstream | Trace from source to where it's wrong | +| Integration | Works alone, fails with other components | Check the boundaries between components | +| Timing | Intermittent, works then doesn't | Check async, race conditions, order | + +Tell the user: "This looks like a {category} issue. Here's my approach..." + +### Step 1.2: Check Recent Changes (ALWAYS do this first) + +```bash +echo "=== Recent commits ===" +git log --oneline -10 2>/dev/null + +echo "=== Files changed in last 3 commits ===" +git diff --name-only HEAD~3 2>/dev/null + +echo "=== Uncommitted changes ===" +git status --short 2>/dev/null +``` + +"These files changed recently: {list}. The bug may be in one of these changes." + +### Step 1.3: Reproduce the Issue + +Ask: "Can you reproduce this consistently? What are the exact steps?" + +If yes: "Good. A reproducible bug is a solvable bug." +If no: "Intermittent issues are often timing-related. Let me check for +race conditions, shared state, or order-dependent behavior." + +### Step 1.4: Read the Actual Code + +Read the files involved. Don't guess. + +```bash +grep -rn "{error-or-function}" --include="*.cs" --include="*.vb" \ + --include="*.ts" --include="*.tsx" . 2>/dev/null | grep -v node_modules | head -20 +``` + +Read the files. Understand the data flow. Note what you see. + +### Step 1.5: Gather Evidence + +For multi-component issues, trace data across boundaries: +- What goes IN to the function? +- What comes OUT? +- Where does the data change from correct to incorrect? + +LOG: "Phase 1 complete. The issue is in {file}:{function}. +The data is correct at {point A} but wrong at {point B}." + +### Step 1.6: Escalation Check (Auto-Detect Multi-Agent Need) + +After Phase 1, assess whether this bug needs parallel investigation. + +ESCALATION CRITERIA (if ANY are true, offer multi-agent): +- Bug spans 3+ layers (frontend + backend + database, or UI + business logic + config) +- 2+ competing hypotheses exist and each requires reading different codebases +- Investigation requires holding 4+ mental contexts simultaneously +- The same investigation area keeps branching (checking one thing reveals 3 more things) +- Serial investigation would take 15+ minutes of context-switching + +If escalation criteria are NOT met (most bugs): + "This looks like a straightforward {category} issue in {file}. Continuing + single-agent investigation." + -> Proceed to Phase 2 (Pattern Analysis) normally. + +If escalation criteria ARE met: + "This looks like a cross-layer issue spanning {areas}. I can investigate + faster by running parallel agents. Escalate to multi-agent mode? [Y/n]" + + If user confirms, switch to MULTI-AGENT MODE (see below). + If user declines, continue single-agent through Phases 2-4. + +--- + diff --git a/plugins/deepgrade/skills/troubleshoot/phases/phase-2-pattern-analysis.md b/plugins/deepgrade/skills/troubleshoot/phases/phase-2-pattern-analysis.md new file mode 100644 index 0000000..d57b140 --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/phases/phase-2-pattern-analysis.md @@ -0,0 +1,28 @@ +# Phase 2: Pattern Analysis (single-agent path) + +Phase file for /deepgrade:troubleshoot, loaded by SKILL.md on entry. + +## Phase 2: PATTERN ANALYSIS (single-agent path) + +### Step 2.1: Find Working Examples + +```bash +grep -rn "{similar-pattern}" --include="*.cs" --include="*.vb" \ + . 2>/dev/null | grep -v node_modules | head -10 +``` + +### Step 2.2: Compare Working vs Broken + +"Working code does {X}. Broken code does {Y}. The difference is {Z}." + +### Step 2.3: Check Dependencies and Assumptions + +What does the broken code ASSUME that might not be true? +- Does it assume data exists? (null check missing) +- Does it assume order? (async timing) +- Does it assume format? (string vs number) +- Does it assume config? (environment-specific) + +LOG: "Phase 2 complete. Working code does {X} differently. +The broken code assumes {Y} which is not true when {Z}." + diff --git a/plugins/deepgrade/skills/troubleshoot/phases/phase-3-hypothesis.md b/plugins/deepgrade/skills/troubleshoot/phases/phase-3-hypothesis.md new file mode 100644 index 0000000..ce2d237 --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/phases/phase-3-hypothesis.md @@ -0,0 +1,36 @@ +# Phase 3: Hypothesis and Testing + +Phase file for /deepgrade:troubleshoot, loaded by SKILL.md on entry. + +## Phase 3: HYPOTHESIS AND TESTING + +### Step 3.1: Form ONE Hypothesis + +"Based on the evidence: the bug is caused by {specific cause} +because {evidence from THIS codebase}." + +The hypothesis MUST: +- Be specific (not "something is wrong with the data") +- Be testable (we can verify with a specific check) +- Reference evidence from Phase 1 and 2 (not general patterns) + +### Step 3.2: Five Whys (for stubborn bugs) + +If the first hypothesis doesn't hold, go deeper: +1. Why did it fail? -> "Because the receipt string was null" +2. Why was it null? -> "Because the lookup returned no result" +3. Why no result? -> "Because the key was French but the table has English keys" +4. Why English keys? -> "Because the migration only loaded English strings" +5. Why only English? -> "Because the French resource file wasn't in the build" + +ROOT CAUSE: French resource file missing from build configuration. + +### Step 3.3: Test Minimally + +Change ONE variable at a time. Never bundle changes. + +If confirmed: "Root cause confirmed: {cause}. Ready to fix." +If disproved: "That wasn't it. New hypothesis based on what we learned." + +LOG: "Phase 3 complete. Root cause: {cause}. Evidence: {what we verified}." + diff --git a/plugins/deepgrade/skills/troubleshoot/phases/phase-4-fix.md b/plugins/deepgrade/skills/troubleshoot/phases/phase-4-fix.md new file mode 100644 index 0000000..981281d --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/phases/phase-4-fix.md @@ -0,0 +1,111 @@ +# Phase 4: Fix + +Phase file for /deepgrade:troubleshoot, loaded by SKILL.md on entry. + +## Contents + +- Step 4.1: Create a Failing Test First +- Step 4.2: Implement Focused Fix +- Step 4.3: Verify +- Step 4.4: Git Bisect (for regression bugs) +- Step 4.5: Guardrail Evaluation (4.5.1 Inspect Each Guardrail Layer, 4.5.2 Classify Why Each Relevant Guardrail Missed, 4.5.3 Generate Recommended Actions) + +## Phase 4: FIX (only NOW can you suggest a fix) + +### Step 4.1: Create a Failing Test First + +"Before fixing, let's prove the bug exists with a test. [Y/n]" + +The test should: +- Set up conditions that trigger the bug +- Call the function that fails +- Assert the CORRECT behavior (which currently fails) + +### Step 4.2: Implement Focused Fix + +The fix should be: +- Single, focused change (not a refactor) +- As small as possible +- Directly addressing the root cause (not the symptom) + +"Here's the fix: {description}. Apply it? [Y/n]" + +### Step 4.3: Verify + +```bash +# Run the failing test (should now pass) +# Run the full test suite (no regressions) +``` + +"Fix verified. Failing test passes. No regressions." + +### Step 4.4: Git Bisect (for regression bugs) + +If the bug worked before and now doesn't: + +```bash +# Find which commit introduced the regression +git bisect start HEAD {last-known-good-commit} +git bisect run {test-command} +git bisect log +``` + +"The regression was introduced in commit {hash}: {message}." + +### Step 4.5: Guardrail Evaluation (Why didn't safeguards catch this?) + +After the fix is verified, inspect the ACTUAL guardrail configuration to understand +why this bug reached the environment where it was found. This step generates +RECOMMENDED follow-up actions — it does NOT automatically apply additional edits. +The fix is already verified; this is analysis, not more fixing. + +#### 4.5.1: Inspect Each Guardrail Layer + +Read the actual config files in THIS repo. For each guardrail, answer: could it +have caught this bug before it reached the user? + +| Guardrail | Check | Files to Inspect | +|-----------|-------|-----------------| +| Unit tests | Does a test exist for the buggy function? | Test directory, test runner config | +| Integration tests | Does a test cover the interaction that broke? | Integration test files | +| Type system | Could stricter types have prevented this? | tsconfig.json, compiler options, type definitions | +| Linter rules | Is there a rule that should catch this pattern? | .eslintrc, linter configs | +| CI pipeline | Does CI run the tests that would catch this? | .github/workflows/, CI config | +| Pre-commit hooks | Would a hook have caught this locally? | .husky/, hooks config | +| Runtime validation | Should input validation have rejected the bad data? | Validation middleware, schema definitions | + +Skip guardrails that clearly don't apply (e.g., don't check linter rules for a +data corruption bug). Only inspect what's relevant to THIS bug's category. + +#### 4.5.2: Classify Why Each Relevant Guardrail Missed + +For each guardrail that SHOULD have caught the bug, classify WHY it missed. +Use machine-friendly tokens in the format `{guardrail-type}:{classification}`: + +| Classification | Token | Meaning | Action | +|---------------|-------|---------|--------| +| Not present | `not-present` | No test/rule exists for this scenario | Write it | +| Present but insufficient | `insufficient` | Test exists but doesn't cover this case | Expand coverage | +| Present but disabled | `disabled` | Rule exists but is disabled or skipped | Re-enable, understand why | +| Present but wrong | `wrong` | Test asserts the wrong thing | Fix the assertion | +| Present and passed | `wrong-layer` | Test ran but bug is at a different layer | Add coverage at correct layer | +| Not applicable | `n-a` | No reasonable guardrail could catch this | Document as accepted risk | + +Examples: `unit-tests:not-present`, `ci:insufficient`, `linter:disabled`, `types:n-a` + +#### 4.5.3: Generate Recommended Actions + +For each missed guardrail, produce ONE specific, actionable recommendation. +These are suggestions for the user, not automatic edits. + +"Guardrail evaluation: +1. `{type}:{classification}` — {specific finding}. + **Recommended action:** {concrete change with file paths}. +2. ..." + +Do NOT say "add more tests." Say "add a test for {function} that covers the +{scenario} path in {file}:{line}." + +LOG: "Guardrail evaluation complete. {N} guardrails inspected, {M} gaps found. +{list of type:classification tokens}." + diff --git a/plugins/deepgrade/skills/troubleshoot/phases/step-5-knowledge-base.md b/plugins/deepgrade/skills/troubleshoot/phases/step-5-knowledge-base.md new file mode 100644 index 0000000..083170f --- /dev/null +++ b/plugins/deepgrade/skills/troubleshoot/phases/step-5-knowledge-base.md @@ -0,0 +1,194 @@ +# Step 5: Log and Update Knowledge Base + +Phase file for /deepgrade:troubleshoot, loaded by SKILL.md on entry. + +## Contents + +- Create Troubleshooting Log (location rules and the full log template: Timeline, Duration Metrics, Phase sections, Guardrail Evaluation, Prevention) +- Update Knowledge Base (KB entry template) +- Update Plan Manifest +- Detect Patterns +- Detect Guardrail Patterns +- Detect Recurrence (Correlation-Driven) +- Flag Impact Review Gaps + +## Step 5: Log and Update Knowledge Base + +### Create Troubleshooting Log + +Location: +- Plan-linked: `docs/plans/{date}-{name}/troubleshooting/YYYY-MM-DD-{issue-slug}.md` +- Standalone: `docs/troubleshooting/YYYY-MM-DD-{issue-slug}.md` + +```markdown +# Troubleshooting: {Issue Title} + +**Date:** {date} +**Severity:** SEV{N} +**Plan:** {plan name or "standalone"} +**Bug Category:** {logic | boundary | error handling | data flow | integration | timing} +**Status:** {investigating | resolved | workaround | escalated} +**Containment:** {mitigation applied, or "N/A" if SEV3/SEV4 or none needed} +**Resolution:** {summary once resolved} + +## Timeline +| Timestamp | Event | +|-----------|-------| +| {T_START} | Troubleshooting started (before plan detection) | +| {T_TRIAGED} | Severity classified as SEV{N} | +| {T_CONTAINED} | Containment applied: {mitigation} (or N/A) | +| {T_CATEGORIZED} | Bug categorized as {category} | +| {T_REPRODUCED} | Issue reproduced (or N/A) | +| {T_ISSUE_LOCATED} | Phase 1 complete: issue located in {file}:{function} | +| {T_ESCALATED} | Multi-agent mode entered (or N/A if single-agent) | +| {T_DEAD_END_N} | Dead end: {hypothesis disproved} | +| {T_SYNTHESIS_COMPLETE} | Multi-agent synthesis complete (or N/A if single-agent) | +| {T_HYPOTHESIS} | Phase 3 complete: root cause hypothesis confirmed (N/A if multi-agent — use T_SYNTHESIS_COMPLETE) | +| {T_FIX_VERIFIED} | Fix verified, tests passing, no regressions | +| {T_GUARDRAILS} | Guardrail evaluation complete | +| {T_LOGGED} | Log and KB updated | + +## Duration Metrics +Derived from raw timestamps above. Do not estimate — calculate from the timeline. + +- **Total time:** T_LOGGED - T_START +- **Time to issue located:** T_ISSUE_LOCATED - T_START +- **Time to root cause:** T_HYPOTHESIS - T_ISSUE_LOCATED (single-agent) or T_SYNTHESIS_COMPLETE - T_ESCALATED (multi-agent) +- **Time to verified fix:** T_FIX_VERIFIED - T_HYPOTHESIS (single-agent) or T_FIX_VERIFIED - T_SYNTHESIS_COMPLETE (multi-agent) +- **Dead end time:** sum of time spent on disproved hypotheses +- **Guardrail eval time:** T_GUARDRAILS - T_FIX_VERIFIED + +## Issue Description +{what the user reported} + +## Environment +- Branch: {git branch} +- Last commit: {git log --oneline -1} +- Recent changes: {git diff --name-only HEAD~3} + +## Phase 1: Root Cause Investigation +{what was found, which files, where data goes wrong} + +## Investigation Path +{which path was taken: single-agent or multi-agent} + +### If Single-Agent: +## Phase 2: Pattern Analysis +{how working code differs, what assumptions broke} + +## Phase 3: Hypothesis +{the theory, Five Whys if used, what was tested} + +### If Multi-Agent: +## Specialist Findings +### Code Tracer +{findings, files read, data flow traced} + +### Git Historian +{recent changes, commits, bisect results} + +### Data Inspector +{database state, config values, feature flags} + +### Integration Checker +{API boundaries tested, schema mismatches found} + +## Orchestrator Synthesis +{where findings agreed, where they conflicted, how hypothesis was formed} + +## Phase 4: Fix +{what was changed, the failing test, verification results} + +## Root Cause +{one sentence} + +## Guardrail Evaluation +| Guardrail | Classification | Finding | +|-----------|---------------|---------| +| {type} | {type}:{classification} | {specific finding} | + +### Recommended Actions +1. {concrete action with file paths} + +## Prevention +{1-2 sentence summary of architectural or process-level prevention beyond guardrails} +``` + +### Update Knowledge Base + +Append to `docs/troubleshooting/knowledge-base.md`: + +```markdown +### {Issue Title} ({date}) +**Severity:** SEV{N} +**Category:** {bug type} +**Service/Module:** {affected file, module, or service — e.g., src/payment/charge.ts} +**Error Signature:** {exact error message, exception type, or error code} +**Code Path:** {function call chain — e.g., checkout → payment → charge → processResponse} +**Containment:** {mitigation applied, or "N/A"} +**Symptom:** {what the user saw} +**Root Cause:** {what was actually wrong} +**Contributing Factors:** {conditions that made the incident possible — e.g., missing null check, no timeout config} +**Investigation:** {single-agent or multi-agent (which specialists)} +**Fix:** {what resolved it} +**Prevention:** {architectural or process-level prevention beyond guardrails} +**Guardrails missed:** {type:classification, type:classification} +**Guardrails added:** {what was added after this fix, or "none yet"} +**Five Whys depth:** {if used, how many levels deep} +**Recurrence count:** {how many times this pattern has occurred — start at 1} +**Related incidents:** {titles/dates of correlated past incidents, or "none"} +**Plan:** {plan name if linked} +**Log:** {path to full troubleshooting log} +``` + +Create the knowledge base file if it doesn't exist. + +### Update Plan Manifest + +If linked to a plan, add to manifest.md Project Documents table. + +### Detect Patterns + +If the knowledge base has 2+ entries with the same bug category: +"Pattern detected: this is the {N}th {category} bug in this project. +Consider adding a {check/test/gate} to catch these earlier." + +### Detect Guardrail Patterns + +If the knowledge base has 2+ entries with the same `{type}:{classification}` token: +"Guardrail pattern detected: this is the {N}th bug missed by +{guardrail type} ({classification}). This suggests a SYSTEMIC gap in +{guardrail type} coverage, not individual omissions. Consider a targeted +review of {guardrail type} configuration across the project." + +Key on the combined token (e.g., `unit-tests:not-present`), not classification +alone. "3 bugs missed by unit-tests:not-present" is actionable. +"3 bugs with not-present guardrails" across different guardrail types is not. + +### Detect Recurrence (Correlation-Driven) + +When a new KB entry has a HIGH correlation (>= 50) with a past entry: + +1. Increment the `Recurrence count` on both the new and matched entries. +2. Add bidirectional links in `Related incidents` on both entries. +3. If recurrence count reaches 3+: + +"RECURRENCE ALERT: This is the {N}th occurrence of this pattern: +- {date}: {title} — fixed with {fix} +- {date}: {title} — fixed with {fix} +- NOW: same pattern recurring + +Previous point fixes were INSUFFICIENT. This needs systemic remediation: +- A guardrail that prevents this CLASS of bug (test, lint rule, CI gate) +- Addressing the underlying architectural condition +- Escalating to tech debt remediation if root cause is known but unfixed" + +Recurrence detection keys on the correlation dimensions, not just bug category. +Two "data flow" bugs in different services are not recurrence. Two bugs with the +same service/module AND same error signature ARE recurrence. + +### Flag Impact Review Gaps + +If the issue reveals something the Impact Review missed: +"This wasn't caught by the Impact Review. Consider adding +'{check}' to future reviews for changes in this area." diff --git a/tests/codex-challenge-test.js b/tests/codex-challenge-test.js index fc9a14e..3111c98 100644 --- a/tests/codex-challenge-test.js +++ b/tests/codex-challenge-test.js @@ -383,20 +383,20 @@ test('JSON rejects out-of-range score', () => { // THE FIXTURE MUST BE BOUND TO THE SHIPPED ARTIFACT FIRST. Until this binding // existed, all five assertions below read only the fixture — a byte-duplicate with // nothing tying it to the schema Claude Code actually passes to `--output-schema`. -// Proven by mutation: gutting the SHIPPED schema in commands/codex-challenge.md +// Proven by mutation: gutting the SHIPPED schema in skills/codex-challenge/phases/output-schema.md // (maxItems 7 -> 99, additionalProperties false -> true, required reduced to // ["scores"], a dimension renamed) left 41/41 passing. A third of the suite was // validating a copy of itself. console.log('\n9. Schema File Validation'); const schema = JSON.parse(fs.readFileSync(path.join(FIXTURES_DIR, 'codex-review-schema.json'), 'utf-8')); -const SHIPPED_CMD = path.join(__dirname, '..', 'plugins', 'deepgrade', 'commands', 'codex-challenge.md'); +const SHIPPED_CMD = path.join(__dirname, '..', 'plugins', 'deepgrade', 'skills', 'codex-challenge', 'phases', 'output-schema.md'); -test('fixture schema is byte-equivalent to the schema shipped in commands/codex-challenge.md', () => { +test('fixture schema is byte-equivalent to the schema shipped in skills/codex-challenge/phases/output-schema.md', () => { const md = fs.readFileSync(SHIPPED_CMD, 'utf-8').replace(/\r\n/g, '\n'); // Take every ```json fence and keep the one that parses as this schema shape. const blocks = [...md.matchAll(/```json\n([\s\S]*?)```/g)].map((m) => m[1]); - assert(blocks.length > 0, 'no ```json block found in commands/codex-challenge.md'); + assert(blocks.length > 0, 'no ```json block found in skills/codex-challenge/phases/output-schema.md'); let shipped = null; for (const b of blocks) { try { diff --git a/tests/layer1-core.sh b/tests/layer1-core.sh index 407603a..cba5ec5 100644 --- a/tests/layer1-core.sh +++ b/tests/layer1-core.sh @@ -880,7 +880,7 @@ rm -f /tmp/dg_valid_agents.$$ echo "" echo "--- Documentation templates (F29/F31) ---" -TPL_DIR="skills/documentation/resources" +TPL_DIR="skills/documentation/references" if [ "$TPL_CHECK" -eq 0 ]; then : # the documentation templates ship with deepgrade only elif [ ! -d "$TPL_DIR" ]; then diff --git a/tests/mutation/wave5-guards.py b/tests/mutation/wave5-guards.py index 41ee4a0..c0402b2 100644 --- a/tests/mutation/wave5-guards.py +++ b/tests/mutation/wave5-guards.py @@ -22,7 +22,7 @@ FILES = ['plugins/deepgrade/commands/quick-cleanup.md', 'plugins/deepgrade/commands/plan-status.md', 'plugins/deepgrade/commands/plan-export.md', - 'plugins/deepgrade-readiness/commands/readiness-generate.md', 'plugins/deepgrade/commands/plan.md', 'plugins/deepgrade/commands/help.md', + 'plugins/deepgrade-readiness/commands/readiness-generate.md', 'plugins/deepgrade/skills/plan/phases/phase-2-research.md', 'plugins/deepgrade/commands/help.md', 'plugins/deepgrade-audit/commands/codebase-gates.md', 'plugins/deepgrade-audit/agents/gate-generator.md', 'plugins/deepgrade/skills/documentation/SKILL.md', 'plugins/deepgrade/skills/mcp-research/SKILL.md', 'README.md', 'CHANGELOG.md', @@ -33,7 +33,7 @@ 'plugins/deepgrade/commands/plan-status.md': 'PLANS_DIR="docs/plans"', 'plugins/deepgrade/commands/quick-cleanup.md': 'No source folder given', 'plugins/deepgrade-audit/agents/gate-generator.md': 'PowerShell variant', - 'plugins/deepgrade/commands/plan.md': 'deepgrade:mcp-research', + 'plugins/deepgrade/skills/plan/phases/phase-2-research.md': 'deepgrade:mcp-research', } FORBIDDEN = { 'README.md': 'deepgrade:doc adr topic', @@ -188,7 +188,7 @@ def append_block(f, body): ('Y8 PowerShell instruction inverted to a prohibition', l1, 'F08', lambda: patch('plugins/deepgrade-audit/agents/gate-generator.md', 'PowerShell variant', 'do not emit a PowerShell variant'), 'catch'), ('Y9 skill wired by PROSE only, not a namespaced name', l1, 'F28', - lambda: patch('plugins/deepgrade/commands/plan.md', '`deepgrade:mcp-research` skill', 'mcp-research skill'), 'catch'), + lambda: patch('plugins/deepgrade/skills/plan/phases/phase-2-research.md', '`deepgrade:mcp-research` skill', 'mcp-research skill'), 'catch'), # --- Codex F4: clauses that previously had no falsifying assertion --- ('Y10 drop the ${CLAUDE_PROJECT_DIR} resolution', l1, 'F10', lambda: patch('plugins/deepgrade/commands/plan-export.md', '${CLAUDE_PROJECT_DIR:-$OLDPWD}', '$PWD', 2), 'catch'), @@ -248,7 +248,7 @@ def append_block(f, body): ('W4 `tree` after an `if` keyword', l1, 'F15', lambda: append_block('plugins/deepgrade-readiness/commands/readiness-generate.md', 'if tree .; then :; fi'), 'catch'), ('W5 skill referenced only to FORBID it', l1, 'F28', - lambda: patch('plugins/deepgrade/commands/plan.md', 'see the `deepgrade:mcp-research` skill', + lambda: patch('plugins/deepgrade/skills/plan/phases/phase-2-research.md', 'see the `deepgrade:mcp-research` skill', 'never invoke `deepgrade:mcp-research`; it is deprecated'), 'catch'), ('W6 description states it should NOT load', l1, 'F28', lambda: patch('plugins/deepgrade/skills/mcp-research/SKILL.md', 'description: ',