From 76e54948a46453c8cc9bdc04ddbbc5c81743b9cd Mon Sep 17 00:00:00 2001 From: KKranthi6881 Date: Mon, 18 May 2026 11:35:23 -0500 Subject: [PATCH] Add spec retention guidance --- CHANGELOG.md | 6 ++ README.md | 39 ++++++++++++ docs/methodology.md | 10 ++- docs/spec-retention-and-repo-hygiene.md | 82 +++++++++++++++++++++++++ docs/team-onboarding-playbook.md | 6 ++ pyproject.toml | 17 ++++- src/dbt_specify/_version.py | 2 +- templates/CLAUDE.md.template | 8 +++ tests/test_docs.py | 35 +++++++++++ tests/test_init.py | 2 +- 10 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 docs/spec-retention-and-repo-hygiene.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c9978e..42c21b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.2.0] — 2026-05-18 + ### Added - Warehouse presets and guides for Redshift, Postgres, SQL Server, Azure SQL, MySQL, DuckDB, MotherDuck, and Athena. - `/dbt.implement-all` command template for sequential multi-task implementation with validation checkpoints and stop conditions. +- Enterprise spec retention and repo hygiene guidance for balancing decision records with repo noise. +- README enterprise adoption choices that surface workflow, retention, rollout, agent, warehouse, and + CI guidance before the detailed docs list. +- README spec folder structure guidance for large-team feature directories. ## [1.1.0] — 2026-05-18 diff --git a/README.md b/README.md index 5088fb2..c825fb6 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,22 @@ Idea -> spec.md -> plan.md -> tasks.md -> dbt changes -> CI report -> review The default is controlled autonomy. Agents can draft and implement, but humans approve the spec, the plan, and the final diff. +## Enterprise adoption choices + +Most teams should start with these defaults, then tighten or relax them as their governance needs +become clear. + +| Decision | Recommended default | Deep dive | +|---|---|---| +| Development workflow | Use the four-phase loop: specify, plan, tasks, implement. Keep human approval at the spec, plan, and final review gates. | [Methodology](docs/methodology.md) | +| Repo retention | Use balanced retention: merge `spec.md`, `plan.md`, and review/report evidence; keep `tasks.md` for complex, regulated, or high-risk work. | [Spec retention and repo hygiene](docs/spec-retention-and-repo-hygiene.md) | +| Brownfield rollout | Add the methodology layer first, capture existing conventions, and prove the flow on one low-risk dbt change before broad rollout. | [Brownfield onboarding](docs/brownfield-onboarding.md), [Team onboarding playbook](docs/team-onboarding-playbook.md) | +| Agent knowledge | Use dbt Labs skills for dbt mechanics. Use dbt-spec-kit skills and sub-agent roles for business meaning, planning, governance, and review evidence. | [Skills and sub-agents](docs/skills-and-sub-agents.md) | +| Warehouse guidance | Pick the closest warehouse preset for cost, materialization, SQL dialect, and governance guardrails. The project still runs through your normal dbt adapter and database connection. | [Warehouse guides](docs/warehouse-guides) | +| CI evidence | Start with local `validate` and `report`; promote `dbt-specify ci` when the team wants lifecycle checks to block PRs. | [Enterprise CI](docs/enterprise-ci.md) | + +The key repo hygiene rule: keep approved decision records, not raw agent scratch work. + ## Try it with jaffle-shop The fastest way to understand the workflow is to apply it to the upstream @@ -90,6 +106,28 @@ Running `dbt-specify init` in an existing dbt project creates: - `CLAUDE.md` or `CLAUDE.md.dbt-specify-suggested` - `specs/` for feature-level SDLC artifacts +## Spec folder structure + +Use one direct child folder under `specs/` for each meaningful dbt change: + +```text +specs/ + 001-core-customer-segmentation/ + spec.md + plan.md + tasks.md + review.md + findings.md +``` + +The folder name should be `--` when the team is large enough to need domain +visibility. Keep domain names in the slug, not as nested folders. `dbt-specify validate project` +treats each direct `specs/*/` child as a feature spec directory. + +`spec.md` is required. `plan.md` is added after spec approval. `tasks.md` is added after plan +approval. Review, governance, findings, and retro files are optional decision records governed by +your team's [spec retention policy](docs/spec-retention-and-repo-hygiene.md). + ## Skills vs sub-agents Skills are reusable knowledge. They teach an agent how to do a category of work better, such as @@ -138,6 +176,7 @@ Use `dbt-specify ci` when the lifecycle and dbt artifact checks should block a P - [Jaffle-shop AI SDLC walkthrough](docs/jaffle-shop-ai-sdlc-walkthrough.md) - [Team onboarding playbook](docs/team-onboarding-playbook.md) - [Methodology](docs/methodology.md) +- [Spec retention and repo hygiene](docs/spec-retention-and-repo-hygiene.md) - [Skills and sub-agents](docs/skills-and-sub-agents.md) - [Enterprise CI](docs/enterprise-ci.md) - [Brownfield onboarding](docs/brownfield-onboarding.md) diff --git a/docs/methodology.md b/docs/methodology.md index 30ae5d1..066ece2 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -10,6 +10,12 @@ Specify → Plan → Tasks → Implement Each phase has a **human checkpoint**. No phase skips, no auto-merge. +The four phases are required during development. After review, artifact retention is a team policy: +keep decision records in `main`, but do not keep every short-lived agent note. The default enterprise +recommendation is balanced retention: merge `spec.md`, `plan.md`, and review/report evidence, and +keep `tasks.md` only for complex, regulated, or high-risk work. See +[Spec retention and repo hygiene](spec-retention-and-repo-hygiene.md). + ## Phase 1: Specify **Input:** a feature description (one sentence or one paragraph). @@ -73,4 +79,6 @@ After ship, the engineer (or agent under direction) writes a retro covering: - What to change for next time (CLAUDE.md updates, new skills, eval fixtures) - Metrics: plan-phase time, implement-phase time, AI/human review findings, post-merge issues -Retros are filed as `specs/-/retro.md` or appended to `implementation-summary.md`. CLAUDE.md and skills updates are filed as separate PRs so the methodology layer keeps improving. +Retros are filed as `specs/-/retro.md` or appended to `implementation-summary.md` when +the team retention policy calls for them. CLAUDE.md and skills updates are filed as separate PRs so +the methodology layer keeps improving. diff --git a/docs/spec-retention-and-repo-hygiene.md b/docs/spec-retention-and-repo-hygiene.md new file mode 100644 index 0000000..d2ab00d --- /dev/null +++ b/docs/spec-retention-and-repo-hygiene.md @@ -0,0 +1,82 @@ +# Spec retention and repo hygiene + +dbt-spec-kit is designed to preserve decision records, not every temporary note an agent creates. +Teams should keep enough context in `main` to explain why a dbt change happened, while avoiding +long-lived noise from scratch work, abandoned drafts, and local logs. + +## Recommended default: balanced retention + +Use balanced retention unless your team has a stronger compliance requirement. + +Merge these artifacts for meaningful dbt changes: + +- `specs/-/spec.md` +- `specs/-/plan.md` +- review evidence, such as `review.md`, `implementation-summary.md`, or a `dbt-specify report` + summary attached to the PR + +Keep `tasks.md` when the change is complex, regulated, or high-risk. For small changes, `tasks.md` +can be omitted from the final merge or archived outside the long-lived repo history after the PR is +reviewed. + +## Supported retention models + +| Model | What gets merged | Best for | Tradeoff | +|---|---|---|---| +| Balanced default | `spec.md`, `plan.md`, review/report summary; `tasks.md` for complex work | Most enterprise teams | Keeps decisions without storing every agent step | +| Full audit | `spec.md`, `plan.md`, `tasks.md`, `review.md`, `retro.md` for every change | Regulated or audit-heavy teams | Strongest traceability, most repo noise | +| Lean | `spec.md` plus PR/report summary for low-risk work | Small teams and low-risk fixes | Lowest noise, less implementation history | + +## When to keep `tasks.md` + +Keep `tasks.md` in `main` when any of these are true: + +- PII, access, masking, or governance is involved +- finance, revenue, or metric definitions change +- semantic-layer objects, exposures, or contracts change +- multiple models, domains, or teams are touched +- the work needs auditability beyond the PR conversation + +For low-risk implementation fixes, it is acceptable for the task breakdown to exist only during the +feature branch and PR review. + +## What not to merge + +Do not commit these artifacts unless a reviewer explicitly asks for them as evidence: + +- raw agent scratch notes +- abandoned drafts +- temporary logs +- exploratory prompts +- local validation output files +- copied terminal output that is already summarized in PR evidence + +If an agent discovers useful context during implementation, promote it into `plan.md`, `review.md`, +`findings.md`, or `CLAUDE.md` instead of keeping raw scratch notes. + +## Suggested PR pattern + +Normal feature PR: + +```text +specs/001-customer-segmentation/spec.md +specs/001-customer-segmentation/plan.md +specs/001-customer-segmentation/review.md +models/marts/customers.sql +models/marts/customers.yml +``` + +High-risk PR: + +```text +specs/042-revenue-definition/spec.md +specs/042-revenue-definition/plan.md +specs/042-revenue-definition/tasks.md +specs/042-revenue-definition/governance-review.md +specs/042-revenue-definition/review.md +models/marts/finance/revenue.sql +models/marts/finance/revenue.yml +``` + +The four-phase workflow still happens during development. The retention model decides which +artifacts remain in `main` after the work is reviewed. diff --git a/docs/team-onboarding-playbook.md b/docs/team-onboarding-playbook.md index 58807d4..d81c017 100644 --- a/docs/team-onboarding-playbook.md +++ b/docs/team-onboarding-playbook.md @@ -26,6 +26,11 @@ Team rules: - one task per implementation pass by default - CI evidence is part of the PR - retros update team context instead of relying on memory +- choose a spec retention model before the first production PR + +Recommended default: balanced retention. Keep `spec.md`, `plan.md`, and review/report evidence in +`main`; keep `tasks.md` for complex, regulated, or high-risk work. See +[Spec retention and repo hygiene](spec-retention-and-repo-hygiene.md). ## Week 1: adopt without disrupting production @@ -74,6 +79,7 @@ Use this agenda for a team rollout meeting: - [ ] `CLAUDE.md` merged or reconciled - [ ] First spec approved by a human reviewer - [ ] First plan approved before implementation +- [ ] Spec retention model chosen - [ ] `dbt-specify report` attached to first PR - [ ] Retros used to improve team context - [ ] `dbt-specify ci` promoted when the team is ready diff --git a/pyproject.toml b/pyproject.toml index 7c2eb07..ed147a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "dbt-spec-kit" -version = "1.1.0" +version = "1.2.0" description = "Enterprise AI SDLC toolkit for dbt projects, with spec-driven workflows, CI validation, and warehouse-specific presets." readme = "README.md" license = { file = "LICENSE" } @@ -12,7 +12,20 @@ requires-python = ">=3.11" authors = [ { name = "duckcode-ai", email = "open-source@duckcode.ai" } ] -keywords = ["dbt", "spec-driven-development", "ai-agents", "analytics-engineering", "snowflake", "databricks", "trino", "bigquery"] +keywords = [ + "dbt", + "spec-driven-development", + "ai-agents", + "analytics-engineering", + "snowflake", + "databricks", + "trino", + "bigquery", + "redshift", + "postgres", + "duckdb", + "athena", +] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", diff --git a/src/dbt_specify/_version.py b/src/dbt_specify/_version.py index 60a5e7b..76ec51c 100644 --- a/src/dbt_specify/_version.py +++ b/src/dbt_specify/_version.py @@ -1,4 +1,4 @@ """Single source of truth for the package version.""" from __future__ import annotations -__version__ = "1.1.0" +__version__ = "1.2.0" diff --git a/templates/CLAUDE.md.template b/templates/CLAUDE.md.template index 3067183..2fb7a5c 100644 --- a/templates/CLAUDE.md.template +++ b/templates/CLAUDE.md.template @@ -59,6 +59,14 @@ The non-negotiable principles for this project are in `.dbt-specify/constitution - Keep implementation inside the approved plan's file list. - Human approval remains the merge gate. +## Spec retention + +- Keep decision artifacts in `specs/-/`. +- Do not commit raw agent scratch notes, abandoned drafts, exploratory prompts, or temporary logs. +- Follow the team's retention policy for `tasks.md`, `findings.md`, review notes, and retro notes. +- Default to balanced retention: merge `spec.md`, `plan.md`, and review/report evidence; keep + `tasks.md` for complex, regulated, or high-risk work. + ## Conventions specific to this project