From e5e685198f8e9afdbc95984ead0219391291bf01 Mon Sep 17 00:00:00 2001 From: KKranthi6881 Date: Mon, 18 May 2026 10:54:24 -0500 Subject: [PATCH] Add database planning presets --- CHANGELOG.md | 4 ++ README.md | 11 ++++- docs/getting-started.md | 5 +- docs/warehouse-guides/athena.md | 26 ++++++++++ docs/warehouse-guides/azure-sql.md | 25 ++++++++++ docs/warehouse-guides/bigquery.md | 6 +++ docs/warehouse-guides/duckdb.md | 26 ++++++++++ docs/warehouse-guides/motherduck.md | 26 ++++++++++ docs/warehouse-guides/mysql.md | 25 ++++++++++ docs/warehouse-guides/postgres.md | 25 ++++++++++ docs/warehouse-guides/redshift.md | 26 ++++++++++ docs/warehouse-guides/sqlserver.md | 26 ++++++++++ presets/athena/constitution-additions.md | 24 +++++++++ presets/athena/plan-additions.md | 25 ++++++++++ .../SKILL.md | 28 +++++++++++ presets/azure-sql/constitution-additions.md | 24 +++++++++ presets/azure-sql/plan-additions.md | 25 ++++++++++ .../azure-sql-service-tier-decisions/SKILL.md | 27 ++++++++++ presets/duckdb/constitution-additions.md | 24 +++++++++ presets/duckdb/plan-additions.md | 25 ++++++++++ .../duckdb-local-analytics-decisions/SKILL.md | 27 ++++++++++ presets/motherduck/constitution-additions.md | 24 +++++++++ presets/motherduck/plan-additions.md | 25 ++++++++++ .../SKILL.md | 27 ++++++++++ presets/mysql/constitution-additions.md | 24 +++++++++ presets/mysql/plan-additions.md | 25 ++++++++++ .../SKILL.md | 26 ++++++++++ presets/postgres/constitution-additions.md | 25 ++++++++++ presets/postgres/plan-additions.md | 25 ++++++++++ .../SKILL.md | 27 ++++++++++ presets/redshift/constitution-additions.md | 25 ++++++++++ presets/redshift/plan-additions.md | 25 ++++++++++ .../redshift-dist-sort-decisions/SKILL.md | 32 ++++++++++++ presets/sqlserver/constitution-additions.md | 23 +++++++++ presets/sqlserver/plan-additions.md | 25 ++++++++++ .../SKILL.md | 26 ++++++++++ src/dbt_specify/cli.py | 4 +- src/dbt_specify/init.py | 16 +++++- tests/test_docs.py | 13 +++++ tests/test_init.py | 49 +++++++++++++++++++ 40 files changed, 920 insertions(+), 6 deletions(-) create mode 100644 docs/warehouse-guides/athena.md create mode 100644 docs/warehouse-guides/azure-sql.md create mode 100644 docs/warehouse-guides/duckdb.md create mode 100644 docs/warehouse-guides/motherduck.md create mode 100644 docs/warehouse-guides/mysql.md create mode 100644 docs/warehouse-guides/postgres.md create mode 100644 docs/warehouse-guides/redshift.md create mode 100644 docs/warehouse-guides/sqlserver.md create mode 100644 presets/athena/constitution-additions.md create mode 100644 presets/athena/plan-additions.md create mode 100644 presets/athena/skills/athena-partition-file-layout-decisions/SKILL.md create mode 100644 presets/azure-sql/constitution-additions.md create mode 100644 presets/azure-sql/plan-additions.md create mode 100644 presets/azure-sql/skills/azure-sql-service-tier-decisions/SKILL.md create mode 100644 presets/duckdb/constitution-additions.md create mode 100644 presets/duckdb/plan-additions.md create mode 100644 presets/duckdb/skills/duckdb-local-analytics-decisions/SKILL.md create mode 100644 presets/motherduck/constitution-additions.md create mode 100644 presets/motherduck/plan-additions.md create mode 100644 presets/motherduck/skills/motherduck-collaboration-decisions/SKILL.md create mode 100644 presets/mysql/constitution-additions.md create mode 100644 presets/mysql/plan-additions.md create mode 100644 presets/mysql/skills/mysql-oltp-safe-analytics-decisions/SKILL.md create mode 100644 presets/postgres/constitution-additions.md create mode 100644 presets/postgres/plan-additions.md create mode 100644 presets/postgres/skills/postgres-index-materialization-decisions/SKILL.md create mode 100644 presets/redshift/constitution-additions.md create mode 100644 presets/redshift/plan-additions.md create mode 100644 presets/redshift/skills/redshift-dist-sort-decisions/SKILL.md create mode 100644 presets/sqlserver/constitution-additions.md create mode 100644 presets/sqlserver/plan-additions.md create mode 100644 presets/sqlserver/skills/sqlserver-index-and-incremental-decisions/SKILL.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d4d61..da68d7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Warehouse presets and guides for Redshift, Postgres, SQL Server, Azure SQL, MySQL, DuckDB, + MotherDuck, and Athena. + ## [1.1.0] — 2026-05-18 ### Added diff --git a/README.md b/README.md index df2a43a..9eca840 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ uv tool install dbt-spec-kit dbt-specify --version ``` -Supported warehouse presets: `snowflake`, `databricks`, `trino`, and `bigquery`. +Supported warehouse presets: `snowflake`, `databricks`, `trino`, `bigquery`, `redshift`, +`postgres`, `sqlserver`, `azure-sql`, `mysql`, `duckdb`, `motherduck`, and `athena`. ## What init adds @@ -145,6 +146,14 @@ Use `dbt-specify ci` when the lifecycle and dbt artifact checks should block a P - [Databricks guide](docs/warehouse-guides/databricks.md) - [Trino guide](docs/warehouse-guides/trino.md) - [BigQuery guide](docs/warehouse-guides/bigquery.md) +- [Redshift guide](docs/warehouse-guides/redshift.md) +- [Postgres guide](docs/warehouse-guides/postgres.md) +- [SQL Server guide](docs/warehouse-guides/sqlserver.md) +- [Azure SQL guide](docs/warehouse-guides/azure-sql.md) +- [MySQL guide](docs/warehouse-guides/mysql.md) +- [DuckDB guide](docs/warehouse-guides/duckdb.md) +- [MotherDuck guide](docs/warehouse-guides/motherduck.md) +- [Athena guide](docs/warehouse-guides/athena.md) ## OSS project diff --git a/docs/getting-started.md b/docs/getting-started.md index 7eb7dcd..a67a4e1 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -42,7 +42,8 @@ This creates: - `CLAUDE.md` — the agent orientation file (or `CLAUDE.md.dbt-specify-suggested` if you already have one) - `specs/` — empty directory for your first spec -Supported warehouse presets: `snowflake`, `databricks`, `trino`, and `bigquery`. +Supported warehouse presets: `snowflake`, `databricks`, `trino`, `bigquery`, `redshift`, +`postgres`, `sqlserver`, `azure-sql`, `mysql`, `duckdb`, `motherduck`, and `athena`. ## Compose with dbt-labs/dbt-agent-skills @@ -102,5 +103,5 @@ dbt-specify doctor - Use the [team onboarding playbook](team-onboarding-playbook.md) when presenting the workflow to an analytics engineering team. - Read [methodology.md](methodology.md) for the full four-phase loop. - Read [skills-and-sub-agents.md](skills-and-sub-agents.md) for dbt Labs skills composition and safe sub-agent delegation. -- Read [warehouse-guides/snowflake.md](warehouse-guides/snowflake.md) or [databricks.md](warehouse-guides/databricks.md) for your warehouse's preset. +- Read the [warehouse guides](warehouse-guides/snowflake.md) for preset-specific planning rules. - See [`examples/jaffle-shop-staging-overhaul/`](../examples/jaffle-shop-staging-overhaul/) for a complete worked example. diff --git a/docs/warehouse-guides/athena.md b/docs/warehouse-guides/athena.md new file mode 100644 index 0000000..8f5d092 --- /dev/null +++ b/docs/warehouse-guides/athena.md @@ -0,0 +1,26 @@ +# Athena guide + +Use this preset when dbt targets Amazon Athena and S3-backed tables. + +## What the preset adds + +- S3 table and file-layout planning +- partitioning and partition-projection decisions +- Glue catalog and Lake Formation/IAM governance review +- workgroup, output location, and scan-cost guardrails +- Iceberg versus Hive-style table format decisions + +## Good fit + +- lakehouse tables queried through Athena +- Iceberg or external Hive-style datasets +- teams that need scan-cost and S3 layout evidence in PRs + +## Use + +```bash +dbt-specify init analytics --warehouse athena +``` + +dbt-spec-kit does not query Athena. dbt, the adapter, AWS credentials, and workgroup configuration +remain outside this toolkit. diff --git a/docs/warehouse-guides/azure-sql.md b/docs/warehouse-guides/azure-sql.md new file mode 100644 index 0000000..7aeebea --- /dev/null +++ b/docs/warehouse-guides/azure-sql.md @@ -0,0 +1,25 @@ +# Azure SQL guide + +Use this preset when dbt targets Azure SQL Database or Azure SQL Managed Instance. + +## What the preset adds + +- service tier, elastic pool, CPU, IO, and log-pressure planning +- Query Store evidence expectations for performance-sensitive work +- index, columnstore, and incremental load guardrails +- Microsoft Entra, role, RLS, dynamic masking, and view-boundary review +- firewall, private endpoint, and deployment-window callouts + +## Good fit + +- Azure-native enterprise teams +- shared elastic pool environments +- governed marts using Azure SQL serving layers + +## Use + +```bash +dbt-specify init analytics --warehouse azure-sql +``` + +This is a planning preset. It does not configure Azure credentials, networking, or dbt profiles. diff --git a/docs/warehouse-guides/bigquery.md b/docs/warehouse-guides/bigquery.md index dc697ad..f0cc474 100644 --- a/docs/warehouse-guides/bigquery.md +++ b/docs/warehouse-guides/bigquery.md @@ -3,6 +3,12 @@ The BigQuery preset adds planning requirements for partitioning, clustering, materialization, cost, policy tags, authorized views, and BI Engine expectations. +Use it with: + +```bash +dbt-specify init analytics --warehouse bigquery +``` + ## Partitioning Large fact-like tables should partition by the date or timestamp column that bounds routine diff --git a/docs/warehouse-guides/duckdb.md b/docs/warehouse-guides/duckdb.md new file mode 100644 index 0000000..dafa9bb --- /dev/null +++ b/docs/warehouse-guides/duckdb.md @@ -0,0 +1,26 @@ +# DuckDB guide + +Use this preset when dbt targets DuckDB for local analytics, CI fixtures, or file-based workflows. + +## What the preset adds + +- local file and object-store path review +- memory and single-process limits +- DuckDB extension planning +- Parquet staging and export expectations +- governance handling for local files and CI artifacts + +## Good fit + +- local development examples +- lightweight analytics projects +- CI smoke tests with deterministic data files + +## Use + +```bash +dbt-specify init analytics --warehouse duckdb +``` + +This preset is useful for AI-agent demos because it makes local file dependencies and reproducibility +explicit. diff --git a/docs/warehouse-guides/motherduck.md b/docs/warehouse-guides/motherduck.md new file mode 100644 index 0000000..e290a18 --- /dev/null +++ b/docs/warehouse-guides/motherduck.md @@ -0,0 +1,26 @@ +# MotherDuck guide + +Use this preset when dbt targets MotherDuck or a hybrid DuckDB/MotherDuck workflow. + +## What the preset adds + +- local versus cloud execution boundary review +- database, share, role, and collaborator planning +- file and object-store reproducibility checks +- cost/quota guardrails +- sensitive data movement review + +## Good fit + +- collaborative DuckDB-style analytics +- shared demo and lightweight team projects +- local-to-cloud workflows where data movement must be explicit + +## Use + +```bash +dbt-specify init analytics --warehouse motherduck +``` + +The preset does not configure MotherDuck authentication. It only adds planning and governance +guidance. diff --git a/docs/warehouse-guides/mysql.md b/docs/warehouse-guides/mysql.md new file mode 100644 index 0000000..aba31d5 --- /dev/null +++ b/docs/warehouse-guides/mysql.md @@ -0,0 +1,25 @@ +# MySQL guide + +Use this preset when dbt reads from or writes to MySQL and operational safety matters. + +## What the preset adds + +- primary versus replica versus analytics database review +- index and `EXPLAIN` expectations for heavy joins +- incremental watermark, batch size, and lock planning +- engine, charset, collation, and timezone assumptions +- grants and PII access review + +## Good fit + +- operational-source staging projects +- MySQL replicas used for analytics +- teams that need to prevent AI agents from overloading OLTP databases + +## Use + +```bash +dbt-specify init analytics --warehouse mysql +``` + +dbt-spec-kit does not run MySQL queries. dbt and the selected adapter own execution. diff --git a/docs/warehouse-guides/postgres.md b/docs/warehouse-guides/postgres.md new file mode 100644 index 0000000..7245e41 --- /dev/null +++ b/docs/warehouse-guides/postgres.md @@ -0,0 +1,25 @@ +# Postgres guide + +Use this preset when dbt targets Postgres or uses Postgres as an analytics-safe serving database. + +## What the preset adds + +- OLTP safety checks for primary versus replica versus analytics database +- index and materialization planning +- lock and transaction impact review +- schema/grant/access planning +- extension and non-portable SQL callouts + +## Good fit + +- smaller analytics projects on Postgres +- product analytics replicas +- local or self-managed dbt deployments where lock safety matters + +## Use + +```bash +dbt-specify init analytics --warehouse postgres +``` + +The preset does not connect to Postgres. dbt and its adapter handle database execution. diff --git a/docs/warehouse-guides/redshift.md b/docs/warehouse-guides/redshift.md new file mode 100644 index 0000000..ffd7890 --- /dev/null +++ b/docs/warehouse-guides/redshift.md @@ -0,0 +1,26 @@ +# Redshift guide + +Use this preset when dbt runs on Amazon Redshift or Redshift is the analytics serving database. + +## What the preset adds + +- distribution and sort key planning +- vacuum and analyze expectations +- Spectrum/external table scan boundaries +- workload queue and full-refresh cost guardrails +- schema, grants, late binding view, and restricted-data review + +## Good fit + +- enterprise marts on Redshift +- Redshift Spectrum plus internal table joins +- teams that need PR evidence for dist/sort decisions + +## Use + +```bash +dbt-specify init analytics --warehouse redshift +``` + +dbt still owns execution through the Redshift adapter and the user's normal profile. This preset only +changes planning guidance and generated agent context. diff --git a/docs/warehouse-guides/sqlserver.md b/docs/warehouse-guides/sqlserver.md new file mode 100644 index 0000000..f7b63cd --- /dev/null +++ b/docs/warehouse-guides/sqlserver.md @@ -0,0 +1,26 @@ +# SQL Server guide + +Use this preset when dbt targets Microsoft SQL Server. + +## What the preset adds + +- clustered, nonclustered, and columnstore index planning +- incremental load and transaction log guardrails +- schema, owner, role, and access-path review +- tempdb and concurrency risk checks +- T-SQL compatibility, collation, and date precision callouts + +## Good fit + +- SQL Server analytics marts +- enterprise teams with strict schema and role ownership +- incremental models where blocking and log pressure matter + +## Use + +```bash +dbt-specify init analytics --warehouse sqlserver +``` + +The preset guides planning. dbt adapter configuration and database credentials remain outside +dbt-spec-kit. diff --git a/presets/athena/constitution-additions.md b/presets/athena/constitution-additions.md new file mode 100644 index 0000000..cadd4ac --- /dev/null +++ b/presets/athena/constitution-additions.md @@ -0,0 +1,24 @@ +## Article A1 — S3 file layout is part of model design + +Athena models document table format, file format, target file size, and small-file risk. Plans for +large tables explain how files are compacted or kept query-efficient. + +## Article A2 — Partitioning and projection are explicit + +Large fact-like tables document partition columns, partition projection usage, and expected partition +pruning. Unpartitioned large scans require an approved justification. + +## Article A3 — Glue and Lake Formation governance is reviewed + +Plans identify Glue catalog/database/table ownership and Lake Formation or IAM access boundaries for +governed outputs. + +## Article A4 — Workgroups and query cost guardrails are required + +Plans name the Athena workgroup, expected scanned data, output location, and cost guardrails for +large models. + +## Article A5 — Iceberg versus Hive table format is deliberate + +Plans choose Iceberg, Hive-style external tables, or another format deliberately and document +incremental, compaction, and schema evolution behavior. diff --git a/presets/athena/plan-additions.md b/presets/athena/plan-additions.md new file mode 100644 index 0000000..9c93d91 --- /dev/null +++ b/presets/athena/plan-additions.md @@ -0,0 +1,25 @@ +## Athena-specific concerns + +### Table and file layout + +| Model | Table format | File format/size | Small-file risk | +|---|---|---|---| +| | Iceberg / Hive external | Parquet/ORC/other, | low/medium/high | + +### Partitioning and pruning + +| Model | Partition columns | Projection used? | Expected scan boundary | +|---|---|---|---| +| | | yes/no | | + +### Governance + +| Output | Glue database/table | Lake Formation/IAM boundary | Sensitive? | +|---|---|---|---| +| | | | yes/no | + +### Workgroup and cost + +| Job | Workgroup | Output location | Scan guardrail | +|---|---|---|---| +| | | s3://... | | diff --git a/presets/athena/skills/athena-partition-file-layout-decisions/SKILL.md b/presets/athena/skills/athena-partition-file-layout-decisions/SKILL.md new file mode 100644 index 0000000..6719bd8 --- /dev/null +++ b/presets/athena/skills/athena-partition-file-layout-decisions/SKILL.md @@ -0,0 +1,28 @@ +--- +name: athena-partition-file-layout-decisions +description: Use when planning Athena dbt models that depend on S3 layout, partitions, Glue/Lake Formation governance, and scan cost. +--- + +# Athena partition and file-layout decisions + +## When to use this skill + +Use this for Athena-backed dbt models, especially large tables, Iceberg tables, external sources, or +governed S3 data. + +## Decision routine + +1. Choose table format deliberately: Iceberg for managed evolution/upserts, Hive-style external + tables for simple append/read patterns. +2. Prefer columnar files such as Parquet or ORC. +3. Document partition columns and expected pruning predicates. +4. Check small-file risk and compaction expectations. +5. Name Glue database/table ownership and Lake Formation or IAM access boundaries. +6. Name workgroup, query result location, and scan cost guardrails. + +## Common failures + +- Partitioning by a high-cardinality column that creates too many S3 prefixes. +- Generating many tiny files that make every query expensive. +- Running unbounded scans because partition projection was not planned. +- Publishing governed outputs without Glue/Lake Formation ownership review. diff --git a/presets/azure-sql/constitution-additions.md b/presets/azure-sql/constitution-additions.md new file mode 100644 index 0000000..df25903 --- /dev/null +++ b/presets/azure-sql/constitution-additions.md @@ -0,0 +1,24 @@ +## Article AZ1 — Service tier and workload impact are documented + +Plans identify Azure SQL service tier, elastic pool impact, expected workload, and whether the change +can cause CPU, IO, or log pressure. + +## Article AZ2 — Query Store evidence guides performance changes + +Performance-sensitive changes document expected query patterns and use Query Store or equivalent +evidence after deployment when available. + +## Article AZ3 — Index, columnstore, and incremental choices are explicit + +Models document clustered index, nonclustered indexes, columnstore usage, incremental strategy, and +transaction log impact. + +## Article AZ4 — Security uses Azure-native controls + +Governed outputs document Microsoft Entra authentication assumptions, roles, row-level security, +dynamic data masking, or views used for access boundaries. + +## Article AZ5 — Operational limits are respected + +Plans call out firewall/private endpoint assumptions, cross-database access, tempdb pressure, and +deployment windows for full refreshes. diff --git a/presets/azure-sql/plan-additions.md b/presets/azure-sql/plan-additions.md new file mode 100644 index 0000000..22a71e7 --- /dev/null +++ b/presets/azure-sql/plan-additions.md @@ -0,0 +1,25 @@ +## Azure SQL-specific concerns + +### Service tier and workload + +| Database/pool | Tier | Workload risk | Guardrail | +|---|---|---|---| +| | | CPU / IO / log / none | | + +### Index and incremental strategy + +| Model | Index/columnstore plan | Incremental strategy | Log risk | +|---|---|---|---| +| | | merge / delete+insert / full refresh | low/medium/high | + +### Security and access + +| Output | Role/access path | RLS/DDM/view boundary | Reviewer | +|---|---|---|---| +| | | | | + +### Operational constraints + +| Constraint | Impact | Mitigation | +|---|---|---| +| firewall/private endpoint/cross-db/tempdb | | | diff --git a/presets/azure-sql/skills/azure-sql-service-tier-decisions/SKILL.md b/presets/azure-sql/skills/azure-sql-service-tier-decisions/SKILL.md new file mode 100644 index 0000000..b3dda29 --- /dev/null +++ b/presets/azure-sql/skills/azure-sql-service-tier-decisions/SKILL.md @@ -0,0 +1,27 @@ +--- +name: azure-sql-service-tier-decisions +description: Use when planning Azure SQL dbt work that affects service tier, elastic pools, indexing, security, or workload pressure. +--- + +# Azure SQL service tier and workload decisions + +## When to use this skill + +Use this when dbt runs against Azure SQL Database, Azure SQL Managed Instance, or an Azure SQL-backed +analytics serving database. + +## Decision routine + +1. Identify service tier, elastic pool, and expected workload pressure. +2. Use Query Store or representative query plans to justify performance-sensitive changes. +3. Choose indexes and columnstore based on query patterns and data volume. +4. Plan incremental loads around transaction log and concurrency limits. +5. Use Azure-native access controls for governed data: roles, views, RLS, or dynamic data masking. +6. Document network/private endpoint and deployment-window constraints. + +## Common failures + +- Treating Azure SQL like a limitless warehouse. +- Full refreshing large marts in a shared elastic pool without a window. +- Adding indexes that help dbt but hurt application writes. +- Missing RLS or masking decisions for governed outputs. diff --git a/presets/duckdb/constitution-additions.md b/presets/duckdb/constitution-additions.md new file mode 100644 index 0000000..fb8d9d9 --- /dev/null +++ b/presets/duckdb/constitution-additions.md @@ -0,0 +1,24 @@ +## Article D1 — Local file boundaries are explicit + +Plans name every local file, object-store path, or external source read by DuckDB models. Paths must +be reproducible for CI or clearly marked local-only. + +## Article D2 — Memory and single-process limits are respected + +Large transformations document expected input size, memory risk, and whether data should be staged as +Parquet before downstream models consume it. + +## Article D3 — Extensions are deliberate dependencies + +Use of DuckDB extensions such as httpfs, spatial, iceberg, or json is named in the plan with install +and CI expectations. + +## Article D4 — Outputs are portable or intentionally local + +Plans say whether outputs are local development artifacts, CI artifacts, Parquet exports, or inputs +to another warehouse. Local-only marts are not presented as enterprise serving tables. + +## Article D5 — Governance still applies to files + +PII in local files, Parquet exports, or CI artifacts requires masking, exclusion, or an explicit +handling decision. diff --git a/presets/duckdb/plan-additions.md b/presets/duckdb/plan-additions.md new file mode 100644 index 0000000..66ad2eb --- /dev/null +++ b/presets/duckdb/plan-additions.md @@ -0,0 +1,25 @@ +## DuckDB-specific concerns + +### File and source boundaries + +| Input/output | Location | Reproducible in CI? | Notes | +|---|---|---|---| +| | local / object storage / generated | yes/no | | + +### Memory and data size + +| Model | Input size estimate | Memory risk | Mitigation | +|---|---|---|---| +| | | low/medium/high | | + +### Extensions + +| Extension | Why needed | CI install covered? | +|---|---|---| +| | | yes/no | + +### Governance for file outputs + +| File/model | Sensitive data? | Handling decision | +|---|---|---| +| | yes/no | mask / exclude / approved | diff --git a/presets/duckdb/skills/duckdb-local-analytics-decisions/SKILL.md b/presets/duckdb/skills/duckdb-local-analytics-decisions/SKILL.md new file mode 100644 index 0000000..d4a48a7 --- /dev/null +++ b/presets/duckdb/skills/duckdb-local-analytics-decisions/SKILL.md @@ -0,0 +1,27 @@ +--- +name: duckdb-local-analytics-decisions +description: Use when planning DuckDB dbt work around local files, Parquet, extensions, memory, and CI reproducibility. +--- + +# DuckDB local analytics decisions + +## When to use this skill + +Use this for local analytics, CI fixtures, Parquet-first workflows, and lightweight dbt projects that +target DuckDB. + +## Decision routine + +1. List every local file, object-store path, or generated artifact. +2. Confirm whether each path exists in CI or is local-only. +3. Estimate data size and memory pressure. +4. Use Parquet staging for large repeated reads. +5. Document required DuckDB extensions and install behavior. +6. Treat file outputs with PII as governed artifacts. + +## Common failures + +- Plans that work only on one developer's absolute file path. +- Loading large CSVs repeatedly instead of staging Parquet. +- Using extensions without CI setup. +- Exporting PII to local files without a handling decision. diff --git a/presets/motherduck/constitution-additions.md b/presets/motherduck/constitution-additions.md new file mode 100644 index 0000000..a5c89ab --- /dev/null +++ b/presets/motherduck/constitution-additions.md @@ -0,0 +1,24 @@ +## Article MD1 — Local versus cloud execution is explicit + +Plans say which work runs locally, which work runs in MotherDuck, and which data crosses that +boundary. This is required for cost, privacy, and reproducibility review. + +## Article MD2 — Shares, databases, and collaborators are governed + +Models and exports document MotherDuck database, share, role, and collaborator assumptions before +implementation. + +## Article MD3 — File and object-store sources are reproducible + +External files, object-store paths, and local development files are listed with CI or production +availability expectations. + +## Article MD4 — Cost and quota impact is reviewed + +Plans call out expected data volume, compute behavior, and any large scans or exports that may affect +MotherDuck quotas or spend. + +## Article MD5 — Sensitive data is not casually synchronized + +PII or restricted data crossing local/cloud boundaries requires masking, exclusion, or approved access +before implementation. diff --git a/presets/motherduck/plan-additions.md b/presets/motherduck/plan-additions.md new file mode 100644 index 0000000..aea66e8 --- /dev/null +++ b/presets/motherduck/plan-additions.md @@ -0,0 +1,25 @@ +## MotherDuck-specific concerns + +### Local/cloud boundary + +| Model/source | Runs local or MotherDuck? | Data crosses boundary? | Reason | +|---|---|---|---| +| | local / cloud / hybrid | yes/no | | + +### Database, share, and role plan + +| Output | Database/share | Role/collaborators | Sensitive? | +|---|---|---|---| +| | | | yes/no | + +### File and object-store reproducibility + +| Input | Location | Available in CI/prod? | Mitigation | +|---|---|---|---| +| | local / object storage | yes/no | | + +### Cost and quota guardrails + +| Operation | Data volume | Guardrail | +|---|---|---| +| | | | diff --git a/presets/motherduck/skills/motherduck-collaboration-decisions/SKILL.md b/presets/motherduck/skills/motherduck-collaboration-decisions/SKILL.md new file mode 100644 index 0000000..bed8512 --- /dev/null +++ b/presets/motherduck/skills/motherduck-collaboration-decisions/SKILL.md @@ -0,0 +1,27 @@ +--- +name: motherduck-collaboration-decisions +description: Use when planning MotherDuck dbt work involving local/cloud execution, sharing, collaborators, files, and sensitive data movement. +--- + +# MotherDuck collaboration decisions + +## When to use this skill + +Use this for MotherDuck-backed dbt projects, shared analytics databases, or local-to-cloud DuckDB +workflows. + +## Decision routine + +1. Decide whether each model runs locally, in MotherDuck, or in a hybrid path. +2. List data that crosses local/cloud boundaries. +3. Document databases, shares, roles, and collaborators. +4. Confirm file/object-store sources are reproducible in CI and production. +5. Estimate large scans/exports and quota risk. +6. Block sensitive data synchronization without an approved handling decision. + +## Common failures + +- Treating local files as if every collaborator can access them. +- Sharing a database before role/access review. +- Moving PII between local and cloud execution without approval. +- Ignoring quota/cost impact for large exploratory scans. diff --git a/presets/mysql/constitution-additions.md b/presets/mysql/constitution-additions.md new file mode 100644 index 0000000..88e222f --- /dev/null +++ b/presets/mysql/constitution-additions.md @@ -0,0 +1,24 @@ +## Article M1 — Operational source safety comes first + +Plans identify whether MySQL is a production OLTP source, replica, or analytics database. Heavy dbt +work must not run against an unprotected production primary. + +## Article M2 — Indexes and explain plans support every heavy model + +Large joins and filters document required indexes and `EXPLAIN` evidence or expected plan behavior. +Views over heavy joins require a justification. + +## Article M3 — Incremental loads avoid large locks + +Incremental models document watermark, primary/unique key, batch size, and lock impact. Full refreshes +of large tables require an approved window. + +## Article M4 — Engine, charset, collation, and timezone assumptions are visible + +Plans call out InnoDB assumptions, collation-sensitive comparisons, and timezone/date handling when +they affect business logic. + +## Article M5 — Grants and PII access are reviewed + +Governed marts document schemas/databases, grants, serving views, and sensitive columns before +implementation. diff --git a/presets/mysql/plan-additions.md b/presets/mysql/plan-additions.md new file mode 100644 index 0000000..7283406 --- /dev/null +++ b/presets/mysql/plan-additions.md @@ -0,0 +1,25 @@ +## MySQL-specific concerns + +### Operational safety + +| Source/model | Database role | Lock/load risk | Mitigation | +|---|---|---|---| +| | primary / replica / analytics | low/medium/high | | + +### Index and query plan + +| Model/query | Required indexes | EXPLAIN expected? | Notes | +|---|---|---|---| +| | | yes/no | | + +### Incremental strategy + +| Model | Watermark/key | Batch size | Full-refresh window | +|---|---|---|---| +| | | | | + +### Compatibility assumptions + +| Area | Assumption | Risk | +|---|---|---| +| engine/charset/collation/timezone | | | diff --git a/presets/mysql/skills/mysql-oltp-safe-analytics-decisions/SKILL.md b/presets/mysql/skills/mysql-oltp-safe-analytics-decisions/SKILL.md new file mode 100644 index 0000000..9b4e7af --- /dev/null +++ b/presets/mysql/skills/mysql-oltp-safe-analytics-decisions/SKILL.md @@ -0,0 +1,26 @@ +--- +name: mysql-oltp-safe-analytics-decisions +description: Use when planning MySQL dbt work that must protect OLTP systems, indexes, locks, and incremental loads. +--- + +# MySQL OLTP-safe analytics decisions + +## When to use this skill + +Use this when MySQL is a source or target for dbt and the workload might affect operational systems. + +## Decision routine + +1. Confirm whether dbt reads a primary, replica, or analytics copy. +2. Require indexes for large joins, filters, and uniqueness checks. +3. Use `EXPLAIN` expectations for heavy queries. +4. Prefer incremental loads with watermarks over repeated full refreshes. +5. Document lock risk, batch size, and deployment window. +6. Call out engine, charset, collation, and timezone assumptions that affect output semantics. + +## Common failures + +- Running analytic joins on a production primary. +- Full refreshing a large table during business hours. +- Comparing text across unexpected collations. +- Forgetting timezone conversion for order/event dates. diff --git a/presets/postgres/constitution-additions.md b/presets/postgres/constitution-additions.md new file mode 100644 index 0000000..f8edc3a --- /dev/null +++ b/presets/postgres/constitution-additions.md @@ -0,0 +1,25 @@ +## Article P1 — Production analytics do not harm OLTP systems + +Plans identify whether Postgres is an analytics warehouse, replica, or operational source. Heavy dbt +models must run on an analytics-safe database or replica, not an unconstrained production OLTP +primary. + +## Article P2 — Index and materialization choices are explicit + +Tables, incremental models, and materialized views document indexes needed for joins, filters, and +refresh behavior. Views over expensive joins require a justification. + +## Article P3 — Locking and transaction impact are reviewed + +Plans for large table rebuilds, materialized view refreshes, or schema changes document lock risk and +deployment timing. Blocking production workloads is not acceptable without an approved window. + +## Article P4 — Grants and schemas are part of the plan + +Marts document schema ownership, grants, and whether access is through direct tables, views, or a +serving schema. Sensitive columns require an access decision. + +## Article P5 — Extensions and non-portable SQL are called out + +Use of Postgres-specific extensions, custom functions, JSON operators, or advanced indexes is named +in the plan with compatibility and maintenance expectations. diff --git a/presets/postgres/plan-additions.md b/presets/postgres/plan-additions.md new file mode 100644 index 0000000..9ba5330 --- /dev/null +++ b/presets/postgres/plan-additions.md @@ -0,0 +1,25 @@ +## Postgres-specific concerns + +### OLTP safety + +| Model/source | Database role | Workload risk | Mitigation | +|---|---|---|---| +| | analytics / replica / primary | low/medium/high | | + +### Index and materialization plan + +| Model | Materialization | Indexes | Refresh/load behavior | +|---|---|---|---| +| | view / table / incremental / materialized view | `` or none | | + +### Locking and deployment + +| Change | Lock risk | Deployment window needed? | Mitigation | +|---|---|---|---| +| | low/medium/high | yes/no | | + +### Grants and schemas + +| Output | Schema | Grant/access path | Sensitive columns | +|---|---|---|---| +| | | table / view / role | | diff --git a/presets/postgres/skills/postgres-index-materialization-decisions/SKILL.md b/presets/postgres/skills/postgres-index-materialization-decisions/SKILL.md new file mode 100644 index 0000000..80282c9 --- /dev/null +++ b/presets/postgres/skills/postgres-index-materialization-decisions/SKILL.md @@ -0,0 +1,27 @@ +--- +name: postgres-index-materialization-decisions +description: Use when planning Postgres dbt materializations, indexes, lock risk, and OLTP-safe execution. +--- + +# Postgres index and materialization decisions + +## When to use this skill + +Use this for Postgres-backed dbt models, especially when the database may also support application +traffic. + +## Decision routine + +1. Confirm whether dbt runs against a primary, replica, or dedicated analytics database. +2. Prefer tables or incremental models for expensive joins used by dashboards. +3. Add indexes only when they support a named join, filter, uniqueness check, or serving query. +4. For materialized views, document refresh strategy and lock impact. +5. For large rebuilds, document transaction/lock risk and acceptable deployment window. +6. Keep grants and schemas explicit for governed marts. + +## Common failures + +- Running heavy transformations on an operational primary without a guardrail. +- Creating views over expensive joins that every dashboard reruns. +- Adding broad indexes without a query pattern. +- Refreshing a materialized view during peak application traffic. diff --git a/presets/redshift/constitution-additions.md b/presets/redshift/constitution-additions.md new file mode 100644 index 0000000..81e0095 --- /dev/null +++ b/presets/redshift/constitution-additions.md @@ -0,0 +1,25 @@ +## Article R1 — Distribution and sort keys are explicit decisions + +Large fact and mart tables document distribution style, distribution key, sort key, and why the +choice matches the dominant joins and filters. `AUTO` is acceptable only when the plan names the +expected table size and review evidence. + +## Article R2 — Vacuum and analyze expectations are planned + +Incremental and append-heavy models document whether `VACUUM` and `ANALYZE` are expected after +production builds. Plans call out the owner of table maintenance when dbt is not responsible. + +## Article R3 — Spectrum and external data are bounded + +Models reading external tables document partition pruning, file layout, and expected scan size. +External joins to large internal tables require an explicit materialization or staging decision. + +## Article R4 — Workload and cost impact are visible + +Plans identify the production workload queue, concurrency scaling expectations, and any large +full-refresh risk. Production plans do not rely on unbounded full scans as routine behavior. + +## Article R5 — Grants, schemas, and late binding views are deliberate + +Governed marts document schema ownership, grants, and whether late binding views are used for +consumer stability. Restricted data requires an access decision before implementation. diff --git a/presets/redshift/plan-additions.md b/presets/redshift/plan-additions.md new file mode 100644 index 0000000..e643050 --- /dev/null +++ b/presets/redshift/plan-additions.md @@ -0,0 +1,25 @@ +## Redshift-specific concerns + +### Distribution and sort decisions + +| Model | Size estimate | Dist style/key | Sort key | Justification | +|---|---|---|---|---| +| | | AUTO / KEY(``) / EVEN / ALL | `` or none | | + +### Maintenance expectations + +| Model | Load pattern | Vacuum needed? | Analyze needed? | Owner | +|---|---|---|---|---| +| | full / incremental / append | yes/no | yes/no | dbt / platform | + +### External and Spectrum access + +| Source | Partition boundary | File layout risk | Mitigation | +|---|---|---|---| +| | | small files / unbounded scan | | + +### Workload and cost guardrails + +| Job | Queue/workload | Full-refresh risk | Guardrail | +|---|---|---|---| +| | | low/medium/high | | diff --git a/presets/redshift/skills/redshift-dist-sort-decisions/SKILL.md b/presets/redshift/skills/redshift-dist-sort-decisions/SKILL.md new file mode 100644 index 0000000..0135ca9 --- /dev/null +++ b/presets/redshift/skills/redshift-dist-sort-decisions/SKILL.md @@ -0,0 +1,32 @@ +--- +name: redshift-dist-sort-decisions +description: Use when choosing Redshift distribution, sort, maintenance, and Spectrum patterns for dbt models. +--- + +# Redshift distribution and sort decisions + +## When to use this skill + +Use this for Redshift tables, incremental models, large marts, or models joining internal and +external data. + +## Decision routine + +1. Identify the largest table and the most common join key. +2. Choose distribution style deliberately: `AUTO` for uncertain/simple cases, `KEY` for stable large + joins, `ALL` only for small dimensions, `EVEN` when no join key dominates. +3. Choose sort keys for the dominant filter range, usually a date/timestamp or a stable business key. +4. For incremental models, document whether `VACUUM` and `ANALYZE` are needed. +5. For Spectrum sources, verify partition pruning and avoid large unbounded external joins. +6. Document grants, schemas, and late binding view decisions for governed outputs. + +## Common failures + +- Distribution key chosen from a column that is not used in the large join. +- Sort key chosen from load order instead of query predicates. +- External table scans without a partition boundary. +- Append-heavy incremental tables with no maintenance plan. + +## Output + +Return distribution, sort, maintenance, and external-access recommendations with review evidence. diff --git a/presets/sqlserver/constitution-additions.md b/presets/sqlserver/constitution-additions.md new file mode 100644 index 0000000..f5218d0 --- /dev/null +++ b/presets/sqlserver/constitution-additions.md @@ -0,0 +1,23 @@ +## Article Q1 — Index strategy is part of model design + +SQL Server tables and marts document clustered index, nonclustered indexes, or columnstore choices. +Index decisions must map to grain, joins, filters, and serving workload. + +## Article Q2 — Incremental loads protect the transaction log + +Large incremental models document batch size, merge/update strategy, and transaction log impact. +Full refreshes of large tables require an approved deployment window. + +## Article Q3 — Schemas, ownership, and permissions are explicit + +Plans name schemas, owners, grants, and downstream access paths. Restricted columns require an access +decision before implementation. + +## Article Q4 — Tempdb and concurrency risks are reviewed + +Large sorts, joins, snapshots, and temporary objects document tempdb/concurrency risk and mitigation. + +## Article Q5 — T-SQL-specific behavior is called out + +Plans name non-portable T-SQL patterns, date/time precision decisions, collations, and compatibility +level assumptions when relevant. diff --git a/presets/sqlserver/plan-additions.md b/presets/sqlserver/plan-additions.md new file mode 100644 index 0000000..9eff438 --- /dev/null +++ b/presets/sqlserver/plan-additions.md @@ -0,0 +1,25 @@ +## SQL Server-specific concerns + +### Index and storage design + +| Model | Clustered index | Nonclustered/columnstore indexes | Justification | +|---|---|---|---| +| | or heap | or none | | + +### Incremental and log impact + +| Model | Strategy | Batch/log risk | Mitigation | +|---|---|---|---| +| | merge / delete+insert / full refresh | low/medium/high | | + +### Permissions and schemas + +| Output | Schema | Owner/role | Access path | +|---|---|---|---| +| | | | table / view / proc | + +### Concurrency and tempdb + +| Operation | Risk | Mitigation | +|---|---|---| +| | tempdb / blocking / spill | | diff --git a/presets/sqlserver/skills/sqlserver-index-and-incremental-decisions/SKILL.md b/presets/sqlserver/skills/sqlserver-index-and-incremental-decisions/SKILL.md new file mode 100644 index 0000000..f10f039 --- /dev/null +++ b/presets/sqlserver/skills/sqlserver-index-and-incremental-decisions/SKILL.md @@ -0,0 +1,26 @@ +--- +name: sqlserver-index-and-incremental-decisions +description: Use when planning SQL Server dbt index, columnstore, incremental, and concurrency decisions. +--- + +# SQL Server index and incremental decisions + +## When to use this skill + +Use this for SQL Server marts, incremental models, large joins, or governed outputs. + +## Decision routine + +1. Confirm the table grain and primary serving query. +2. Choose clustered rowstore, clustered columnstore, or heap deliberately. +3. Add nonclustered indexes only for named joins, filters, or uniqueness checks. +4. For incremental models, document merge key, batch size, and transaction log risk. +5. Check whether large operations can stress tempdb or block concurrent readers. +6. Document schemas, ownership, grants, and restricted column access. + +## Common failures + +- Clustered index does not match the grain or access pattern. +- Large `MERGE` operations with no batch/log plan. +- Columnstore used on small or highly volatile tables by reflex. +- Hidden collation/date precision changes in mart outputs. diff --git a/src/dbt_specify/cli.py b/src/dbt_specify/cli.py index 3fccb81..2141ad9 100644 --- a/src/dbt_specify/cli.py +++ b/src/dbt_specify/cli.py @@ -8,7 +8,7 @@ from dbt_specify._version import __version__ from dbt_specify.dbt_artifacts import validate_dbt_project from dbt_specify.doctor import doctor_project -from dbt_specify.init import init_project +from dbt_specify.init import SUPPORTED_WAREHOUSES, init_project from dbt_specify.lifecycle import validate_lifecycle from dbt_specify.reporting import ValidationReport, combine_reports from dbt_specify.validate import validate_spec @@ -24,7 +24,7 @@ def main() -> None: @click.argument("project_name") @click.option( "--warehouse", - type=click.Choice(["snowflake", "databricks", "trino", "bigquery"], case_sensitive=False), + type=click.Choice(SUPPORTED_WAREHOUSES, case_sensitive=False), required=True, help="Warehouse preset to install alongside the base constitution.", ) diff --git a/src/dbt_specify/init.py b/src/dbt_specify/init.py index 6096b1b..d0be3b6 100644 --- a/src/dbt_specify/init.py +++ b/src/dbt_specify/init.py @@ -11,6 +11,20 @@ # Filter junk that may live in the source tree during editable installs # (macOS Finder leftovers, pyc caches, etc.) so user-visible output stays clean. _IGNORE_JUNK = shutil.ignore_patterns(".DS_Store", "__pycache__", "*.pyc", ".gitkeep") +SUPPORTED_WAREHOUSES = ( + "snowflake", + "databricks", + "trino", + "bigquery", + "redshift", + "postgres", + "sqlserver", + "azure-sql", + "mysql", + "duckdb", + "motherduck", + "athena", +) def init_project( @@ -30,7 +44,7 @@ def init_project( Raises: SystemExit: On any precondition failure. """ - if warehouse not in {"snowflake", "databricks", "trino", "bigquery"}: + if warehouse not in SUPPORTED_WAREHOUSES: click.echo(f"error: unknown warehouse '{warehouse}'", err=True) raise SystemExit(2) diff --git a/tests/test_docs.py b/tests/test_docs.py index 097b6f2..5749e0c 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -83,6 +83,19 @@ def test_skills_and_sub_agents_doc_covers_composition() -> None: assert role in text +def test_warehouse_guides_exist_for_supported_presets() -> None: + from dbt_specify.init import SUPPORTED_WAREHOUSES + + readme = (ROOT / "README.md").read_text() + getting_started = (ROOT / "docs" / "getting-started.md").read_text() + for warehouse in SUPPORTED_WAREHOUSES: + guide = ROOT / "docs" / "warehouse-guides" / f"{warehouse}.md" + assert guide.exists(), f"Missing warehouse guide: {guide.relative_to(ROOT)}" + assert f"--warehouse {warehouse}" in guide.read_text() + assert warehouse in readme + assert warehouse in getting_started + + def test_tutorials_cover_enterprise_onboarding_path() -> None: tutorials_dir = ROOT / "docs" / "tutorials" required_paths = [ diff --git a/tests/test_init.py b/tests/test_init.py index b60e429..d480854 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -6,6 +6,7 @@ from click.testing import CliRunner from dbt_specify.cli import main +from dbt_specify.init import SUPPORTED_WAREHOUSES from dbt_specify.templates_loader import asset_dir, load_template @@ -49,6 +50,8 @@ def test_init_help_shows_flags() -> None: assert "--warehouse" in result.output assert "--force" in result.output assert "--target" in result.output + for warehouse in SUPPORTED_WAREHOUSES: + assert warehouse in result.output def test_init_creates_dbt_specify_dir(minimal_dbt_project: Path) -> None: @@ -184,6 +187,52 @@ def test_init_bigquery_preset(minimal_dbt_project: Path) -> None: assert "BigQuery-specific concerns" in plan +def test_every_supported_warehouse_preset_initializes(tmp_path: Path) -> None: + runner = CliRunner() + expected_skill_by_warehouse = { + "snowflake": "snowflake-clustering-decisions", + "databricks": "databricks-liquid-clustering-decisions", + "trino": "trino-federated-query-patterns", + "bigquery": "bigquery-partitioning-decisions", + "redshift": "redshift-dist-sort-decisions", + "postgres": "postgres-index-materialization-decisions", + "sqlserver": "sqlserver-index-and-incremental-decisions", + "azure-sql": "azure-sql-service-tier-decisions", + "mysql": "mysql-oltp-safe-analytics-decisions", + "duckdb": "duckdb-local-analytics-decisions", + "motherduck": "motherduck-collaboration-decisions", + "athena": "athena-partition-file-layout-decisions", + } + assert set(expected_skill_by_warehouse) == set(SUPPORTED_WAREHOUSES) + + for warehouse in SUPPORTED_WAREHOUSES: + project_dir = tmp_path / warehouse + project_dir.mkdir() + (project_dir / "dbt_project.yml").write_text( + "name: test_project\nversion: '1.0.0'\nprofile: test\n" + ) + (project_dir / "models").mkdir() + + result = runner.invoke( + main, + ["init", "test", "--warehouse", warehouse, "--target", str(project_dir)], + ) + assert result.exit_code == 0, result.output + + constitution = (project_dir / ".dbt-specify" / "constitution.md").read_text() + plan = (project_dir / ".dbt-specify" / "templates" / "plan-template.md").read_text() + expected_marker = f"BEGIN {warehouse.upper()} ADDITIONS" + assert expected_marker in constitution + assert f"BEGIN {warehouse.upper()} PLAN ADDITIONS" in plan + assert ( + project_dir + / ".dbt-specify" + / "skills" + / expected_skill_by_warehouse[warehouse] + / "SKILL.md" + ).exists() + + def test_init_existing_claude_md_writes_suggested_file(minimal_dbt_project: Path) -> None: (minimal_dbt_project / "CLAUDE.md").write_text("# pre-existing\n") runner = CliRunner()