diff --git a/.github/workflows/docs-gates.yml b/.github/workflows/docs-gates.yml new file mode 100644 index 000000000..b4918523c --- /dev/null +++ b/.github/workflows/docs-gates.yml @@ -0,0 +1,36 @@ +# Documentation gates — MANDATORY, BLOCKING (required status check `docs-gates`). +# +# 1. Every component/domain has specs/PRD.md + specs/DESIGN.md — no docs, no pass. +# 2. PRD/DESIGN follow the single canonical template — same structure everywhere. +# 3. Artifacts are registered in cypilot/config/artifacts.toml — mapped to code. +# 4. docs/DOCS_MAP.md is regenerated and committed — the map never goes stale. +# +# Exceptions only via docs/.docs-gate-waivers (reason + expiry, CODEOWNERS-reviewed; +# expired waivers fail the gate). CI calls the same entrypoint developers run +# locally (`make docs-check`) — shift-left, zero local/CI drift. + +name: Docs Gates + +on: + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + docs-gates: + name: docs-gates + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Documentation gate (presence, template conformance, code mapping) + run: python3 scripts/ci/docs_map.py --check + - name: DOCS_MAP.md must be regenerated and committed + run: | + git diff --exit-code docs/DOCS_MAP.md || { + echo "::error::docs/DOCS_MAP.md is stale. Run: make docs-map && commit."; exit 1; } diff --git a/docs/.docs-gate-waivers b/docs/.docs-gate-waivers new file mode 100644 index 000000000..f718e666f --- /dev/null +++ b/docs/.docs-gate-waivers @@ -0,0 +1,7 @@ +# Documentation-gate waivers — the ONLY way past the docs gate. +# Format: | | +# Expired waivers FAIL the gate by themselves. Reviewed via CODEOWNERS. +docs/components/connectors | index of per-source specs; per-source PRD/DESIGN exist under each source dir — top-level specs to be added | 2026-07-31 +docs/domain/ingestion-data-flow | DESIGN-only domain (FULL traceability); PRD backlog item | 2026-07-31 +docs/components/frontend | PRD/DESIGN are placeholders not on the canonical template; rewrite + register scheduled | 2026-07-15 +docs/components/orchestrator | DESIGN not on canonical template; align + register scheduled | 2026-07-15 diff --git a/docs/DOCS_MAP.md b/docs/DOCS_MAP.md new file mode 100644 index 000000000..c9253a798 --- /dev/null +++ b/docs/DOCS_MAP.md @@ -0,0 +1,411 @@ + +# Documentation Map + +Total markdown files: **355**. + +## Coverage gate — components & domains + +| Unit | PRD | DESIGN | ADRs | Mapped (artifacts.toml) | Gate | +|---|---|---|---|---|---| +| `docs/components/airbyte-toolkit` | ✅ | ✅ | 16 | ✅ | ✅ | +| `docs/components/backend` | ✅ | ✅ | 1 | ✅ | ✅ | +| `docs/components/connectors` | ❌ | ❌ | 0 | ✅ | ⚠️ waived→2026-07-31 | +| `docs/components/deployment` | ✅ | ✅ | 1 | ✅ | ✅ | +| `docs/components/frontend` | ✅ | ✅ | 0 | ❌ | ⚠️ waived→2026-07-15 | +| `docs/components/orchestrator` | ✅ | ✅ | 0 | ❌ | ⚠️ waived→2026-07-15 | +| `docs/domain/bronze-to-api-e2e` | ✅ | ✅ | 0 | ✅ | ✅ | +| `docs/domain/connector` | ✅ | ✅ | 3 | ✅ | ✅ | +| `docs/domain/identity-resolution` | ✅ | ✅ | 1 | ✅ | ✅ | +| `docs/domain/ingestion` | ✅ | ✅ | 6 | ✅ | ✅ | +| `docs/domain/ingestion-data-flow` | ❌ | ✅ | 5 | ✅ | ⚠️ waived→2026-07-31 | +| `docs/domain/metric-catalog` | ✅ | ✅ | 3 | ✅ | ✅ | +| `docs/domain/org-chart` | ✅ | ✅ | 0 | ✅ | ✅ | +| `docs/domain/person` | ✅ | ✅ | 1 | ✅ | ✅ | + +## Inventory by category + +### PRD (55) + +- `docs/components/airbyte-toolkit/specs/PRD.md` +- `docs/components/backend/api-gateway/PRD.md` +- `docs/components/backend/api-gateway/bff/PRD.md` +- `docs/components/backend/api-gateway/router/PRD.md` +- `docs/components/backend/identity-resolution/identity/specs/PRD.md` +- `docs/components/backend/specs/PRD.md` +- `docs/components/connectors/ai/chatgpt-team/specs/PRD.md` +- `docs/components/connectors/ai/claude-admin/specs/PRD.md` +- `docs/components/connectors/ai/claude-enterprise/specs/PRD.md` +- `docs/components/connectors/ai/claude-team/specs/PRD.md` +- `docs/components/connectors/ai/cursor/specs/PRD.md` +- `docs/components/connectors/ai/github-copilot/specs/PRD.md` +- `docs/components/connectors/ai/jetbrains/specs/PRD.md` +- `docs/components/connectors/ai/openai-api/specs/PRD.md` +- `docs/components/connectors/ai/windsurf/specs/PRD.md` +- `docs/components/connectors/collaboration/m365/specs/PRD.md` +- `docs/components/connectors/collaboration/slack/specs/PRD.md` +- `docs/components/connectors/collaboration/zoom/specs/PRD.md` +- `docs/components/connectors/collaboration/zulip/specs/PRD.md` +- `docs/components/connectors/collaboration/zulip-proxy/specs/PRD.md` +- `docs/components/connectors/crm/hubspot/specs/PRD.md` +- `docs/components/connectors/crm/salesforce/specs/PRD.md` +- `docs/components/connectors/git/bitbucket-server/specs/PRD.md` +- `docs/components/connectors/git/github/specs/PRD.md` +- `docs/components/connectors/git/gitlab/specs/PRD.md` +- `docs/components/connectors/hr-directory/bamboohr/specs/PRD.md` +- `docs/components/connectors/hr-directory/ldap/specs/PRD.md` +- `docs/components/connectors/hr-directory/ms-entra/specs/PRD.md` +- `docs/components/connectors/hr-directory/workday/specs/PRD.md` +- `docs/components/connectors/support/jsm/specs/PRD.md` +- `docs/components/connectors/support/zendesk/specs/PRD.md` +- `docs/components/connectors/task-tracking/jira/specs/PRD.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/PRD.md` +- `docs/components/connectors/task-tracking/silver/specs/PRD.md` +- `docs/components/connectors/task-tracking/youtrack/specs/PRD.md` +- `docs/components/connectors/testing/allure/specs/PRD.md` +- `docs/components/connectors/ui-design/domain/specs/PRD.md` +- `docs/components/connectors/ui-design/figma/specs/PRD.md` +- `docs/components/connectors/wiki/confluence/specs/PRD.md` +- `docs/components/connectors/wiki/outline/specs/PRD.md` +- `docs/components/deployment/specs/PRD.md` +- `docs/components/frontend/specs/PRD.md` +- `docs/components/orchestrator/specs/PRD.md` +- `docs/domain/bronze-to-api-e2e/specs/PRD.md` +- `docs/domain/connector/specs/PRD.md` +- `docs/domain/identity-resolution/specs/PRD.md` +- `docs/domain/ingestion/specs/PRD.md` +- `docs/domain/metric-catalog/specs/PRD.md` +- `docs/domain/metric-catalog/specs/PRD_human_readable.md` +- `docs/domain/org-chart/specs/PRD.md` +- `docs/domain/person/specs/PRD.md` +- `inbox/architecture/PRODUCT_SPECIFICATION.md` +- `inbox/architecture/permissions/PERMISSION_PRD.md` +- `inbox/stats/backend/PRD.md` +- `inbox/stats/frontend/PRD.md` + +### DESIGN (54) + +- `docs/components/airbyte-toolkit/specs/DESIGN.md` +- `docs/components/backend/analytics-api/DESIGN.md` +- `docs/components/backend/api-gateway/DESIGN.md` +- `docs/components/backend/api-gateway/bff/DESIGN.md` +- `docs/components/backend/api-gateway/router/DESIGN.md` +- `docs/components/backend/identity-resolution/identity/specs/DESIGN.md` +- `docs/components/backend/specs/DESIGN.md` +- `docs/components/connectors/ai/chatgpt-team/specs/DESIGN.md` +- `docs/components/connectors/ai/claude-admin/specs/DESIGN.md` +- `docs/components/connectors/ai/claude-enterprise/specs/DESIGN.md` +- `docs/components/connectors/ai/claude-team/specs/DESIGN.md` +- `docs/components/connectors/ai/cursor/specs/DESIGN.md` +- `docs/components/connectors/ai/github-copilot/specs/DESIGN.md` +- `docs/components/connectors/ai/jetbrains/specs/DESIGN.md` +- `docs/components/connectors/ai/openai-api/specs/DESIGN.md` +- `docs/components/connectors/ai/windsurf/specs/DESIGN.md` +- `docs/components/connectors/collaboration/m365/specs/DESIGN.md` +- `docs/components/connectors/collaboration/slack/specs/DESIGN.md` +- `docs/components/connectors/collaboration/zoom/specs/DESIGN.md` +- `docs/components/connectors/collaboration/zulip/specs/DESIGN.md` +- `docs/components/connectors/collaboration/zulip-proxy/specs/DESIGN.md` +- `docs/components/connectors/crm/hubspot/specs/DESIGN.md` +- `docs/components/connectors/crm/salesforce/specs/DESIGN.md` +- `docs/components/connectors/git/bitbucket-server/specs/DESIGN.md` +- `docs/components/connectors/git/github/specs/DESIGN.md` +- `docs/components/connectors/git/gitlab/specs/DESIGN.md` +- `docs/components/connectors/hr-directory/bamboohr/specs/DESIGN.md` +- `docs/components/connectors/hr-directory/ldap/specs/DESIGN.md` +- `docs/components/connectors/hr-directory/ms-entra/specs/DESIGN.md` +- `docs/components/connectors/hr-directory/workday/specs/DESIGN.md` +- `docs/components/connectors/support/jsm/specs/DESIGN.md` +- `docs/components/connectors/support/zendesk/specs/DESIGN.md` +- `docs/components/connectors/task-tracking/jira/specs/DESIGN.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/DESIGN.md` +- `docs/components/connectors/task-tracking/silver/specs/DESIGN.md` +- `docs/components/connectors/task-tracking/youtrack/specs/DESIGN.md` +- `docs/components/connectors/testing/allure/specs/DESIGN.md` +- `docs/components/connectors/ui-design/domain/specs/DESIGN.md` +- `docs/components/connectors/ui-design/figma/specs/DESIGN.md` +- `docs/components/connectors/wiki/confluence/specs/DESIGN.md` +- `docs/components/connectors/wiki/outline/specs/DESIGN.md` +- `docs/components/deployment/specs/DESIGN.md` +- `docs/components/frontend/specs/DESIGN.md` +- `docs/components/orchestrator/specs/DESIGN.md` +- `docs/domain/bronze-to-api-e2e/specs/DESIGN.md` +- `docs/domain/connector/specs/DESIGN.md` +- `docs/domain/identity-resolution/specs/DESIGN.md` +- `docs/domain/ingestion/specs/DESIGN.md` +- `docs/domain/ingestion-data-flow/specs/DESIGN.md` +- `docs/domain/metric-catalog/specs/DESIGN.md` +- `docs/domain/org-chart/specs/DESIGN.md` +- `docs/domain/person/specs/DESIGN.md` +- `inbox/architecture/permissions/PERMISSION_DESIGN.md` +- `src/backend/services/api-gateway/specs/DESIGN.md` + +### ADR (77) + +- `docs/components/airbyte-toolkit/specs/ADR/0001-version-driven-reconcile.md` +- `docs/components/airbyte-toolkit/specs/ADR/0002-adoption-of-existing-resources.md` +- `docs/components/airbyte-toolkit/specs/ADR/0003-credential-rotation-no-env.md` +- `docs/components/airbyte-toolkit/specs/ADR/0004-cluster-config-via-configmap.md` +- `docs/components/airbyte-toolkit/specs/ADR/0005-connection-name-as-argo-identifier.md` +- `docs/components/airbyte-toolkit/specs/ADR/0006-cron-self-run-with-file-persistent-logs.md` +- `docs/components/airbyte-toolkit/specs/ADR/0007-required-fields-in-descriptor-not-example.md` +- `docs/components/airbyte-toolkit/specs/ADR/0008-auto-trigger-sync-on-data-change.md` +- `docs/components/airbyte-toolkit/specs/ADR/0009-airbyte-workspace-as-namespace.md` +- `docs/components/airbyte-toolkit/specs/ADR/0010-nocode-via-builder-projects.md` +- `docs/components/airbyte-toolkit/specs/ADR/0011-cdk-prebuilt-images.md` +- `docs/components/airbyte-toolkit/specs/ADR/0012-destination-owned-by-reconcile.md` +- `docs/components/airbyte-toolkit/specs/ADR/0013-oauth-everywhere.md` +- `docs/components/airbyte-toolkit/specs/ADR/0014-enrich-image-in-descriptor.md` +- `docs/components/airbyte-toolkit/specs/ADR/0015-semver-and-full-refresh.md` +- `docs/components/airbyte-toolkit/specs/ADR/0016-descriptor-images-block.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0002-read-from-mariadb-persons.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0003-latest-per-source-semantics.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0004-lowercase-email-lookup.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0005-tenant-context-strategy.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0006-display-name-split-fallback.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0007-value-type-routing.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0008-bamboohr-identity-inputs-extension.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0009-post-profile-with-uniqueness-invariant.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0010-org-chart-cache.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0011-persons-relax-uniqueness-and-collation.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0012-admin-only-orgchart-visibility-reads.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0013-roles-hard-delete-with-in-use-guard.md` +- `docs/components/backend/identity-resolution/identity/specs/ADR/0014-last-admin-protection.md` +- `docs/components/backend/specs/ADR/0001-redpanda-over-kafka.md` +- `docs/components/connectors/ai/chatgpt-team/specs/ADR/ADR-001-browser-proxy-architecture.md` +- `docs/components/connectors/ai/claude-admin/specs/ADR/0001-cursor-granularity-boundary-fix.md` +- `docs/components/connectors/ai/cursor/specs/ADR/0001-usage-events-dedup-key.md` +- `docs/components/connectors/ai/github-copilot/specs/ADR/0001-python-cdk-over-declarative-manifest.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/ADR/ADR-001-rust-single-binary.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/ADR/ADR-002-core-io-split.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/ADR/ADR-003-ddl-owned-by-dbt.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/ADR/ADR-004-cursorless-incremental.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/ADR/ADR-005-event-id-traceability.md` +- `docs/components/connectors/task-tracking/silver/jira/specs/ADR/ADR-006-event-kind-column.md` +- `docs/components/connectors/task-tracking/youtrack/specs/ADR/ADR-001-project-scoped-custom-fields.md` +- `docs/components/connectors/task-tracking/youtrack/specs/ADR/ADR-002-activitiespage-cursor-pagination.md` +- `docs/components/connectors/task-tracking/youtrack/specs/ADR/ADR-003-no-whitelist-full-ingestion.md` +- `docs/components/deployment/specs/ADR/0001-chart-publishing-on-merge.md` +- `docs/domain/connector/specs/ADR/0001-connector-integration-protocol.md` +- `docs/domain/connector/specs/ADR/0002-connector-responsibility-scope.md` +- `docs/domain/connector/specs/ADR/0003-connector-message-protocol.md` +- `docs/domain/identity-resolution/specs/ADR/0002-stable-person-id-via-persons-observations.md` +- `docs/domain/ingestion/specs/ADR/0001-kestra-over-airflow.md` +- `docs/domain/ingestion/specs/ADR/0002-argo-over-kestra.md` +- `docs/domain/ingestion/specs/ADR/0003-k8s-secrets-credentials.md` +- `docs/domain/ingestion/specs/ADR/0006-service-owned-migrations.md` +- `docs/domain/ingestion/specs/ADR/0007-fresh-cluster-placeholders.md` +- `docs/domain/ingestion/specs/ADR/0008-clickhouse-system-logs-ttl.md` +- `docs/domain/ingestion-data-flow/specs/ADR/0001-rmt-with-version-and-unique-key.md` +- `docs/domain/ingestion-data-flow/specs/ADR/0002-promote-bronze-to-rmt.md` +- `docs/domain/ingestion-data-flow/specs/ADR/0003-ephemeral-rust-passthrough.md` +- `docs/domain/ingestion-data-flow/specs/ADR/0004-unique-key-formula.md` +- `docs/domain/ingestion-data-flow/specs/ADR/0005-data-quality-checks-as-dbt-tests.md` +- `docs/domain/metric-catalog/specs/ADR/ADR-001-query-catalog-junction-fk.md` +- `docs/domain/metric-catalog/specs/ADR/ADR-002-metric-key-on-wire-for-fe-bridge.md` +- `docs/domain/metric-catalog/specs/ADR/ADR-003-link-map-on-catalog-read-response.md` +- `docs/domain/person/specs/ADR/0001-shared-unmapped-table.md` +- `docs/shared/glossary/ADR/0001-uuidv7-primary-key.md` +- `docs/shared/glossary/ADR/0002-database-field-conventions.md` +- `docs/shared/glossary/ADR/0003-insight-prefixed-tenant-id.md` +- `inbox/architecture/permissions/ADR_001_ROLE_SCOPE_GRANT.md` +- `inbox/architecture/permissions/ADR_002_WORKSPACE_ISOLATION.md` +- `inbox/stats/backend/ADR-001-backend-framework-django.md` +- `inbox/stats/backend/ADR-002-user-database-sqlite.md` +- `inbox/stats/backend/ADR-003-analytics-database-clickhouse.md` +- `inbox/stats/backend/ADR-004-authentication-oidc-zta-passport.md` +- `inbox/stats/frontend/ADR-001-ui-framework-react.md` +- `inbox/stats/frontend/ADR-002-styling-framework-tailwindcss.md` +- `inbox/stats/frontend/ADR-003-chart-library-recharts.md` +- `inbox/stats/frontend/ADR-004-type-system-typescript.md` +- `inbox/stats/frontend/ADR-005-build-tooling-create-react-app.md` + +### DECOMPOSITION (5) + +- `docs/components/backend/specs/DECOMPOSITION.md` +- `docs/components/connectors/task-tracking/youtrack/specs/DECOMPOSITION.md` +- `docs/domain/bronze-to-api-e2e/specs/DECOMPOSITION.md` +- `docs/domain/identity-resolution/specs/DECOMPOSITION.md` +- `docs/domain/ingestion/specs/DECOMPOSITION.md` + +### FEATURE (6) + +- `docs/components/airbyte-toolkit/specs/feature-reconcile/FEATURE.md` +- `docs/components/connectors/ai/claude-team/specs/FEATURE.md` +- `docs/components/connectors/collaboration/zulip-proxy/specs/FEATURE.md` +- `docs/domain/bronze-to-api-e2e/specs/feature-csv-rig/FEATURE.md` +- `docs/domain/bronze-to-api-e2e/specs/feature-yaml-rig/FEATURE.md` +- `docs/domain/ingestion/specs/feature-k8s-secret-credentials/FEATURE.md` + +### TEST-SCENARIOS (1) + +- `docs/components/connectors/task-tracking/jira/specs/test-scenarios.md` + +### RULES (4) + +- `cypilot/config/rules/architecture.md` +- `cypilot/config/rules/code-conventions.md` +- `cypilot/config/rules/conventions.md` +- `cypilot/config/rules/patterns.md` + +### README (90) + +- `README.md` +- `docs/components/backend/README.md` +- `docs/components/backend/identity-resolution/identity/README.md` +- `docs/components/connectors/README.md` +- `docs/components/connectors/ai/README.md` +- `docs/components/connectors/ai/chatgpt-team/README.md` +- `docs/components/connectors/ai/claude-admin/README.md` +- `docs/components/connectors/ai/cursor/README.md` +- `docs/components/connectors/ai/github-copilot/README.md` +- `docs/components/connectors/ai/jetbrains/README.md` +- `docs/components/connectors/ai/openai-api/README.md` +- `docs/components/connectors/ai/windsurf/README.md` +- `docs/components/connectors/collaboration/README.md` +- `docs/components/connectors/collaboration/m365/README.md` +- `docs/components/connectors/collaboration/slack/README.md` +- `docs/components/connectors/collaboration/zoom/README.md` +- `docs/components/connectors/collaboration/zulip/README.md` +- `docs/components/connectors/crm/README.md` +- `docs/components/connectors/crm/hubspot/README.md` +- `docs/components/connectors/crm/salesforce/README.md` +- `docs/components/connectors/git/README.md` +- `docs/components/connectors/git/bitbucket-server/README.md` +- `docs/components/connectors/git/github/README.md` +- `docs/components/connectors/git/gitlab/README.md` +- `docs/components/connectors/hr-directory/README.md` +- `docs/components/connectors/hr-directory/bamboohr/README.md` +- `docs/components/connectors/hr-directory/ldap/README.md` +- `docs/components/connectors/hr-directory/ms-entra/README.md` +- `docs/components/connectors/hr-directory/workday/README.md` +- `docs/components/connectors/support/README.md` +- `docs/components/connectors/support/jsm/README.md` +- `docs/components/connectors/support/zendesk/README.md` +- `docs/components/connectors/task-tracking/README.md` +- `docs/components/connectors/task-tracking/jira/README.md` +- `docs/components/connectors/task-tracking/youtrack/README.md` +- `docs/components/connectors/task-tracking/youtrack/specs/README.md` +- `docs/components/connectors/testing/allure/README.md` +- `docs/components/connectors/ui-design/README.md` +- `docs/components/connectors/ui-design/domain/README.md` +- `docs/components/connectors/ui-design/figma/README.md` +- `docs/components/connectors/wiki/README.md` +- `docs/components/connectors/wiki/confluence/README.md` +- `docs/components/connectors/wiki/outline/README.md` +- `docs/components/deployment/gitops/README.md` +- `docs/components/frontend/README.md` +- `docs/components/orchestrator/README.md` +- `docs/domain/README.md` +- `docs/domain/connector/README.md` +- `docs/domain/identity-resolution/README.md` +- `docs/domain/ingestion/README.md` +- `docs/domain/org-chart/README.md` +- `docs/domain/person/README.md` +- `docs/shared/api-guideline/README.md` +- `docs/shared/glossary/README.md` +- `src/backend/plugins/oidc-authn-plugin/README.md` +- `src/backend/services/api-gateway/README.md` +- `src/ingestion/README.md` +- `src/ingestion/connectors/ai/chatgpt-team/README.md` +- `src/ingestion/connectors/ai/claude-admin/README.md` +- `src/ingestion/connectors/ai/claude-enterprise/README.md` +- `src/ingestion/connectors/ai/claude-team/README.md` +- `src/ingestion/connectors/ai/cursor/README.md` +- `src/ingestion/connectors/ai/github-copilot/README.md` +- `src/ingestion/connectors/ai/openai/README.md` +- `src/ingestion/connectors/collaboration/m365/README.md` +- `src/ingestion/connectors/collaboration/slack/README.md` +- `src/ingestion/connectors/collaboration/zoom/README.md` +- `src/ingestion/connectors/collaboration/zulip-proxy/README.md` +- `src/ingestion/connectors/crm/hubspot/README.md` +- `src/ingestion/connectors/crm/salesforce/README.md` +- `src/ingestion/connectors/git/github-v2/README.md` +- `src/ingestion/connectors/git/gitlab/README.md` +- `src/ingestion/connectors/hr-directory/bamboohr/README.md` +- `src/ingestion/connectors/hr-directory/ms-entra/README.md` +- `src/ingestion/connectors/hr-directory/workday/README.md` +- `src/ingestion/connectors/support/zendesk/README.md` +- `src/ingestion/connectors/task-tracking/jira/README.md` +- `src/ingestion/connectors/task-tracking/jira/enrich/README.md` +- `src/ingestion/connectors/task-tracking/youtrack/README.md` +- `src/ingestion/connectors/ui-design/figma/README.md` +- `src/ingestion/connectors/wiki/confluence/README.md` +- `src/ingestion/connectors/wiki/outline/README.md` +- `src/ingestion/dbt/tests/README.md` +- `src/ingestion/dbt/tests/collaboration/README.md` +- `src/ingestion/dbt/tests/task/README.md` +- `src/ingestion/dbt/tests/wiki/README.md` +- `src/ingestion/reconcile-connectors/README.md` +- `src/ingestion/silver/git/README.md` +- `src/ingestion/tests/e2e/README.md` +- `src/ingestion/tools/declarative-connector/README.md` + +### OTHER (62) + +- `AGENTS.md` +- `CLAUDE.md` +- `CONTRIBUTING.md` +- `docs/components/airbyte-toolkit/specs/AIRBYTE-DEPLOY-NOTES.md` +- `docs/components/backend/specs/analytics-views-api.md` +- `docs/components/connectors/ai/cursor/cursor.md` +- `docs/components/connectors/ai/github-copilot/github-copilot.md` +- `docs/components/connectors/ai/jetbrains/jetbrains.md` +- `docs/components/connectors/ai/windsurf/windsurf.md` +- `docs/components/connectors/collaboration/slack/slack.md` +- `docs/components/connectors/collaboration/zoom/zoom.md` +- `docs/components/connectors/collaboration/zulip/zulip.md` +- `docs/components/connectors/collaboration/zulip-proxy/REPRODUCIBILITY-LOG.md` +- `docs/components/connectors/crm/hubspot/hubspot.md` +- `docs/components/connectors/hr-directory/ldap/ldap.md` +- `docs/components/connectors/hr-directory/workday/workday.md` +- `docs/components/connectors/support/jsm/jsm.md` +- `docs/components/connectors/support/zendesk/zendesk.md` +- `docs/components/connectors/task-tracking/jira/jira.md` +- `docs/components/connectors/task-tracking/specs/task-metrics-map.md` +- `docs/components/connectors/task-tracking/youtrack/youtrack.md` +- `docs/components/connectors/testing/allure/allure.md` +- `docs/components/connectors/ui-design/figma/figma.md` +- `docs/components/connectors/wiki/confluence/confluence.md` +- `docs/components/connectors/wiki/outline/outline.md` +- `docs/components/deployment/specs/sop/connector-image-rebuild.md` +- `docs/shared/api-guideline/API.md` +- `docs/shared/api-guideline/BATCH.md` +- `docs/shared/api-guideline/QUERYING.md` +- `docs/shared/api-guideline/STATUS_CODES.md` +- `docs/shared/api-guideline/VERSIONING.md` +- `inbox/CONNECTORS_REFERENCE.md` +- `inbox/IDENTITY_RESOLUTION.md` +- `inbox/architecture/CONNECTORS_ARCHITECTURE.md` +- `inbox/architecture/CONNECTOR_AUTOMATION.md` +- `inbox/architecture/EXAMPLE_IDENTITY_PIPELINE.md` +- `inbox/architecture/IDENTITY_RESOLUTION_V2.md` +- `inbox/architecture/IDENTITY_RESOLUTION_V3.md` +- `inbox/architecture/IDENTITY_RESOLUTION_V4.md` +- `inbox/architecture/STORAGE_TECHNOLOGY_EVALUATION.md` +- `inbox/streams/raw_cursor/cursor_daily_usage.md` +- `inbox/streams/raw_cursor/cursor_events.md` +- `inbox/streams/raw_cursor/cursor_events_token_usage.md` +- `inbox/streams/raw_git/git_author.md` +- `inbox/streams/raw_git/git_branch.md` +- `inbox/streams/raw_git/git_commit.md` +- `inbox/streams/raw_git/git_file.md` +- `inbox/streams/raw_git/git_loc.md` +- `inbox/streams/raw_git/git_num_stat.md` +- `inbox/streams/raw_git/git_repo.md` +- `inbox/streams/raw_ms365/ms365_email_activity.md` +- `inbox/streams/raw_ms365/ms365_onedrive_activity.md` +- `inbox/streams/raw_ms365/ms365_sharepoint_activity.md` +- `inbox/streams/raw_ms365/ms365_teams_activity.md` +- `inbox/streams/raw_youtrack/youtrack_issue.md` +- `inbox/streams/raw_youtrack/youtrack_issue_history.md` +- `inbox/streams/raw_youtrack/youtrack_user.md` +- `inbox/streams/raw_zulip/zulip_messages.md` +- `inbox/streams/raw_zulip/zulip_users.md` +- `inbox/streams/stream_communication/communication_events.md` +- `inbox/streams/stream_task_tracker/table_example.md` +- `inbox/streams/stream_task_tracker/task_tracker_activities.md` + +### GENERATED (1) + +- `docs/DOCS_MAP.md` + diff --git a/scripts/ci/docs_map.py b/scripts/ci/docs_map.py new file mode 100644 index 000000000..f2096827f --- /dev/null +++ b/scripts/ci/docs_map.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""docs_map.py — documentation map generator + MANDATORY documentation gate. + +Generates docs/DOCS_MAP.md: every markdown in the repo, categorized +(PRD / DESIGN / ADR / DECOMPOSITION / FEATURE / ...), per-component coverage, +and code-mapping status (registered in cypilot/config/artifacts.toml). + +Gate (--check), enforced as a required CI status check: + 1. Every component under docs/components/* and docs/domain/* MUST have + specs/PRD.md and specs/DESIGN.md. -> no PRD+DESIGN, no pass. + 2. PRD and DESIGN MUST follow the single canonical template + (cypilot/config/kits/sdlc/artifacts/{PRD,DESIGN}/template.md). + Same structure everywhere — no differences. + 3. Every PRD/DESIGN/ADR must be registered in artifacts.toml (mapped to code). + 4. Exceptions only via docs/.docs-gate-waivers (reason + expiry; expired + waivers fail the gate by themselves). + +Usage: docs_map.py [--check] (always regenerates docs/DOCS_MAP.md) +""" +import re, sys, datetime, pathlib + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DOCS = ROOT / "docs" +MAP = DOCS / "DOCS_MAP.md" +WAIVERS = DOCS / ".docs-gate-waivers" +ARTIFACTS = ROOT / "cypilot/config/artifacts.toml" + +# Canonical section sets — mirror cypilot/config/kits/sdlc/artifacts/*/template.md. +PRD_SECTIONS = ["Overview", "Actors", "Operational Concept", "Scope", + "Functional Requirements", "Non-Functional Requirements", + "Use Cases", "Acceptance Criteria", "Dependencies", "Assumptions"] +DESIGN_SECTIONS = ["Architecture Overview", "Principles & Constraints", + "Technical Architecture", "Traceability"] + +SCAN_GLOBS = ["docs/**/*.md", "src/**/specs/**/*.md", "src/**/README.md", + "inbox/**/*.md", "cypilot/config/rules/*.md", "*.md"] +SKIP_PARTS = {"node_modules", ".git", "target", ".hive-tmp"} + +def kind_of(p: pathlib.Path) -> str: + n, parts = p.name.upper(), {q.upper() for q in p.parts} + if n == "DOCS_MAP.MD": return "GENERATED" + if n.startswith("PRD") or n.endswith("_PRD.MD") or "PRODUCT_SPECIFICATION" in n: return "PRD" + if n.startswith("DESIGN") or n.endswith("_DESIGN.MD"): return "DESIGN" + if "ADR" in parts or n.startswith("ADR") or re.match(r"^\d{4}-", p.name): return "ADR" + if n.startswith("DECOMPOSITION"): return "DECOMPOSITION" + if n.startswith("FEATURE") or "FEATURES" in parts: return "FEATURE" + if "TEST" in n and "SCENARIO" in n: return "TEST-SCENARIOS" + if n == "README.MD": return "README" + if "RUNBOOK" in n: return "RUNBOOK" + if "RULES" in parts or p.parent.name == "rules": return "RULES" + return "OTHER" + +def headings(p: pathlib.Path): + try: text = p.read_text(errors="replace") + except OSError: return [] + return [m.group(1).strip() for m in re.finditer(r"^##\s+(.+)$", text, re.M)] + +def _norm_heading(h: str) -> str: + # Drop a leading section number ("1. ", "2) ") and normalize whitespace/case. + return re.sub(r"^\s*\d+[.)]?\s+", "", h.strip()).lower() + +def template_ok(p: pathlib.Path, required): + # Structural match against actual `##` headings (numbering-tolerant), NOT a + # substring search over prose — a required section is satisfied only by a + # real heading that equals it or extends it ("Operational Concept & + # Environment" satisfies "Operational Concept"). + hs = [_norm_heading(h) for h in headings(p)] + out = [] + for s in required: + s_norm = s.strip().lower() + if not any(h == s_norm or h.startswith(s_norm) for h in hs): + out.append(s) + return out + +def load_waivers(): + out = {} + if WAIVERS.exists(): + for line in WAIVERS.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): continue + parts = [x.strip() for x in line.split("|")] + if len(parts) == 3: out[parts[0]] = (parts[1], parts[2]) + return out + +def main(): + check = "--check" in sys.argv + today = datetime.date.today().isoformat() + art_reg = ARTIFACTS.read_text() if ARTIFACTS.exists() else "" + waivers, errors, warns = load_waivers(), [], [] + + files = [] + for g in SCAN_GLOBS: + for p in ROOT.glob(g): + if p.is_file() and not (SKIP_PARTS & set(p.parts)): + files.append(p) + files = sorted(set(files)) + inv = {} + for p in files: + inv.setdefault(kind_of(p), []).append(p.relative_to(ROOT)) + + # per-component coverage + comps = sorted([d for base in ("docs/components", "docs/domain") + for d in (ROOT / base).glob("*/") if d.is_dir()]) + rows = [] + for c in comps: + rel = str(c.relative_to(ROOT)).rstrip("/") + prd, des = c / "specs/PRD.md", c / "specs/DESIGN.md" + adrs = len(list(c.glob("specs/ADR/*.md"))) + missing = [n for n, f in (("PRD", prd), ("DESIGN", des)) if not f.exists()] + tmpl_bad = [] + if prd.exists(): + miss = template_ok(prd, PRD_SECTIONS) + if miss: tmpl_bad.append(f"PRD lacks: {', '.join(miss)}") + if des.exists(): + miss = template_ok(des, DESIGN_SECTIONS) + if miss: tmpl_bad.append(f"DESIGN lacks: {', '.join(miss)}") + mapped = all(str(f.relative_to(ROOT)) in art_reg for f in (prd, des) if f.exists()) + status = "✅" + for problem in ([f"missing specs/{m}.md" for m in missing] + tmpl_bad + + ([] if mapped else ["not registered in artifacts.toml"])): + key = rel + if key in waivers: + reason, expiry = waivers[key] + try: + expired = datetime.date.fromisoformat(expiry) < datetime.date.today() + except ValueError: + errors.append(f"{rel}: waiver has invalid expiry '{expiry}' (expected YYYY-MM-DD) — {problem}") + status = "❌" + continue + if expired: + errors.append(f"{rel}: waiver EXPIRED ({expiry}) — {problem}") + status = "❌" + else: + warns.append(f"{rel}: WAIVED until {expiry} ({reason}) — {problem}") + status = f"⚠️ waived→{expiry}" + else: + errors.append(f"{rel}: {problem}") + status = "❌" + rows.append((rel, "✅" if prd.exists() else "❌", "✅" if des.exists() else "❌", + adrs, "✅" if mapped else "❌", status)) + + order = ["PRD", "DESIGN", "ADR", "DECOMPOSITION", "FEATURE", "TEST-SCENARIOS", + "RULES", "RUNBOOK", "README", "OTHER", "GENERATED"] + out = ["", + "# Documentation Map", "", + f"Total markdown files: **{len(files)}**.", "", + "## Coverage gate — components & domains", + "", "| Unit | PRD | DESIGN | ADRs | Mapped (artifacts.toml) | Gate |", "|---|---|---|---|---|---|"] + out += [f"| `{r[0]}` | {r[1]} | {r[2]} | {r[3]} | {r[4]} | {r[5]} |" for r in rows] + out += ["", "## Inventory by category", ""] + for k in order: + if k not in inv: continue + out.append(f"### {k} ({len(inv[k])})\n") + out += [f"- `{p}`" for p in inv[k]] + out.append("") + MAP.write_text("\n".join(out) + "\n") + print(f"wrote {MAP.relative_to(ROOT)} ({len(files)} files, {len(rows)} units)") + + for w in warns: print(f" WAIVED {w}") + if check and errors: + print(f"\n✗ documentation gate FAILED ({len(errors)}):") + for e in errors: print(f" ✗ {e}") + sys.exit(1) + if check: print("✓ documentation gate passed") + +if __name__ == "__main__": + main()