diff --git a/charts/insight/templates/ingestion/dbt-source-freshness.yaml b/charts/insight/templates/ingestion/dbt-source-freshness.yaml
new file mode 100644
index 000000000..a2f2dc9f9
--- /dev/null
+++ b/charts/insight/templates/ingestion/dbt-source-freshness.yaml
@@ -0,0 +1,632 @@
+{{- if and .Values.ingestion.templates.enabled .Values.ingestion.freshness.enabled }}
+{{- /*
+Bronze freshness check: runs `dbt source freshness` against ClickHouse, reads
+the resulting `target/sources.json`, and emits a notification when any source
+is `warn` or `error`. Single workflow covers every connector — new bronze
+sources are picked up automatically as soon as they appear in any
+`schema.yml` (defaults inherited from `dbt_project.yml`).
+
+How to reason about it operationally:
+ - `pass` : MAX(_airbyte_extracted_at) within the warn window. Sync OK.
+ - `warn` : older than warn_after but younger than error_after. One missed
+ run; alert lower priority.
+ - `error` : older than error_after. Multiple missed runs / source dead.
+ Page on these.
+
+Two failure modes this catches:
+ 1. Argo CronWorkflow didn't run / couldn't talk to Airbyte → no new rows.
+ 2. Sync ran but source API returned 0 rows (token expired, API outage).
+
+It does NOT distinguish (1) from (2) — for that we need an independent
+"last successful Airbyte job" signal (future work, see MONITORING.md).
+*/}}
+---
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+ name: dbt-source-freshness
+ labels:
+ {{- include "insight.labels" . | nindent 4 }}
+ app.kubernetes.io/component: ingestion-template
+spec:
+ serviceAccountName: argo-workflow
+ entrypoint: run
+ templates:
+ - name: run
+ inputs:
+ parameters:
+ # Selector applied to `dbt source freshness`. Default covers every
+ # bronze_* source declared anywhere in the project. Override to
+ # narrow scope (e.g., per-tenant smoke check).
+ - name: dbt_select
+ value: "source:*"
+ - name: toolbox_image
+ - name: clickhouse_host
+ - name: clickhouse_port
+ - name: clickhouse_user
+ # Identification labels for routing when one bot fans out to
+ # multiple deployments. `cluster` = installation tier;
+ # `tenant` = customer/workspace. Both empty by default.
+ - name: cluster
+ value: ""
+ - name: tenant
+ value: ""
+ # Notification delivery. `driver` selects the parser branch
+ # ("" / webhook / zulip / slack / email / future). Credential
+ # material (webhook URL, SMTP password) is **not** a workflow
+ # parameter — it is read from a k8s Secret via `secretKeyRef`
+ # on the env block below so the token never lands in rendered
+ # manifests or the Argo UI. Plain per-driver settings
+ # (channel, stream, smtp host/port, from/to) come through as
+ # ordinary parameters because they aren't credentials.
+ - name: notification_driver
+ value: ""
+ # zulip driver
+ - name: notification_zulip_stream
+ value: ""
+ - name: notification_zulip_topic
+ value: ""
+ # slack driver
+ - name: notification_slack_channel
+ value: ""
+ # email driver
+ - name: notification_email_smtp_host
+ value: ""
+ - name: notification_email_smtp_port
+ value: "587"
+ - name: notification_email_smtp_username
+ value: ""
+ - name: notification_email_smtp_starttls
+ value: "true"
+ - name: notification_email_from
+ value: ""
+ - name: notification_email_to
+ value: ""
+ - name: notification_email_subject_prefix
+ value: "[ingestion-freshness]"
+ # NOTE: SLA thresholds (warn_after / error_after) are NOT passed
+ # from here. They live as literals on each bronze source's
+ # `freshness:` block in the connector `schema.yml` files (see
+ # constructorfabric/insight#1346 / EPIC #1321). This workflow only
+ # *runs* the check and acts on the verdict in `target/sources.json`;
+ # it does not own the threshold values. Recommended per-tier
+ # thresholds (default / event / report) are documented in
+ # `src/ingestion/MONITORING.md` for whoever sets the declarations.
+ # 20 min: dbt source freshness issues one CH query per source. With
+ # the current connector set this finishes in ~2 min; the headroom
+ # is for connector growth and CH cold-start cases.
+ activeDeadlineSeconds: 1200
+ script:
+ image: {{ `"{{inputs.parameters.toolbox_image}}"` }}
+ imagePullPolicy: Always
+ command: [bash]
+ env:
+ - name: CLICKHOUSE_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ required "clickhouse.passwordSecret.name is required" .Values.clickhouse.passwordSecret.name | quote }}
+ key: {{ required "clickhouse.passwordSecret.key is required" .Values.clickhouse.passwordSecret.key | quote }}
+ - name: DBT_SELECT
+ value: {{ `"{{inputs.parameters.dbt_select}}"` }}
+ - name: CLICKHOUSE_HOST
+ value: {{ `"{{inputs.parameters.clickhouse_host}}"` }}
+ - name: CLICKHOUSE_PORT
+ value: {{ `"{{inputs.parameters.clickhouse_port}}"` }}
+ - name: CLICKHOUSE_USER
+ value: {{ `"{{inputs.parameters.clickhouse_user}}"` }}
+ - name: CLUSTER
+ value: {{ `"{{inputs.parameters.cluster}}"` }}
+ - name: TENANT
+ value: {{ `"{{inputs.parameters.tenant}}"` }}
+ - name: NOTIFICATION_DRIVER
+ value: {{ `"{{inputs.parameters.notification_driver}}"` }}
+ - name: NOTIFICATION_ZULIP_STREAM
+ value: {{ `"{{inputs.parameters.notification_zulip_stream}}"` }}
+ - name: NOTIFICATION_ZULIP_TOPIC
+ value: {{ `"{{inputs.parameters.notification_zulip_topic}}"` }}
+ - name: NOTIFICATION_SLACK_CHANNEL
+ value: {{ `"{{inputs.parameters.notification_slack_channel}}"` }}
+ - name: NOTIFICATION_EMAIL_SMTP_HOST
+ value: {{ `"{{inputs.parameters.notification_email_smtp_host}}"` }}
+ - name: NOTIFICATION_EMAIL_SMTP_PORT
+ value: {{ `"{{inputs.parameters.notification_email_smtp_port}}"` }}
+ - name: NOTIFICATION_EMAIL_SMTP_USERNAME
+ value: {{ `"{{inputs.parameters.notification_email_smtp_username}}"` }}
+ - name: NOTIFICATION_EMAIL_SMTP_STARTTLS
+ value: {{ `"{{inputs.parameters.notification_email_smtp_starttls}}"` }}
+ - name: NOTIFICATION_EMAIL_FROM
+ value: {{ `"{{inputs.parameters.notification_email_from}}"` }}
+ - name: NOTIFICATION_EMAIL_TO
+ value: {{ `"{{inputs.parameters.notification_email_to}}"` }}
+ - name: NOTIFICATION_EMAIL_SUBJECT_PREFIX
+ value: {{ `"{{inputs.parameters.notification_email_subject_prefix}}"` }}
+ # Credential material — sourced from the k8s Secret of the
+ # active driver. Rendered conditionally so an unconfigured
+ # cluster (driver="") does not try to resolve a non-existent
+ # Secret. The parser treats a missing env var as "log only".
+ # `NOTIFICATION_URL` for URL-based drivers (webhook/zulip/slack);
+ # `NOTIFICATION_EMAIL_SMTP_PASSWORD` for the email driver.
+ {{- $n := .Values.ingestion.freshness.notification }}
+ {{- if eq $n.driver "webhook" }}
+ - name: NOTIFICATION_URL
+ valueFrom:
+ secretKeyRef:
+ name: {{ required "ingestion.freshness.notification.webhook.urlSecret.name is required when driver=webhook" $n.webhook.urlSecret.name | quote }}
+ key: {{ ($n.webhook.urlSecret.key | default "url") | quote }}
+ {{- else if eq $n.driver "zulip" }}
+ - name: NOTIFICATION_URL
+ valueFrom:
+ secretKeyRef:
+ name: {{ required "ingestion.freshness.notification.zulip.urlSecret.name is required when driver=zulip" $n.zulip.urlSecret.name | quote }}
+ key: {{ ($n.zulip.urlSecret.key | default "url") | quote }}
+ {{- else if eq $n.driver "slack" }}
+ - name: NOTIFICATION_URL
+ valueFrom:
+ secretKeyRef:
+ name: {{ required "ingestion.freshness.notification.slack.urlSecret.name is required when driver=slack" $n.slack.urlSecret.name | quote }}
+ key: {{ ($n.slack.urlSecret.key | default "url") | quote }}
+ {{- else if eq $n.driver "teams" }}
+ - name: NOTIFICATION_URL
+ valueFrom:
+ secretKeyRef:
+ name: {{ required "ingestion.freshness.notification.teams.urlSecret.name is required when driver=teams" $n.teams.urlSecret.name | quote }}
+ key: {{ ($n.teams.urlSecret.key | default "url") | quote }}
+ {{- else if eq $n.driver "email" }}
+ {{- if $n.email.smtp.passwordSecret.name }}
+ - name: NOTIFICATION_EMAIL_SMTP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ $n.email.smtp.passwordSecret.name | quote }}
+ key: {{ ($n.email.smtp.passwordSecret.key | default "password") | quote }}
+ {{- end }}
+ {{- end }}
+ source: |
+ # SECURITY: every parameter reaches us via env vars only — no
+ # interpolation into shell or YAML source. Same convention as
+ # dbt-run.yaml.
+ set -euo pipefail
+ cd /ingestion/dbt
+ python3 - <<'PY'
+ import os, yaml
+ profile = {
+ "ingestion": {
+ "target": "k8s",
+ "outputs": {
+ "k8s": {
+ "type": "clickhouse",
+ "host": os.environ["CLICKHOUSE_HOST"],
+ "port": int(os.environ["CLICKHOUSE_PORT"]),
+ # Sentinel: source freshness reads each source's
+ # declared schema (`bronze_*`), never the profile
+ # default. Pick a name that won't collide if dbt
+ # ever falls back here.
+ "schema": "_freshness_check",
+ "user": os.environ["CLICKHOUSE_USER"],
+ "password": os.environ["CLICKHOUSE_PASSWORD"],
+ "secure": False,
+ "send_receive_timeout": 600,
+ "query_limit": 0,
+ "connect_timeout": 30,
+ }
+ }
+ }
+ }
+ with open("profiles.yml", "w") as f:
+ yaml.safe_dump(profile, f)
+ PY
+
+ # Drop any stale freshness report first. If dbt crashes mid-run we
+ # want exit code 2 below, not a misclassification using a leftover
+ # `target/sources.json` from a prior invocation that still happens
+ # to live in the toolbox image's working directory.
+ rm -f target/sources.json
+
+ # Narrow the freshness selector to bronze_* databases that actually
+ # exist in this deployment. The freshness check answers "is the
+ # data we ARE collecting stale?" — it deliberately does not detect
+ # "is a connector that should be deployed actually deployed?".
+ # That is a separate deployment-health concern (Helm probes / sync
+ # workflow status) tracked under a separate issue. Without this
+ # narrowing, a tenant that doesn't run, say, the OpenAI connector
+ # would see one ERROR per OpenAI source on every freshness run.
+ if [ "$DBT_SELECT" = "source:*" ]; then
+ EXISTING_DBS=$(python3 - <<'PY'
+ import os, sys, urllib.parse, urllib.request
+ url = "http://" + os.environ["CLICKHOUSE_HOST"] + ":" + os.environ["CLICKHOUSE_PORT"] + "/"
+ # ClickHouse LIKE has no ESCAPE clause; `startsWith` avoids the
+ # underscore-as-wildcard pitfall. We don't currently have any
+ # database whose name begins with `bronze` but isn't a bronze
+ # source, so the prefix is precise enough.
+ sql = "SELECT name FROM system.databases WHERE startsWith(name, 'bronze_') FORMAT TabSeparated"
+ qs = urllib.parse.urlencode({
+ "user": os.environ["CLICKHOUSE_USER"],
+ "password": os.environ["CLICKHOUSE_PASSWORD"],
+ "query": sql,
+ })
+ try:
+ with urllib.request.urlopen(url + "?" + qs, timeout=30) as resp:
+ print(resp.read().decode("utf-8"))
+ except Exception as e:
+ print(f"freshness: failed to enumerate bronze databases ({e}) — falling back to full source:* selector", file=sys.stderr)
+ sys.exit(0)
+ PY
+ )
+ if [ -n "$EXISTING_DBS" ]; then
+ EFFECTIVE_SELECT=""
+ for db in $EXISTING_DBS; do
+ EFFECTIVE_SELECT="$EFFECTIVE_SELECT source:$db"
+ done
+ EFFECTIVE_SELECT="${EFFECTIVE_SELECT# }"
+ echo "freshness: narrowed selector to deployed bronze databases — $EFFECTIVE_SELECT"
+ else
+ EFFECTIVE_SELECT="$DBT_SELECT"
+ fi
+ else
+ # User-supplied selector is already specific; honor it verbatim.
+ EFFECTIVE_SELECT="$DBT_SELECT"
+ fi
+
+ # `dbt source freshness` exits non-zero when any source breaches
+ # SLA. We swallow the exit code here and re-derive the workflow
+ # outcome from `target/sources.json` below — dbt does not
+ # distinguish warn from error in its exit code, but we want to.
+ # SLA thresholds come from each source's `freshness:` block in the
+ # connector schema.yml (constructorfabric/insight#1346) — this
+ # workflow does not pass or override them.
+ set +e
+ dbt source freshness --profiles-dir . --select "$EFFECTIVE_SELECT"
+ set -e
+
+ # Parse target/sources.json. Exit semantics (set by the script):
+ # 0 — no breaches, OR only `warn` (visible in log; one missed run)
+ # 1 — at least one `error` or `runtime error` (page-worthy)
+ # 2 — `target/sources.json` missing (dbt crashed before checking)
+ # Argo's `failedJobsHistoryLimit` retains the breaching runs.
+ # `set +e` so we keep going to the trap detector even when the
+ # freshness parser exits non-zero (which is the page-worthy case).
+ set +e
+ python3 - <<'PY'
+ import json, os, sys, urllib.request
+
+ path = "target/sources.json"
+ if not os.path.exists(path):
+ print(f"freshness: no {path} produced — dbt likely crashed before checking sources", file=sys.stderr)
+ sys.exit(2)
+
+ with open(path) as f:
+ report = json.load(f)
+
+ # `runtime error` = dbt couldn't check the source at all (CH down,
+ # schema drift, query failure). Treated as page-worthy alongside
+ # `error`, since "we don't know" is worse than "we know it's stale".
+ PAGE_STATUSES = ("error", "runtime error")
+ BREACH_STATUSES = ("warn",) + PAGE_STATUSES
+
+ breaches = []
+ for r in report.get("results", []):
+ status = r.get("status")
+ if status in BREACH_STATUSES:
+ unique_id = r.get("unique_id", "?")
+ max_loaded_at = r.get("max_loaded_at")
+ age_seconds = r.get("max_loaded_at_time_ago_in_s")
+ # Empty-table sentinel: dbt-clickhouse computes
+ # `MAX(
)` over zero rows as NULL, which serialises
+ # as the Unix epoch (`1970-01-01T00:00:00+00:00`) plus
+ # an enormous `age_in_s`. Raw output reads as ~500k
+ # hours of lag, which looks like a parser bug rather
+ # than the real signal ("the source has never received
+ # a single row"). Flag it explicitly.
+ empty = (
+ isinstance(max_loaded_at, str)
+ and max_loaded_at.startswith("1970-01-01")
+ )
+ breaches.append({
+ "source": unique_id,
+ "status": status,
+ "max_loaded_at": "(table is empty)" if empty else max_loaded_at,
+ "age_hours": None if empty else (
+ round(age_seconds / 3600, 1) if age_seconds is not None else None
+ ),
+ "empty": empty,
+ })
+
+ if not breaches:
+ print("freshness: all sources within SLA")
+ sys.exit(0)
+
+ print(f"freshness: {len(breaches)} source(s) breaching SLA:")
+ for b in breaches:
+ if b.get("empty"):
+ detail = "(table is empty — no rows ever ingested)"
+ else:
+ age = f"{b['age_hours']}h" if b['age_hours'] is not None else "n/a"
+ detail = f"max_loaded_at={b['max_loaded_at']}, age={age}"
+ print(f" [{b['status']:>5}] {b['source']} {detail}")
+
+ # Notification fan-out. The driver selects the body format and
+ # any driver-specific transport. URL comes from a k8s Secret
+ # (resolved by the workflow YAML, not by us) and lands in
+ # `NOTIFICATION_URL` only when a driver is configured.
+ #
+ # Adding a new driver is one match arm here + one Helm subblock
+ # in `values.yaml` + one secretKeyRef branch in the workflow
+ # YAML. Nothing else changes.
+ import urllib.parse
+ driver = os.environ.get("NOTIFICATION_DRIVER", "").strip()
+ url = os.environ.get("NOTIFICATION_URL", "").strip()
+ cluster = os.environ.get("CLUSTER", "").strip()
+ tenant = os.environ.get("TENANT", "").strip()
+ # Summary prefix: every label that's set, in a single bracket,
+ # so receivers handling multi-tenant / multi-cluster bots can
+ # route on either dimension without parsing the breach list.
+ # Empty labels drop out entirely — single-deployment installs
+ # see plain "N bronze source(s) breaching ...".
+ _labels = []
+ if cluster: _labels.append(f"cluster={cluster}")
+ if tenant: _labels.append(f"tenant={tenant}")
+ _prefix = f"[{', '.join(_labels)}] " if _labels else ""
+ summary = f"{_prefix}{len(breaches)} bronze source(s) breaching freshness SLA"
+ has_page_status = any(b["status"] in PAGE_STATUSES for b in breaches)
+
+ def _detail(b):
+ if b.get("empty"):
+ return "table is empty (no rows ingested)"
+ age = f"{b['age_hours']}h" if b['age_hours'] is not None else "n/a"
+ return f"max_loaded_at={b['max_loaded_at']}, age={age}"
+
+ def _line_md(b):
+ return f"- `[{b['status']}]` `{b['source']}` — {_detail(b)}"
+
+ def _line_plain(b):
+ return f" [{b['status']}] {b['source']} — {_detail(b)}"
+
+ def _post(req_url, body, headers):
+ # Some receivers sit behind Cloudflare / WAF and reject the
+ # default `Python-urllib/3.x` UA as a bot signature. Set an
+ # explicit, identifying UA so the request is allowed.
+ headers = {**headers, "User-Agent": "insight-freshness-monitor/1.0"}
+ req = urllib.request.Request(req_url, data=body, headers=headers, method="POST")
+ try:
+ with urllib.request.urlopen(req, timeout=10) as resp:
+ print(f"freshness: notification posted via {driver} ({resp.status})")
+ except urllib.error.HTTPError as e:
+ # Surface the upstream's response body — `urllib` only
+ # exposes the status line by default and 4xx responses
+ # from Zulip / Slack / Teams typically carry a JSON
+ # error message that explains the rejection (wrong
+ # API key, stream not subscribed, channel-locked, etc.).
+ reason = ""
+ try:
+ reason = e.read().decode("utf-8", errors="replace")[:400]
+ except Exception:
+ pass
+ raise RuntimeError(f"HTTP {e.code}: {reason}") from None
+
+ def _send_webhook():
+ payload = {
+ "topic": "ingestion-freshness",
+ "cluster": cluster,
+ "tenant": tenant,
+ "summary": summary,
+ "breaches": breaches,
+ }
+ _post(url, json.dumps(payload).encode("utf-8"),
+ {"Content-Type": "application/json"})
+
+ def _send_zulip():
+ # Zulip exposes two incoming-webhook integrations:
+ # /api/v1/external/json — dumps the body as a JSON code
+ # block (a debugging/logging integration, no markdown).
+ # /api/v1/external/slack — Slack-compatible, takes
+ # `{"text": ""}` and renders properly.
+ # The chart's URL is whatever the operator configured. If
+ # they pointed at `.../external/json` we silently rewrite
+ # the path to `.../external/slack` and use Slack mrkdwn so
+ # the message arrives formatted instead of dumped as JSON.
+ stream = os.environ.get("NOTIFICATION_ZULIP_STREAM", "").strip()
+ topic = os.environ.get("NOTIFICATION_ZULIP_TOPIC", "").strip()
+ target = url.replace("/api/v1/external/json", "/api/v1/external/slack", 1)
+ extras = []
+ if stream: extras.append("stream=" + urllib.parse.quote(stream))
+ if topic: extras.append("topic=" + urllib.parse.quote(topic))
+ if extras:
+ target += ("&" if "?" in target else "?") + "&".join(extras)
+ # Slack mrkdwn: single `*` for bold (vs `**` in Zulip
+ # native), backtick spans and bullet lists work the same.
+ def _line_slack(b):
+ return f"• `[{b['status']}]` `{b['source']}` — {_detail(b)}"
+ md = [f"*{summary}*", "", *(_line_slack(b) for b in breaches)]
+ # Zulip's Slack-compatible incoming webhook expects
+ # legacy-Slack form fields: `user_name`, `text`,
+ # `channel_name`. URL query carries stream/topic; the
+ # form fields carry the message itself.
+ body = urllib.parse.urlencode({
+ "user_name": "ingestion-freshness",
+ "channel_name": "freshness",
+ "text": "\n".join(md),
+ })
+ _post(target, body.encode("utf-8"),
+ {"Content-Type": "application/x-www-form-urlencoded"})
+
+ def _send_slack():
+ channel = os.environ.get("NOTIFICATION_SLACK_CHANNEL", "").strip()
+ text = "\n".join([f"*{summary}*", *(_line_md(b) for b in breaches)])
+ payload = {"text": text}
+ if channel:
+ payload["channel"] = channel
+ _post(url, json.dumps(payload).encode("utf-8"),
+ {"Content-Type": "application/json"})
+
+ def _send_teams():
+ theme = "FF0000" if has_page_status else "FFA500"
+ payload = {
+ "@type": "MessageCard",
+ "@context": "https://schema.org/extensions",
+ "summary": summary,
+ "themeColor": theme,
+ "title": summary,
+ "text": "\n\n".join(_line_md(b) for b in breaches),
+ }
+ _post(url, json.dumps(payload).encode("utf-8"),
+ {"Content-Type": "application/json"})
+
+ def _send_email():
+ import smtplib
+ from email.message import EmailMessage
+ host = os.environ.get("NOTIFICATION_EMAIL_SMTP_HOST", "").strip()
+ port = int(os.environ.get("NOTIFICATION_EMAIL_SMTP_PORT", "587") or "587")
+ username = os.environ.get("NOTIFICATION_EMAIL_SMTP_USERNAME", "").strip()
+ password = os.environ.get("NOTIFICATION_EMAIL_SMTP_PASSWORD", "")
+ starttls = os.environ.get("NOTIFICATION_EMAIL_SMTP_STARTTLS", "true").lower() != "false"
+ sender = os.environ.get("NOTIFICATION_EMAIL_FROM", "").strip()
+ to_csv = os.environ.get("NOTIFICATION_EMAIL_TO", "").strip()
+ prefix = os.environ.get("NOTIFICATION_EMAIL_SUBJECT_PREFIX", "").strip()
+ if not (host and sender and to_csv):
+ raise RuntimeError(
+ "email driver requires notification.email.smtp.host, "
+ "notification.email.from and notification.email.to"
+ )
+ recipients = [r.strip() for r in to_csv.split(",") if r.strip()]
+ msg = EmailMessage()
+ msg["Subject"] = (f"{prefix} {summary}" if prefix else summary).strip()
+ msg["From"] = sender
+ msg["To"] = ", ".join(recipients)
+ msg.set_content("\n".join([summary, "", *(_line_plain(b) for b in breaches)]))
+ smtp_cls = smtplib.SMTP_SSL if port == 465 else smtplib.SMTP
+ with smtp_cls(host, port, timeout=15) as s:
+ if starttls and port != 465:
+ s.starttls()
+ if username:
+ s.login(username, password)
+ s.send_message(msg)
+ print(f"freshness: notification posted via email ({len(recipients)} recipient(s))")
+
+ dispatchers = {
+ "webhook": _send_webhook,
+ "zulip": _send_zulip,
+ "slack": _send_slack,
+ "teams": _send_teams,
+ "email": _send_email,
+ }
+
+ if not driver:
+ print("freshness: notification driver is empty — log only")
+ elif driver not in dispatchers:
+ print(f"freshness: unknown driver '{driver}' — log only", file=sys.stderr)
+ elif driver != "email" and not url:
+ print(f"freshness: driver={driver} but NOTIFICATION_URL is empty — log only", file=sys.stderr)
+ else:
+ try:
+ dispatchers[driver]()
+ except Exception as e:
+ # Notification failure does not change the run's primary
+ # exit code — we'd rather have a noisy log + breached SLA
+ # in workflow status than swallow the upstream failure.
+ print(f"freshness: {driver} notification failed: {e}", file=sys.stderr)
+
+ # Page only on `error` / `runtime error`. Warn-only runs stay
+ # exit 0 so they don't drown on-call in noise; they're still
+ # printed above and POSTed in the payload for triage.
+ sys.exit(1 if any(b["status"] in PAGE_STATUSES for b in breaches) else 0)
+ PY
+ freshness_exit=$?
+ set -e
+
+ # Trap detector — runs after the freshness check itself. Catches
+ # bronze sources whose `_airbyte_extracted_at` anchor is masking
+ # a re-emit / incremental-topup pattern (the freshness check
+ # returns PASS even though the upstream has stopped publishing).
+ # Its findings are *warnings*: logged for visibility, but the
+ # workflow's page/no-page decision is owned by the freshness
+ # check itself — we do not override it.
+ set +e
+ python3 /ingestion/scripts/freshness-trap-detect.py /ingestion/connectors
+ trap_exit=$?
+ set -e
+ if [ "$trap_exit" -eq 1 ]; then
+ echo "freshness-trap-detect: found suspect(s) — review the connector schema.yml for those tables (see src/ingestion/MONITORING.md)"
+ elif [ "$trap_exit" -ne 0 ]; then
+ echo "freshness-trap-detect: script error (exit $trap_exit) — not affecting workflow status" >&2
+ fi
+
+ exit "$freshness_exit"
+---
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+ name: dbt-source-freshness-check
+ namespace: {{ .Release.Namespace }}
+ labels:
+ app.kubernetes.io/component: ingestion-monitoring
+ {{- include "insight.labels" . | nindent 4 }}
+ {{- with .Values.ingestion.reconcile.argoInstanceId }}
+ # When the cluster's Argo workflow-controller runs with `instanceID:`
+ # set, it only schedules CronWorkflows carrying a matching
+ # controller-instanceid label. Without this the cron silently never
+ # ticks (lastScheduled stays `never`). Reuses the reconcile-loop key.
+ workflows.argoproj.io/controller-instanceid: {{ . | quote }}
+ {{- end }}
+spec:
+ schedules:
+ - {{ .Values.ingestion.freshness.schedule | quote }}
+ timezone: UTC
+ concurrencyPolicy: Forbid
+ startingDeadlineSeconds: 600
+ successfulJobsHistoryLimit: 3
+ failedJobsHistoryLimit: 5
+ workflowSpec:
+ serviceAccountName: argo-workflow
+ podGC:
+ strategy: OnPodCompletion
+ {{- with .Values.global.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 6 }}
+ {{- end }}
+ entrypoint: run
+ templates:
+ - name: run
+ steps:
+ - - name: check
+ templateRef:
+ name: dbt-source-freshness
+ template: run
+ arguments:
+ parameters:
+ - name: dbt_select
+ value: {{ .Values.ingestion.freshness.dbtSelect | quote }}
+ - name: toolbox_image
+ value: {{ required "ingestion.toolboxImage is required for the freshness CronWorkflow — pin a tag in values overrides" .Values.ingestion.toolboxImage | quote }}
+ - name: clickhouse_host
+ value: {{ include "insight.clickhouse.host" . | quote }}
+ - name: clickhouse_port
+ value: {{ include "insight.clickhouse.port" . | quote }}
+ - name: clickhouse_user
+ value: {{ .Values.clickhouse.username | quote }}
+ - name: cluster
+ value: {{ .Values.ingestion.freshness.cluster | quote }}
+ - name: tenant
+ value: {{ .Values.ingestion.freshness.tenant | default "" | quote }}
+ {{- $n := .Values.ingestion.freshness.notification }}
+ - name: notification_driver
+ value: {{ $n.driver | default "" | quote }}
+ - name: notification_zulip_stream
+ value: {{ $n.zulip.stream | default "" | quote }}
+ - name: notification_zulip_topic
+ value: {{ $n.zulip.topic | default "" | quote }}
+ - name: notification_slack_channel
+ value: {{ $n.slack.channel | default "" | quote }}
+ - name: notification_email_smtp_host
+ value: {{ $n.email.smtp.host | default "" | quote }}
+ - name: notification_email_smtp_port
+ value: {{ ($n.email.smtp.port | default 587) | quote }}
+ - name: notification_email_smtp_username
+ value: {{ $n.email.smtp.username | default "" | quote }}
+ - name: notification_email_smtp_starttls
+ value: {{ ($n.email.smtp.starttls | default true) | quote }}
+ - name: notification_email_from
+ value: {{ $n.email.from | default "" | quote }}
+ - name: notification_email_to
+ value: {{ $n.email.to | default "" | quote }}
+ - name: notification_email_subject_prefix
+ value: {{ $n.email.subjectPrefix | default "[ingestion-freshness]" | quote }}
+{{- end }}
diff --git a/charts/insight/values.schema.json b/charts/insight/values.schema.json
index 9529843f6..b22e5663f 100644
--- a/charts/insight/values.schema.json
+++ b/charts/insight/values.schema.json
@@ -50,6 +50,63 @@
}
}
}
+ },
+ "freshness": {
+ "type": "object",
+ "properties": {
+ "enabled": { "type": "boolean" },
+ "schedule": { "type": "string", "default": "0 13 * * *" },
+ "dbtSelect": { "type": "string", "default": "source:*" },
+ "cluster": { "type": "string" },
+ "tenant": { "type": "string" },
+ "notification": {
+ "type": "object",
+ "properties": {
+ "driver": { "enum": ["", "webhook", "zulip", "slack", "teams", "email"] },
+ "webhook": {
+ "type": "object",
+ "properties": { "urlSecret": { "$ref": "#/definitions/urlSecret" } }
+ },
+ "zulip": {
+ "type": "object",
+ "properties": {
+ "urlSecret": { "$ref": "#/definitions/urlSecret" },
+ "stream": { "type": "string" },
+ "topic": { "type": "string" }
+ }
+ },
+ "slack": {
+ "type": "object",
+ "properties": {
+ "urlSecret": { "$ref": "#/definitions/urlSecret" },
+ "channel": { "type": "string" }
+ }
+ },
+ "teams": {
+ "type": "object",
+ "properties": { "urlSecret": { "$ref": "#/definitions/urlSecret" } }
+ },
+ "email": {
+ "type": "object",
+ "properties": {
+ "smtp": {
+ "type": "object",
+ "properties": {
+ "host": { "type": "string" },
+ "port": { "type": "integer", "minimum": 1, "maximum": 65535 },
+ "username": { "type": "string" },
+ "passwordSecret": { "$ref": "#/definitions/urlSecret" },
+ "starttls": { "type": "boolean" }
+ }
+ },
+ "from": { "type": "string" },
+ "to": { "type": "string" },
+ "subjectPrefix": { "type": "string" }
+ }
+ }
+ }
+ }
+ }
}
}
},
@@ -140,6 +197,13 @@
},
"definitions": {
+ "urlSecret": {
+ "type": "object",
+ "properties": {
+ "name": { "type": "string" },
+ "key": { "type": "string", "minLength": 1 }
+ }
+ },
"infraDep": {
"type": "object",
"properties": {
diff --git a/charts/insight/values.yaml b/charts/insight/values.yaml
index c6cf558f9..14efbd1f7 100644
--- a/charts/insight/values.yaml
+++ b/charts/insight/values.yaml
@@ -236,6 +236,122 @@ ingestion:
limits:
cpu: "500m"
memory: "512Mi"
+
+ # ─── bronze source freshness (ingestion-monitoring) ──────────────────────
+ # Runtime half of source-freshness monitoring. The per-source `freshness:`
+ # blocks (loaded_at_field + warn_after/error_after) live in each connector
+ # `schema.yml` — see constructorfabric/insight#1346 / EPIC #1321. This block
+ # only governs the CronWorkflow that *runs* `dbt source freshness` on a
+ # schedule, reads `target/sources.json`, and fans the verdict out to a
+ # notification channel. It does NOT own threshold values.
+ freshness:
+ enabled: true
+ # Cron expression for the freshness check. Daily at 13:00 UTC sits past
+ # all current connector sync slots (02:00–11:00 UTC) plus a 2h grace.
+ schedule: "0 13 * * *"
+ # dbt selector for the freshness check. `source:*` covers every declared
+ # source. Narrow this for smoke-testing in dev.
+ dbtSelect: "source:*"
+ # Identification labels included in every notification payload so
+ # receivers can route alerts when one bot is shared across multiple
+ # deployments / customers. Both default to empty — leaving them out
+ # of the summary entirely.
+ # cluster — deployment / installation tier (e.g. "prod", "staging").
+ # tenant — customer / workspace identifier. Set when one Insight
+ # installation serves multiple tenants.
+ cluster: ""
+ tenant: ""
+ # Notification delivery. Each driver has its own subblock with the
+ # settings it needs; `driver` selects which one is active and the
+ # rest are inert. New drivers drop in by adding a sibling block +
+ # extending the parser branch in `dbt-source-freshness.yaml` — no
+ # other Helm surface needs to change.
+ #
+ # Every driver that needs an outgoing URL reads it from a Kubernetes
+ # Secret (mirrors the `clickhouse.passwordSecret` pattern) so the
+ # token is not rendered into manifests and not visible in the Argo UI.
+ notification:
+ # "" — no fan-out (default). Breaches still surface via Argo run
+ # status / `failedJobsHistoryLimit`. Use this for clusters
+ # that don't have an outgoing channel wired yet.
+ # "webhook" — generic JSON POST. Body shape:
+ # { topic, cluster, summary, breaches: [...] }
+ # Suitable for Mattermost / a custom relay / opsgenie ingest.
+ # "zulip" — Zulip incoming-webhook integration.
+ # "slack" — Slack incoming-webhook.
+ # "teams" — Microsoft Teams incoming-webhook (MessageCard).
+ # "email" — SMTP via stdlib smtplib (no extra deps).
+ driver: ""
+
+ # ─── webhook driver ─────────────────────────────────────────────
+ webhook:
+ # Required when driver=webhook. The full POST URL lives in
+ # this Secret under `key`.
+ urlSecret:
+ name: ""
+ key: "url"
+
+ # ─── zulip driver ───────────────────────────────────────────────
+ zulip:
+ # Required when driver=zulip. The integration URL Zulip's UI
+ # gives you when you add an "Incoming webhook" integration
+ # already encodes stream/topic in its query string — paste it
+ # verbatim here. The parser silently rewrites the path from
+ # `.../external/json` (which dumps the body as a JSON code
+ # block) to `.../external/slack` (Zulip's Slack-compatible
+ # webhook, which renders markdown), so a JSON-style URL still
+ # produces a properly-formatted message.
+ urlSecret:
+ name: ""
+ key: "url"
+ # Routing overrides. Empty → use whatever's in the integration
+ # URL's query string. Note Zulip stream names can contain a
+ # trailing zero-width space (U+200B); copy-pasting from the UI
+ # is safer than retyping.
+ stream: ""
+ topic: ""
+
+ # ─── slack driver ───────────────────────────────────────────────
+ slack:
+ # Required when driver=slack. Slack incoming-webhook URL.
+ urlSecret:
+ name: ""
+ key: "url"
+ # Optional channel override. Slack incoming webhooks are bound
+ # to a default channel at creation; setting this overrides
+ # only if the workspace allows it (admin policy).
+ channel: ""
+
+ # ─── teams driver ───────────────────────────────────────────────
+ teams:
+ # Required when driver=teams. Microsoft Teams incoming-webhook
+ # URL. The URL is bound to a single channel at creation; there
+ # is no override knob.
+ urlSecret:
+ name: ""
+ key: "url"
+
+ # ─── email driver ───────────────────────────────────────────────
+ email:
+ smtp:
+ host: ""
+ port: 587 # 587 = STARTTLS, 465 = SMTPS, 25 = plain
+ username: "" # SMTP AUTH user; password lives in Secret
+ # SMTP password — required when driver=email and the SMTP
+ # server requires AUTH. Put the password in a k8s Secret.
+ passwordSecret:
+ name: ""
+ key: "password"
+ # Use STARTTLS on connect. Ignored when port=465 (already
+ # implicit TLS).
+ starttls: true
+ # Envelope.
+ from: "" # e.g. "ingestion-monitor@example.com"
+ # Comma-separated list of recipient addresses. Helm values do
+ # not support arrays through to env vars cleanly without a
+ # template loop; a single string is the simplest shape.
+ to: "" # e.g. "oncall@example.com,sre@example.com"
+ subjectPrefix: "[ingestion-freshness]"
# ═══════════════════════════════════════════════════════════════════════════
# INFRASTRUCTURE
# ═══════════════════════════════════════════════════════════════════════════
diff --git a/docs/domain/README.md b/docs/domain/README.md
index dde5fe40f..002dcd284 100644
--- a/docs/domain/README.md
+++ b/docs/domain/README.md
@@ -7,5 +7,6 @@ Domain-level specifications for the Insight platform. Each domain represents a b
| Domain | Description | Status |
|---|---|---|
| [`ingestion/`](ingestion/) | Data pipeline from source APIs to Silver step 1 (Airbyte + Argo Workflows + dbt) | Accepted |
+| [`ingestion-monitoring/`](ingestion-monitoring/) | Bronze freshness SLA, trap detection, multi-driver alert delivery | Proposed |
| [`connector/`](connector/) | Connector development: Insight Connector packages, nocode and CDK patterns, packaging, debugging | Accepted |
| [`identity-resolution/`](identity-resolution/) | Person identity matching and resolution across sources | Proposed |
diff --git a/docs/domain/ingestion-monitoring/specs/DESIGN.md b/docs/domain/ingestion-monitoring/specs/DESIGN.md
new file mode 100644
index 000000000..f47c5d988
--- /dev/null
+++ b/docs/domain/ingestion-monitoring/specs/DESIGN.md
@@ -0,0 +1,874 @@
+---
+status: proposed
+date: 2026-05-04
+---
+
+# Technical Design — Ingestion Monitoring
+
+
+
+- [1. Architecture Overview](#1-architecture-overview)
+ - [1.1 Architectural Vision](#11-architectural-vision)
+ - [1.2 Architecture Drivers](#12-architecture-drivers)
+ - [1.3 Architecture Layers](#13-architecture-layers)
+- [2. Principles & Constraints](#2-principles--constraints)
+ - [2.1 Design Principles](#21-design-principles)
+ - [2.2 Constraints](#22-constraints)
+- [3. Technical Architecture](#3-technical-architecture)
+ - [3.1 Domain Model](#31-domain-model)
+ - [3.2 Component Model](#32-component-model)
+ - [3.3 API Contracts](#33-api-contracts)
+ - [3.4 Internal Dependencies](#34-internal-dependencies)
+ - [3.5 External Dependencies](#35-external-dependencies)
+ - [3.6 Interactions & Sequences](#36-interactions--sequences)
+ - [3.7 Database schemas & tables](#37-database-schemas--tables)
+ - [3.8 Deployment Topology](#38-deployment-topology)
+- [4. Additional context](#4-additional-context)
+ - [4.1 Why two report tiers (`report`, `report_extended`) instead of one](#41-why-two-report-tiers-report-reportextended-instead-of-one)
+ - [4.2 Why `_airbyte_extracted_at` for streaming and a business date for re-emit](#42-why-airbyteextractedat-for-streaming-and-a-business-date-for-re-emit)
+ - [4.3 Why per-driver subblocks instead of a flat `notification.url`](#43-why-per-driver-subblocks-instead-of-a-flat-notificationurl)
+ - [4.4 Why the Zulip dispatcher silently rewrites the endpoint](#44-why-the-zulip-dispatcher-silently-rewrites-the-endpoint)
+ - [4.5 Why narrow the selector at runtime](#45-why-narrow-the-selector-at-runtime)
+ - [4.6 Why the trap detector is advisory](#46-why-the-trap-detector-is-advisory)
+ - [4.7 Why every `freshness: null` opt-out needs a rationale](#47-why-every-freshness-null-opt-out-needs-a-rationale)
+- [5. Traceability](#5-traceability)
+
+
+
+## 1. Architecture Overview
+
+### 1.1 Architectural Vision
+
+Ingestion Monitoring is a single daily-cadence Argo `CronWorkflow`
+(`dbt-source-freshness-check`) that delegates to one stateless
+`WorkflowTemplate` (`dbt-source-freshness`). The template runs `dbt
+source freshness` against ClickHouse, parses `target/sources.json` with
+an inline Python step, and dispatches breaches to one of five
+notification drivers (webhook / zulip / slack / teams / email). Two
+scripts sit at the edges: a CI lint that prevents bad anchors from
+landing, and a runtime trap detector that flags re-emit patterns the
+freshness check itself cannot see.
+
+The design's defining property is **everything is config-driven from a
+single Helm block** (`ingestion.freshness.*` in
+[`charts/insight/values.yaml`](../../../../charts/insight/values.yaml)).
+Adding a connector is a `schema.yml` edit; switching notification
+channels is a `values.yaml` edit; tuning thresholds is a `values.yaml`
+edit. No Rust services, no extra Deployments, no per-connector pipeline
+plumbing.
+
+The second defining property is **credential isolation by
+construction**. The driver's URL or SMTP password is bound to the pod's
+env from a `secretKeyRef` resolved server-side by Argo; the rendered
+manifest, the Argo UI, and the workflow-controller logs only ever see
+the Secret reference, never the raw value
+([`charts/insight/templates/ingestion/dbt-source-freshness.yaml:161-194`](../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml)).
+
+### 1.2 Architecture Drivers
+
+#### Functional Drivers
+
+| PRD Requirement | Design Response |
+|---|---|
+| `cpt-insightspec-fr-mon-daily-cronworkflow` | Argo `CronWorkflow` `dbt-source-freshness-check`, one daily run at the Helm-configured cron. |
+| `cpt-insightspec-fr-mon-selector-narrowing` | Pre-step queries `system.databases`, builds the effective dbt selector. |
+| `cpt-insightspec-fr-mon-stale-report-cleanup` | `rm -f target/sources.json` before every dbt invocation. |
+| `cpt-insightspec-fr-mon-exit-rederivation` | Inline Python parser re-derives outcome from `target/sources.json`; dbt's exit code is swallowed. |
+| `cpt-insightspec-fr-mon-thresholds-sot` | Literal `warn_after`/`error_after` per source in `schema.yml`; the CronWorkflow reads them via `--select source:*`. No Helm/env-var layer. |
+| `cpt-insightspec-fr-mon-four-tiers` | Per-source `freshness:` blocks with literal tier values `default` / `event` / `report` / `report_extended`. |
+| `cpt-insightspec-fr-mon-tier-assignment` | Tier choice lives as literal values in connector `schema.yml`; no runtime tuning knob. |
+| `cpt-insightspec-fr-mon-optout-pertable` | Per-table `freshness: null` in `schema.yml`. |
+| `cpt-insightspec-fr-mon-optout-rationale` | One-line rationale comment beside each `freshness: null`; reserved for incremental streams. |
+| `cpt-insightspec-fr-mon-mandatory-anchor` | Per-source `loaded_at_field`; coverage counted by the QA `dbt_coverage.py` gate (#1321). A missing/invalid anchor surfaces as a dbt `runtime error`. |
+| `cpt-insightspec-fr-mon-trap-post-parser` | Workflow runs `freshness-trap-detect.py` after the parser, advisory-only. |
+| `cpt-insightspec-fr-mon-trap-two-modes` | Heuristic full-reemit + opt-in business-date divergence in the same script. |
+| `cpt-insightspec-fr-mon-trap-skip-annotation` | `meta.bronze_freshness_trap_check: skip` honoured by detector. |
+| `cpt-insightspec-fr-mon-trap-advisory` | Detector exit only writes a notice; freshness parser owns the page-worthy verdict. |
+| `cpt-insightspec-fr-mon-driver-selection` | `notification.driver` enum validated by `values.schema.json`. |
+| `cpt-insightspec-fr-mon-credential-isolation` | Per-driver `secretKeyRef` branches in the workflow template. |
+| `cpt-insightspec-fr-mon-dispatch-impl` | `dispatchers` dict in the inline parser. |
+| `cpt-insightspec-fr-mon-delivery-failure-isolation` | `try/except` around dispatch; primary exit code untouched. |
+| `cpt-insightspec-fr-mon-identity-helm` | `cluster` / `tenant` Helm values → `CLUSTER` / `TENANT` env vars. |
+| `cpt-insightspec-fr-mon-identity-summary` | Summary prefix builder drops empty labels. |
+| `cpt-insightspec-fr-mon-identity-payload` | `cluster` / `tenant` written as raw JSON keys. |
+
+#### NFR Allocation
+
+| NFR | Component(s) | Verification |
+|---|---|---|
+| `cpt-insightspec-nfr-mon-idempotency` | Inline parser + workflow template (no persistent state) | Re-run; verify identical exit + payload. |
+| `cpt-insightspec-nfr-mon-daily-cadence` | `CronWorkflow.schedule` | Inspect rendered CronWorkflow; manual ad-hoc submission still works. |
+| `cpt-insightspec-nfr-mon-credential-isolation` | `secretKeyRef` branches in workflow template | `kubectl get workflowtemplate ... -o yaml` carries no raw URLs/passwords. |
+| `cpt-insightspec-nfr-mon-exit-codes` | Inline parser exit logic | Inject fixtures; assert exits 0 / 1 / 2. |
+| `cpt-insightspec-nfr-mon-ci-gated` | QA-owned `dbt_coverage.py` gate (#1321) | Open PR with an undeclared `bronze_*` source; coverage gate fails. |
+| `cpt-insightspec-nfr-mon-trap-advisory-mode` | Trap detector exit handling | Force trap finding; verify primary exit code unchanged. |
+| `cpt-insightspec-nfr-mon-activation-deadline` | `WorkflowTemplate.activeDeadlineSeconds: 1200` | Inspect rendered template. |
+| `cpt-insightspec-nfr-mon-empty-sentinel` | Inline parser `1970-01-01` detection | Empty bronze table fixture; verify `(table is empty)` line. |
+
+### 1.3 Architecture Layers
+
+```
+ingestion.freshness.* (charts/insight/values.yaml)
+ │
+ ▼
+CronWorkflow.spec.workflowSpec parameters
+(charts/insight/templates/ingestion/dbt-source-freshness.yaml:583-662)
+ │
+ ▼
+WorkflowTemplate inputs.parameters → env vars + secretKeyRef
+(.yaml:34-212)
+ │
+ ▼
+dbt source freshness ─► target/sources.json
+(reads literal warn_after/error_after per source in schema.yml)
+ │
+ ▼
+Inline Python parser (.yaml:317-562)
+ │ │
+ │ └─► driver dispatcher (webhook / zulip / slack / teams / email)
+ │ │
+ │ └─► external sink (Zulip / Slack / Teams / SMTP / generic)
+ ▼
+freshness-trap-detect.py (advisory)
+(src/ingestion/scripts/freshness-trap-detect.py)
+```
+
+## 2. Principles & Constraints
+
+### 2.1 Design Principles
+
+#### Advisory failure mode for traps
+
+- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-trap-advisory`
+
+The trap detector logs but never overrides the freshness parser's exit
+code (workflow template lines 575–579). Page-worthy decisions belong to
+one place — making heuristic checks page-worthy would erode trust in
+the whole monitoring domain.
+
+#### Log loud on failure
+
+- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-log-loud`
+
+Notification HTTP errors surface the upstream's body (workflow template
+lines 421–432) — the `Python-urllib` default exposes only the status
+line, which hides Zulip / Slack / Teams JSON error payloads. Loud logs
+are the difference between "fix in 5 min" and "fix in 5 hours".
+
+#### Scope freshness to what's collected
+
+- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-scope-collected`
+
+Selector is narrowed from `source:*` to `bronze_*` databases that
+exist in `system.databases` (lines 262–297); a tenant that didn't
+deploy OpenAI doesn't see phantom OpenAI ERRORs. Deployment health is
+the deployment-health workstream
+([#272](https://github.com/cyberfabric/insight/issues/272)), not this
+domain.
+
+#### Vendor-documented thresholds, not gut feeling
+
+- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-vendor-thresholds`
+
+Each tier's warn / error pair sits at the upper edge of the vendor's
+documented normal cadence (M365 24–48 h → warn 48 h; Slack ~3 d →
+warn 72 h). Documented in `values.yaml` lines 156–183. Re-tiering
+requires either a vendor change or evidence from a multi-week
+observation window.
+
+### 2.2 Constraints
+
+#### dbt-clickhouse needs explicit `loaded_at_field`
+
+- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-explicit-anchor`
+
+The dbt-clickhouse adapter does not support metadata-based freshness;
+a missing field → `runtime error`. `+loaded_at_field` at project level
+is silently ignored (`loaded_at_field` is a *property*, not a config).
+Hence every source declares its own `loaded_at_field`, and coverage is
+counted by the QA `dbt_coverage.py` gate (#1321).
+
+#### Thresholds are literal per source, not inherited
+
+- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-freshness-propagates`
+
+Each source carries its own literal `freshness:` block (`warn_after` /
+`error_after`) and `loaded_at_field` directly in `schema.yml`. There is no
+project-level `+freshness` default and no Helm/env-var threshold layer — the
+`dbt_coverage.py` gate counts per-source declarations, so inheritance would not
+register as coverage anyway. Tier values live with the source.
+
+#### Zulip's `/external/json` is not a markdown sender
+
+- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-zulip-endpoint`
+
+`/api/v1/external/json` dumps the request body as a JSON code block.
+The Slack-compatible endpoint (`/api/v1/external/slack`) renders
+Slack-style markdown. The dispatcher silently rewrites the path so an
+operator who pasted the JSON URL still gets formatted output (workflow
+template lines 445–478).
+
+#### Cloudflare-managed receivers reject the default UA
+
+- [ ] `p2` - **ID**: `cpt-insightspec-constraint-mon-explicit-ua`
+
+A bare `Python-urllib/3.x` looks like a bot. The dispatcher sets an
+explicit `User-Agent: insight-freshness-monitor/1.0` (workflow
+template line 416).
+
+#### Empty table sentinel must not read as 500 000 h lag
+
+- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-empty-sentinel`
+
+`MAX(
)` over zero rows is `NULL`. dbt-clickhouse serialises it as
+the Unix epoch with an enormous `age_in_s`, which reads as ~500 000 h
+of lag. The parser detects the sentinel
+(`max_loaded_at.startswith("1970-01-01")`) and surfaces "table is
+empty (no rows ingested)" instead (workflow template lines 348–360).
+
+## 3. Technical Architecture
+
+### 3.1 Domain Model
+
+| Concept | Where it's defined |
+|---|---|
+| **Bronze source** | `sources:` entry in any connector's `dbt/schema.yml`; identified by `name: bronze_*` / `schema: bronze_*`. |
+| **Freshness anchor** | `loaded_at_field` property at source or table level. Two valid forms: `_airbyte_extracted_at` (incremental) or a business-date expression (`parseDateTimeBestEffortOrNull(...)`, etc.) for windowed connectors. |
+| **Opt-out** | Per-table `freshness: null` plus a one-line rationale comment; reserved for incremental streams that legitimately go quiet. |
+| **SLA tier** | One of `default`, `event`, `report`, `report_extended`; chosen by the literal `warn_after`/`error_after` values in the connector's source-level `freshness:` block. |
+| **Breach** | A row in the parser's `breaches` list: `{source, status, max_loaded_at, age_hours, empty}` (workflow template lines 352–360). |
+| **Trap suspect** | A finding from the trap detector: `kind ∈ {full-reemit, incremental-topup}` plus row-level evidence (`freshness-trap-detect.py:186-227`). |
+| **Driver** | A parser dispatch arm + Helm subblock + workflow-yaml `secretKeyRef` branch. Five today (`webhook`, `zulip`, `slack`, `teams`, `email`); `""` = log-only. |
+
+### 3.2 Component Model
+
+#### Workflow Template (`dbt-source-freshness`)
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-workflow-template`
+
+##### Why this component exists
+
+Provides the stateless Argo `WorkflowTemplate` that any caller
+(scheduled or ad-hoc) can invoke with a single set of parameters.
+Centralises the dbt invocation, parser, dispatcher, and trap detector
+so that switching tenants or drivers does not duplicate workflow
+plumbing.
+
+##### Responsibility scope
+
+- Accept every per-deployment parameter via `inputs.parameters`
+ (charts/insight/templates/ingestion/dbt-source-freshness.yaml:34-106).
+- Bind credentials via `secretKeyRef` only when the matching driver is
+ configured (lines 161–194).
+- Run `dbt source freshness` with the effective selector (lines
+ 254–307).
+- Hand off to the inline parser, then the trap detector, with the
+ shell wrapper preserving the parser's exit code (lines 561–581).
+
+##### Responsibility boundaries
+
+- Does NOT decide *when* to run — that is the `CronWorkflow`'s job.
+- Does NOT own credential storage — Kubernetes Secrets do.
+- Does NOT classify breaches — the inline parser does (see
+ `cpt-insightspec-component-mon-parser`).
+
+##### Related components (by ID)
+
+- `cpt-insightspec-component-mon-cronworkflow` — schedules this template.
+- `cpt-insightspec-component-mon-parser` — invoked inline.
+- `cpt-insightspec-component-mon-trap-detector` — invoked after the parser.
+- `cpt-insightspec-component-mon-helm-surface` — supplies parameters.
+
+#### CronWorkflow (`dbt-source-freshness-check`)
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-cronworkflow`
+
+##### Why this component exists
+
+Wraps the workflow template with a daily schedule and feeds Helm values
+into its parameters. Argo handles cron parsing, missed-fire policy,
+and `failedJobsHistoryLimit` retention.
+
+##### Responsibility scope
+
+- `spec.schedule` from `ingestion.freshness.schedule`.
+- `spec.workflowSpec` parameters from the Helm block (lines 583–662).
+- Default `concurrencyPolicy: Forbid` so a long run cannot be doubled.
+
+##### Responsibility boundaries
+
+- Does NOT execute logic itself — it only delegates to the
+ `WorkflowTemplate`.
+- Does NOT carry credentials or notification config beyond what the
+ template parameters require.
+
+##### Related components (by ID)
+
+- `cpt-insightspec-component-mon-workflow-template` — the delegate.
+- `cpt-insightspec-component-mon-helm-surface` — Helm values it
+ flattens into parameters.
+
+#### Inline Freshness Parser
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-parser`
+
+##### Why this component exists
+
+dbt's exit code does not distinguish PASS / WARN / ERROR / RUNTIME
+ERROR; we need a deterministic verdict. The parser reads
+`target/sources.json` and computes the canonical breach list and exit
+code that the dispatcher uses.
+
+##### Responsibility scope
+
+- Read `target/sources.json`; if missing, exit 2 (line 323).
+- Classify each source as `pass` / `warn` / `error` / `runtime error`.
+- Detect the empty-table sentinel and emit `(table is empty)` instead
+ of ~500 000 h lag (lines 348–360).
+- Build the canonical breach record `{source, status, max_loaded_at,
+ age_hours, empty}`.
+- Decide page vs no-page (`sys.exit(1)` if any breach is `error` or
+ `runtime error`, else `0`; line 559).
+- Call the dispatcher when a driver is configured.
+
+##### Responsibility boundaries
+
+- Does NOT run dbt itself.
+- Does NOT format driver-specific payloads — that is the dispatcher's
+ job.
+- Does NOT write any state to disk other than the dbt-produced
+ `target/sources.json` (already there).
+
+##### Related components (by ID)
+
+- `cpt-insightspec-component-mon-workflow-template` — invokes it.
+- `cpt-insightspec-component-mon-driver-dispatcher` — invoked from inside
+ the parser.
+- `cpt-insightspec-component-mon-trap-detector` — runs after.
+
+#### Driver Dispatcher
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-driver-dispatcher`
+
+##### Why this component exists
+
+Different sinks (Zulip, Slack, Teams, SMTP, generic webhook) need
+different payload shapes, content types, and quirks. A single
+dispatcher dict (workflow template lines 533–539) keeps the parser
+simple and makes adding a new driver a one-arm change.
+
+##### Responsibility scope
+
+- Map driver name → format function → `_post`/SMTP transport.
+- Set `User-Agent: insight-freshness-monitor/1.0` (line 416).
+- Surface upstream HTTP body on `HTTPError` (lines 421–432).
+- Rewrite Zulip's `/external/json` → `/external/slack` for the Zulip
+ driver (line 457).
+- Catch and log every dispatch failure without changing the workflow's
+ primary exit code (lines 550–554).
+
+##### Responsibility boundaries
+
+- Does NOT classify breaches.
+- Does NOT decide whether to dispatch — the parser does.
+- Does NOT manage Secret content; only consumes env vars bound from
+ Secrets.
+
+##### Related components (by ID)
+
+- `cpt-insightspec-component-mon-parser` — caller.
+- `cpt-insightspec-component-mon-helm-surface` — driver enum + per-driver
+ Helm subblocks.
+
+#### Trap Detector
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-trap-detector`
+
+##### Why this component exists
+
+The freshness check on `_airbyte_extracted_at` is fooled by re-emit /
+incremental-topup patterns. A second, independent script flags those
+suspects so authors and operators see them — without overriding the
+freshness verdict.
+
+##### Responsibility scope
+
+- Mode 1 (heuristic full-reemit): ≥ 95 % rows within last 30 h, ≤ 2
+ distinct extract days, ≥ 100 rows
+ (`freshness-trap-detect.py:188-200`).
+- Mode 2 (opt-in business-date divergence): compare
+ `MAX()` to `MAX(_airbyte_extracted_at)`;
+ flag ≥ 24 h gap (lines 202–227).
+- Honour `meta.bronze_freshness_trap_check: skip` per source/table.
+- Print finding lists to stdout; per-table query failures and bootstrap
+ errors go to stderr. Exit code is captured by the workflow but not
+ used to override the parser's verdict.
+
+##### Responsibility boundaries
+
+- Does NOT page anyone.
+- Does NOT mutate ClickHouse data.
+- Does NOT read driver Helm config.
+
+##### Related components (by ID)
+
+- `cpt-insightspec-component-mon-workflow-template` — invokes it after the
+ parser.
+- `cpt-insightspec-component-mon-parser` — its verdict is preserved.
+
+#### Per-source declaration coverage (external — QA gate)
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-ci-lint`
+
+##### Why this is not a component of this module
+
+Catching an undeclared / mis-anchored `bronze_*` source at PR time is real,
+but it is owned by the QA `scripts/ci/dbt_coverage.py` gate (EPIC #1321 /
+#1322), which counts per-source `loaded_at_field` + `freshness` declarations.
+This module does not ship its own connector-schema lint. The two failure
+modes that gate does not cover — choosing the *wrong* anchor (windowed source
+left on `_airbyte_extracted_at` → false-green) or a non-timestamp column
+(→ dbt `runtime error`) — are caught by confirming `ext_age` vs `biz_age`
+against live data, not statically.
+
+##### Related components (by ID)
+
+- `cpt-insightspec-component-mon-workflow-template` — runtime counterpart.
+
+#### Thresholds are literal per source (no env-var layer)
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-threshold-env`
+
+##### Why there is no threshold component
+
+Earlier drafts routed tier values through Helm → workflow params →
+`FRESHNESS_*_H` env → dbt `env_var(...)`. That was dropped: the
+`dbt_coverage.py` gate counts per-source declarations, so thresholds live as
+**literal `warn_after`/`error_after` values in each source's `schema.yml`** —
+a single source of truth with no second place to drift and no Helm/env layer.
+The CronWorkflow simply runs `dbt source freshness --select source:*` and acts
+on whatever each source declares.
+
+#### Helm Surface
+
+- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-helm-surface`
+
+##### Why this component exists
+
+Operators need a single tunable surface. `ingestion.freshness.*` in
+`values.yaml` is the documented configuration entrypoint;
+`values.schema.json` validates it.
+
+##### Responsibility scope
+
+- `ingestion.freshness.{enabled,schedule,dbtSelect,cluster,tenant}`
+ top-level keys. (No `thresholds.*` block — thresholds are literal per
+ source in `schema.yml`, not Helm-tuned.)
+- `ingestion.freshness.notification.{driver, webhook, zulip, slack,
+ teams, email}` per-driver subblocks with `urlSecret` references.
+- `values.schema.json` enforces the driver enum (line 37) and the
+ shape of each subblock.
+
+##### Responsibility boundaries
+
+- Does NOT carry credentials directly — only `secretKeyRef` pointers.
+- Does NOT decide tier choice for connectors.
+
+##### Related components (by ID)
+
+- `cpt-insightspec-component-mon-cronworkflow` — flattens this into
+ parameters.
+- `cpt-insightspec-component-mon-driver-dispatcher` — reads driver
+ selection.
+
+### 3.3 API Contracts
+
+The driver dispatcher is the `dispatchers` dict at workflow template
+lines 533–539. Common preamble: every dispatch goes through `_post`
+(lines 412–432), which sets `User-Agent: insight-freshness-monitor/1.0`
+and surfaces the upstream's response body on `HTTPError`.
+
+#### 3.3.1 `webhook`
+
+- Endpoint: whatever is in `NOTIFICATION_URL` (verbatim).
+- Body shape (workflow template lines 434–443):
+
+```json
+{
+ "topic": "ingestion-freshness",
+ "cluster": "",
+ "tenant": "",
+ "summary": "[cluster=…, tenant=…] N bronze source(s) breaching freshness SLA",
+ "breaches": [
+ {
+ "source": "source.ingestion.bronze_jira.jira_issue",
+ "status": "error",
+ "max_loaded_at": "2026-04-28T03:14:21Z",
+ "age_hours": 51.2,
+ "empty": false
+ }
+ ]
+}
+```
+
+- Header: `Content-Type: application/json`.
+- Secret binding:
+ `ingestion.freshness.notification.webhook.urlSecret.{name,key}`
+ (default `key: url`), workflow YAML lines 162–167.
+
+#### 3.3.2 `zulip`
+
+- Endpoint: `NOTIFICATION_URL` with the path `/api/v1/external/json`
+ rewritten to `/api/v1/external/slack` (line 457). The two paths are
+ different Zulip integrations — `/external/json` dumps the body as a
+ JSON code block (debugging integration); `/external/slack` is the
+ Slack-compatible incoming webhook that renders Slack mrkdwn.
+ Operators who pasted the JSON URL still get a formatted message.
+- Routing: `stream` and `topic` from `NOTIFICATION_ZULIP_STREAM` /
+ `NOTIFICATION_ZULIP_TOPIC` are URL-encoded and appended to the query
+ string (lines 458–462).
+- Body: `application/x-www-form-urlencoded` carrying
+ `user_name=ingestion-freshness`, `channel_name=freshness`,
+ `text=` — the Slack-compatible webhook expects these
+ legacy-Slack form fields (lines 472–478).
+- Markdown: single `*` for bold (Slack mrkdwn), bullet `•` (line 466).
+- Secret binding:
+ `ingestion.freshness.notification.zulip.urlSecret.{name,key}`
+ (workflow YAML lines 168–173).
+- Quirk: Zulip stream names can include trailing zero-width-space
+ U+200B; `values.yaml:243-249` warns operators to copy from Zulip's
+ UI rather than retype.
+
+#### 3.3.3 `slack`
+
+- Endpoint: `NOTIFICATION_URL` (Slack incoming-webhook URL).
+- Body: `application/json` with `{"text": ""}`, optionally
+ `channel: ` if `NOTIFICATION_SLACK_CHANNEL` is set and
+ workspace policy allows it (lines 480–487).
+- Secret binding:
+ `ingestion.freshness.notification.slack.urlSecret.{name,key}`
+ (workflow YAML lines 174–179).
+
+#### 3.3.4 `teams`
+
+- Endpoint: `NOTIFICATION_URL` (Microsoft Teams incoming webhook).
+- Body: `application/json`, `MessageCard` schema (lines 489–500).
+ `themeColor` is `FF0000` (red) when any breach is page-worthy,
+ `FFA500` (orange) otherwise.
+- No channel override — Teams binds the URL to a single channel at
+ creation.
+- Secret binding:
+ `ingestion.freshness.notification.teams.urlSecret.{name,key}`
+ (workflow YAML lines 180–185).
+
+#### 3.3.5 `email`
+
+- Transport: `smtplib` from the standard library (no extra deps).
+ `SMTP_SSL` when port=465, otherwise plain `SMTP` with optional
+ `STARTTLS` (lines 502–531).
+- Required Helm values: `notification.email.smtp.host`,
+ `notification.email.from`, `notification.email.to`. The dispatcher
+ raises `RuntimeError` if any is missing (lines 513–517).
+- Subject: `` when prefix is non-empty.
+- Recipients: comma-separated `NOTIFICATION_EMAIL_TO` split on `,`,
+ whitespace-stripped.
+- Secret binding:
+ `ingestion.freshness.notification.email.smtp.passwordSecret.{name,key}`
+ (workflow YAML lines 187–193, rendered only when
+ `$n.email.smtp.passwordSecret.name` is set).
+
+### 3.4 Internal Dependencies
+
+| From | To | Mechanism |
+|---|---|---|
+| CronWorkflow | WorkflowTemplate | Argo `templateRef` (same namespace). |
+| WorkflowTemplate | dbt-clickhouse | Container exec inside `insight-toolbox`. |
+| WorkflowTemplate | ClickHouse | dbt profile points at CH HTTP endpoint. |
+| WorkflowTemplate | Trap detector | Container exec inside `insight-toolbox` after parser. |
+| Trap detector | ClickHouse | HTTP queries on port 8123. |
+| Inline parser | Driver dispatcher | In-process Python call. |
+| Driver dispatcher | Kubernetes Secret | `secretKeyRef` env binding (server-side resolution). |
+
+### 3.5 External Dependencies
+
+#### Argo Workflows controller
+
+Provides `WorkflowTemplate` / `CronWorkflow` reconciliation. Argo is
+**not** a chart dependency of the umbrella — operators install it out
+of band (see [`scripts/install-argo.sh`](../../../../scripts/install-argo.sh)
+referenced in `charts/insight/Chart.yaml`'s preamble). Tested against
+v3.5.10 and v3.6.x; minimum CRDs needed are `workflows.argoproj.io`,
+`workflowtemplates.argoproj.io`, and `cronworkflows.argoproj.io`.
+
+#### dbt-clickhouse adapter
+
+Bundled in `insight-toolbox`. Required because the adapter does not
+support metadata-based freshness — `loaded_at_field` is the only
+supported anchor expression.
+
+#### ClickHouse
+
+Provides `system.databases` for selector narrowing and the HTTP
+endpoint used by the trap detector.
+
+#### Kubernetes Secret store
+
+`secretKeyRef` resolution for `clickhouse.passwordSecret` and the
+active driver's URL / SMTP password Secret.
+
+#### Notification receivers
+
+Zulip / Slack / Microsoft Teams incoming-webhook endpoints, or any
+generic JSON webhook, or an SMTP relay. Treated as black-box; their
+uptime is out of this domain's NFR scope.
+
+### 3.6 Interactions & Sequences
+
+#### Daily freshness happy path
+
+**ID**: `cpt-insightspec-seq-mon-happy-path`
+
+```mermaid
+sequenceDiagram
+ participant Cron as CronWorkflow
+ participant WT as WorkflowTemplate
+ participant CH as ClickHouse
+ participant DBT as dbt
+ participant P as Inline Parser
+ participant D as Driver Dispatcher
+ participant Sink as External Sink
+ participant T as Trap Detector
+
+ Cron->>WT: spawn (params)
+ WT->>WT: rm -f target/sources.json
+ WT->>CH: SELECT name FROM system.databases
+ CH-->>WT: existing bronze_* list
+ WT->>DBT: dbt source freshness --select
+ DBT->>CH: per-source freshness queries
+ CH-->>DBT: anchors
+ DBT-->>WT: target/sources.json
+ WT->>P: python parser
+ P->>P: classify breaches, build payload
+ alt driver != ""
+ P->>D: dispatch(driver, payload)
+ D->>Sink: HTTP POST / SMTP
+ Sink-->>D: 200 / 250
+ D-->>P: ok
+ end
+ P-->>WT: exit code (0 or 1)
+ WT->>T: freshness-trap-detect.py
+ T->>CH: trap-mode queries
+ CH-->>T: row-level evidence
+ T-->>WT: stderr findings (advisory)
+ WT-->>Cron: exit = parser exit
+```
+
+#### Failure path: dbt crash
+
+**ID**: `cpt-insightspec-seq-mon-dbt-crash`
+
+```mermaid
+sequenceDiagram
+ participant WT as WorkflowTemplate
+ participant DBT as dbt
+ participant P as Inline Parser
+
+ WT->>WT: rm -f target/sources.json
+ WT->>DBT: dbt source freshness ...
+ DBT--xWT: crash (no target/sources.json written)
+ WT->>P: python parser
+ P->>P: target/sources.json missing → exit 2
+ P-->>WT: exit 2
+ WT-->>WT: trap detector still runs (advisory)
+```
+
+#### Failure path: notification delivery error
+
+**ID**: `cpt-insightspec-seq-mon-delivery-error`
+
+```mermaid
+sequenceDiagram
+ participant P as Inline Parser
+ participant D as Driver Dispatcher
+ participant Sink as External Sink
+
+ P->>D: dispatch(driver, payload)
+ D->>Sink: HTTP POST
+ Sink--xD: 500 Internal Server Error (body: "...")
+ D-->>D: log status + body to stderr
+ D-->>P: caught (no re-raise)
+ P-->>P: exit code unchanged (still page-worthy if any error breach)
+```
+
+### 3.7 Database schemas & tables
+
+#### `target/sources.json` (dbt artefact)
+
+This is not a ClickHouse table; it is the dbt-produced JSON artefact
+that the parser consumes. Shape (subset):
+
+| Path | Type | Purpose |
+|---|---|---|
+| `results[i].unique_id` | string | dbt source unique id (`source.ingestion.bronze_*.
`). |
+| `results[i].status` | enum | `pass` / `warn` / `error` / `runtime error`. |
+| `results[i].max_loaded_at` | string | ISO-8601 of `MAX()` (or `1970-01-01T...` sentinel for empty tables). |
+| `results[i].max_loaded_at_time_ago_in_s` | number | Seconds since the anchor (parser converts to hours). |
+| `results[i].criteria.{warn_after,error_after}` | object | The active tier values for this source. |
+
+#### ClickHouse `system.databases` (read-only system table)
+
+Used by the workflow's selector-narrowing step to enumerate which
+`bronze_*` databases actually exist on this deployment. No writes.
+
+#### ClickHouse `bronze_.` (bronze tables)
+
+The freshness check reads `MAX()` and `count()` from
+each declared bronze source. The trap detector additionally reads
+`_airbyte_extracted_at` distribution and the optional
+`` column. No writes from this domain.
+
+### 3.8 Deployment Topology
+
+#### Argo namespace
+
+The `WorkflowTemplate`, `CronWorkflow`, and the workflow pods live in
+the `argo` namespace. The `insight-toolbox` image carries dbt,
+dbt-clickhouse, and the freshness scripts.
+
+#### Secret access
+
+Driver Secrets and `clickhouse.passwordSecret` live in the same
+namespace as the workflow pods so `secretKeyRef` resolves
+namespace-locally. Cross-namespace Secret access is out of scope.
+
+#### Kind / prod parity
+
+Local Kind clusters use the same Helm chart + values as production —
+the only differences are the per-deployment `cluster` / `tenant`
+labels and the driver selection (typically `""` locally, an actual
+driver in production). This keeps the dev/prod-parity NFR honest.
+
+| Helm path | Type | Default | Purpose |
+|---|---|---|---|
+| `ingestion.freshness.enabled` | bool | `true` | Top-level kill switch. |
+| `ingestion.freshness.schedule` | string | `"0 13 * * *"` | Cron — sits past every connector sync window (02:00–11:00 UTC) plus 2 h grace. |
+| `ingestion.freshness.dbtSelect` | string | `"source:*"` | Default selector. Narrowed at runtime to deployed bronze databases. |
+| `ingestion.freshness.cluster` | string | `""` | Identity label — installation tier. |
+| `ingestion.freshness.tenant` | string | `""` | Identity label — customer / workspace. |
+| _(thresholds)_ | — | — | Not a Helm value — `warn_after`/`error_after` are literal per source in each connector's `schema.yml`. |
+| `ingestion.freshness.notification.driver` | enum | `""` | One of `""`, `webhook`, `zulip`, `slack`, `teams`, `email`. Schema-validated. |
+| `ingestion.freshness.notification.webhook.urlSecret.{name,key}` | secretRef | — | Required when driver=webhook. |
+| `ingestion.freshness.notification.zulip.urlSecret.{name,key}` | secretRef | — | Required when driver=zulip. |
+| `ingestion.freshness.notification.zulip.stream` | string | `""` | Optional override; else inherits from URL query. |
+| `ingestion.freshness.notification.zulip.topic` | string | `""` | Optional override. |
+| `ingestion.freshness.notification.slack.urlSecret.{name,key}` | secretRef | — | Required when driver=slack. |
+| `ingestion.freshness.notification.slack.channel` | string | `""` | Optional channel override. |
+| `ingestion.freshness.notification.teams.urlSecret.{name,key}` | secretRef | — | Required when driver=teams. |
+| `ingestion.freshness.notification.email.smtp.host` | string | `""` | Required when driver=email. |
+| `ingestion.freshness.notification.email.smtp.port` | int | `587` | 587 STARTTLS / 465 SMTPS / 25 plain. |
+| `ingestion.freshness.notification.email.smtp.username` | string | `""` | Optional SMTP AUTH user. |
+| `ingestion.freshness.notification.email.smtp.passwordSecret.{name,key}` | secretRef | — | SMTP AUTH password (rendered only when set). |
+| `ingestion.freshness.notification.email.smtp.starttls` | bool | `true` | Ignored when port=465. |
+| `ingestion.freshness.notification.email.from` | string | `""` | Required when driver=email. |
+| `ingestion.freshness.notification.email.to` | string | `""` | Required when driver=email — comma-separated. |
+| `ingestion.freshness.notification.email.subjectPrefix` | string | `"[ingestion-freshness]"` | Prefix prepended to `summary`. |
+
+## 4. Additional context
+
+### 4.1 Why two report tiers (`report`, `report_extended`) instead of one
+
+Microsoft Graph reports document a 24–48 h publish lag — sliding the
+warn to 48 h is just "the upper edge of stated normal". Slack
+admin.analytics is observed and documented to lag ~3 days, with
+stretches up to 5 days during Slack maintenance — applying the
+report-tier (96 h error) to Slack would page on every Slack
+maintenance window. They needed different bands, but lumping every
+3-day vendor into "event" would lose the semantic signal that `event`
+is for natural quiet days, not for vendor lag.
+
+### 4.2 Why `_airbyte_extracted_at` for streaming and a business date for re-emit
+
+Streaming connectors land rows in bronze approximately when they
+happen — `_airbyte_extracted_at` tracks reality. Report-style
+connectors re-fetch a fixed window every run: even when the upstream
+stops publishing, Airbyte still writes "fresh" rows for older business
+days, so `_airbyte_extracted_at` keeps advancing forever. The
+`feature-bronze-freshness-sla/FEATURE.md` example (M365 9 h fresh
+extracted-at vs 96 h-stale `reportRefreshDate` on 2026-05-04) is the
+canonical evidence. Anchoring on the business-date column flips the
+verdict to ERROR, which is correct.
+
+### 4.3 Why per-driver subblocks instead of a flat `notification.url`
+
+Different drivers need different *non-credential* settings (Zulip
+stream/topic, Slack channel, SMTP host/port/from/to/subjectPrefix). A
+flat `notification.url` would force these into either the URL itself
+(Zulip's `?stream=...` works; Slack's channel override does not always)
+or into an out-of-band lookup table. Per-driver subblocks let each
+driver own its shape; the dispatcher just reads the env vars it cares
+about. Adding a sixth driver is one Helm subblock + one match arm +
+one `secretKeyRef` branch — no other Helm surface changes (workflow
+template line 380–382, `values.yaml:195-201`).
+
+### 4.4 Why the Zulip dispatcher silently rewrites the endpoint
+
+Zulip exposes two incoming-webhook integrations under the same UI
+("Incoming webhook"): `/api/v1/external/json` (intended for debugging,
+dumps body as JSON code block) and `/api/v1/external/slack`
+(Slack-compatible, renders mrkdwn). The UI does not warn that the
+former is the wrong integration for routine notifications. Rather
+than require operators to know the Zulip-internal distinction, the
+dispatcher rewrites the path on the way out (workflow template line
+457). This is documented in the chart values comment
+(`values.yaml:230-239`) and in the dispatcher's docstring (lines
+445–478).
+
+### 4.5 Why narrow the selector at runtime
+
+`source:*` against a partial deployment (e.g. a tenant who hasn't
+deployed OpenAI) would emit one ERROR per missing source on every run,
+flooding on-call with noise about something they deliberately didn't
+deploy. Narrowing to `bronze_*` databases that exist in
+`system.databases` (workflow template lines 262–297) keeps the check
+scoped to "what we collect", and explicitly leaves "what should be
+collected" to the deployment-health workstream
+([#272](https://github.com/cyberfabric/insight/issues/272)).
+
+### 4.6 Why the trap detector is advisory
+
+The freshness check is the page-worthy signal — its threshold tiers are
+documented per source, its semantics are deterministic, its output is
+the canonical payload. The trap detector is heuristic — its thresholds
+(`SUSPECT_PCT_RECENT = 95.0`, `SUSPECT_MAX_DISTINCT_DAYS = 2`,
+`MIN_ROWS = 100`, `RECENT_WINDOW_HOURS = 30`) are hardcoded constants
+in [`freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py)
+rather than Helm-tunable values, so retuning is a code change with a
+PR review attached. False positives on a heuristic that pages on-call
+would erode trust in the whole monitoring domain. So the trap detector
+logs, the freshness parser owns the verdict (workflow template lines
+575–579).
+
+### 4.7 Why every `freshness: null` opt-out needs a rationale
+
+Bare `freshness: null` is too easy to leave in by accident. Once it
+lands, the table is invisible to monitoring forever, with no comment
+explaining why. The convention is a one-line rationale comment beside
+each opt-out, kept grep-friendly: `grep -A1 'freshness: null'` shows
+what every opt-out is for. Opt-out is reserved for incremental streams
+that legitimately go quiet — a full-refresh roster/lookup keeps
+`_airbyte_extracted_at` as a sync-liveness signal instead.
+
+## 5. Traceability
+
+- Feature spec: [`feature-bronze-freshness-sla/FEATURE.md`](feature-bronze-freshness-sla/FEATURE.md)
+- Operator runbook: [`src/ingestion/MONITORING.md`](../../../../src/ingestion/MONITORING.md)
+- Implementation is split across two PRs into `constructorfabric/insight`:
+ - **Runtime / alerting** (this module — branch `feat/ingestion-freshness-runtime`, closes #949, refs #1321):
+ - Workflow + parser: [`charts/insight/templates/ingestion/dbt-source-freshness.yaml`](../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml)
+ - Helm surface: [`charts/insight/values.yaml`](../../../../charts/insight/values.yaml)
+ - Schema: [`charts/insight/values.schema.json`](../../../../charts/insight/values.schema.json)
+ - Trap detector: [`src/ingestion/scripts/freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py)
+ - **Per-source declarations** (EPIC #1321 / #1322): PR #1346 (base freshness blocks) + PR SharedQA/insight#1 (business-date anchors + tiers) — `src/ingestion/connectors/*/*/dbt/schema.yml`.
+- Future work cross-references:
+ - Deployment health — [issue #272](https://github.com/constructorfabric/insight/issues/272).
+ - Volume baseline / source-vs-bronze attribution / ownership — "Open work" sections of [`MONITORING.md`](../../../../src/ingestion/MONITORING.md).
diff --git a/docs/domain/ingestion-monitoring/specs/PRD.md b/docs/domain/ingestion-monitoring/specs/PRD.md
new file mode 100644
index 000000000..feeeea154
--- /dev/null
+++ b/docs/domain/ingestion-monitoring/specs/PRD.md
@@ -0,0 +1,730 @@
+---
+status: proposed
+date: 2026-05-04
+---
+
+# PRD — Ingestion Monitoring
+
+
+
+- [1. Overview](#1-overview)
+ - [1.1 Purpose](#11-purpose)
+ - [1.2 Background / Problem Statement](#12-background--problem-statement)
+ - [1.3 Goals (Business Outcomes)](#13-goals-business-outcomes)
+ - [1.4 Glossary](#14-glossary)
+- [2. Actors](#2-actors)
+ - [2.1 Human Actors](#21-human-actors)
+ - [2.2 System Actors](#22-system-actors)
+- [3. Operational Concept & Environment](#3-operational-concept--environment)
+ - [3.1 Module-Specific Environment Constraints](#31-module-specific-environment-constraints)
+ - [3.2 Expected Scale](#32-expected-scale)
+- [4. Scope](#4-scope)
+ - [4.1 In Scope](#41-in-scope)
+ - [4.2 Out of Scope](#42-out-of-scope)
+- [5. Functional Requirements](#5-functional-requirements)
+ - [5.1 Freshness Check](#51-freshness-check)
+ - [5.2 Threshold Tiers](#52-threshold-tiers)
+ - [5.3 Opt-Out Hygiene](#53-opt-out-hygiene)
+ - [5.4 Trap Detection](#54-trap-detection)
+ - [5.5 Notification Delivery](#55-notification-delivery)
+ - [5.6 Identity Labels](#56-identity-labels)
+- [6. Non-Functional Requirements](#6-non-functional-requirements)
+ - [6.1 NFR Inclusions](#61-nfr-inclusions)
+ - [6.2 NFR Exclusions](#62-nfr-exclusions)
+- [7. Public Library Interfaces](#7-public-library-interfaces)
+ - [7.1 Public API Surface](#71-public-api-surface)
+ - [7.2 External Integration Contracts](#72-external-integration-contracts)
+- [8. Use Cases](#8-use-cases)
+- [9. Acceptance Criteria](#9-acceptance-criteria)
+- [10. Dependencies](#10-dependencies)
+- [11. Assumptions](#11-assumptions)
+- [12. Risks](#12-risks)
+
+
+
+## 1. Overview
+
+### 1.1 Purpose
+
+Catch ingestion-layer breaches — bronze tables that have stopped receiving
+fresh rows, or whose freshness anchor is masking a re-emit pattern — before
+the silence propagates to silver/gold metrics and to the dashboard. The
+domain owns observability *of what we collect*; it deliberately does not
+own deployment health, volume baselines, or vendor-job attribution (see
+[§4.2](#42-out-of-scope)).
+
+### 1.2 Background / Problem Statement
+
+The ingestion pipeline previously had no signal for "data is missing".
+Connectors that silently stopped emitting rows (expired tokens, upstream
+API outages, Argo CronWorkflow failures, syncs that ran but produced 0
+rows) only became visible when a downstream metric went flat days later.
+
+Two classes of silent failure motivated the domain:
+
+1. **Sync stops, anchor goes stale.** The straightforward case — Airbyte
+ connector dies, no new rows land, `MAX(_airbyte_extracted_at)` ages
+ out of the warn window.
+2. **Sync runs, anchor stays green, upstream has stopped publishing.**
+ The trap case. Some connectors (Microsoft Graph reports, Slack
+ admin.analytics, Cursor daily_usage, Confluence page versions)
+ re-fetch a fixed window every run. Even when the upstream advances
+ no new business days, Airbyte keeps writing rows for older days, so
+ `_airbyte_extracted_at` looks fresh forever. The local CH dump on
+ 2026-05-04 had M365 9 hours fresh on `_airbyte_extracted_at` but
+ `reportRefreshDate` 96 hours behind reality (see
+ [feature-bronze-freshness-sla/FEATURE.md](feature-bronze-freshness-sla/FEATURE.md)).
+
+### 1.3 Goals (Business Outcomes)
+
+- Every Airbyte-managed `bronze_*` source has a verifiable
+ PASS/WARN/ERROR freshness verdict produced once per day.
+- Adding a new connector gains monitoring without per-pipeline
+ plumbing — a `freshness:` block in the connector's `schema.yml` is
+ enough.
+- Operators receive breaches via a notification driver of their choice
+ (webhook, Zulip, Slack, Teams, email) — without exposing the channel's
+ credentials in rendered manifests or in the Argo UI.
+- Connector authors who pick the wrong anchor are caught either at PR
+ time (CI lint) or at runtime (trap detector), not by silent green
+ dashboards.
+
+### 1.4 Glossary
+
+| Term | Meaning |
+|---|---|
+| Bronze | The first ClickHouse-side schema written by Airbyte: `bronze_.`. |
+| Freshness anchor | The SQL expression in `loaded_at_field` against which `dbt source freshness` measures lag. |
+| Streaming connector | Connector whose Airbyte cursor follows business time; rows land in bronze approximately when they happen. Anchor = `_airbyte_extracted_at`. |
+| Report-style connector | Connector that re-fetches a fixed window every run (Microsoft Graph reports, Slack admin.analytics). Anchor = a business-date column. |
+| Event-style connector | Streaming connector with legitimate quiet days (Confluence edits, Zoom meetings). Uses a wider tier. |
+| SLA tier | A pair `(warn_after, error_after)` shared by connectors with similar cadence: `default`, `event`, `report`, `report_extended`. |
+| Trap | A bronze table whose `_airbyte_extracted_at` looks fresh while the upstream has stopped publishing — the freshness check passes for the wrong reason. |
+| Driver | A notification dispatch backend selected via `ingestion.freshness.notification.driver`: `webhook` / `zulip` / `slack` / `teams` / `email`. |
+
+## 2. Actors
+
+### 2.1 Human Actors
+
+#### Ingestion On-Call
+
+Triages freshness breaches. Reads Argo workflow status / notification
+payloads. Acts on `error` and `runtime error` runs.
+
+#### Connector Owner
+
+Owns a specific `bronze_*` source. Receives an issue handed off by
+on-call when their connector breaches. Fixes the connector or revisits
+the SLA tier.
+
+#### Tenant On-Call (post-MVP)
+
+Per-deployment triage when one bot fans out to multiple installations.
+Reads the notification payload's `cluster` / `tenant` labels. Same
+triage as ingestion on-call, scoped to one deployment.
+
+Rotation / `CODEOWNERS` for `src/ingestion/connectors/` is **not yet
+assigned** — see
+[`src/ingestion/MONITORING.md` §"Rotation / ownership — not assigned"](../../../../src/ingestion/MONITORING.md).
+
+### 2.2 System Actors
+
+#### `dbt-source-freshness` WorkflowTemplate (Argo)
+
+Stateless template — runs `dbt source freshness`, parses
+`target/sources.json`, dispatches to the active driver. Defined in
+[`charts/insight/templates/ingestion/dbt-source-freshness.yaml`](../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml).
+
+#### `dbt-source-freshness-check` CronWorkflow (Argo)
+
+Schedules the template once a day at 13:00 UTC; carries the
+per-deployment Helm config into the template's parameters. Same file.
+
+#### Inline freshness parser (Python)
+
+Reads `target/sources.json`, classifies breaches, calls the driver
+dispatcher, sets the workflow exit code. Inline in the workflow
+template, lines 317–562.
+
+#### Trap detector
+
+[`src/ingestion/scripts/freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py)
+— advisory script that spots full-reemit and incremental-topup patterns
+the dbt freshness check cannot see. Runs after the parser.
+
+#### Coverage gate (QA-owned, out of this module's scope)
+
+Per-source declaration coverage is measured by the QA-owned
+`scripts/ci/dbt_coverage.py` gate (EPIC #1321 / #1322), which counts every
+`bronze_*` source that declares `loaded_at_field` + `freshness`. This module
+relies on that gate rather than shipping its own connector-schema lint.
+
+## 3. Operational Concept & Environment
+
+### 3.1 Module-Specific Environment Constraints
+
+- Production: Argo Workflows + ClickHouse in a Kubernetes cluster; the
+ `dbt-source-freshness-check` CronWorkflow runs once per day at 13:00
+ UTC, sitting past every connector's sync window (02:00–11:00 UTC) plus
+ a 2 h grace.
+- Local development: Kind K8s cluster (`insight`) with the same Helm
+ values; ClickHouse reachable on `host.docker.internal:8123` for the
+ trap detector.
+- The `insight-toolbox` image must carry `dbt`, `dbt-clickhouse`, and
+ the freshness trap detector (`freshness-trap-detect.py`).
+- Notification driver credentials must live in Kubernetes Secrets
+ referenced by `secretKeyRef`; the rendered `WorkflowTemplate` /
+ `CronWorkflow` YAML must never carry the raw URL or password.
+
+### 3.2 Expected Scale
+
+| Dimension | Current | Projected |
+|---|---|---|
+| Bronze sources monitored | 12 (one per declared `bronze_*` schema) | 50+ |
+| Bronze tables monitored | ~80 (82 entries across 12 sources today) | 200+ |
+| Notification drivers supported | 5 | 5 (stable) |
+| SLA tiers | 4 | 4 (stable) |
+| Workflow runtime (warm CH) | ~2 min | < 5 min |
+| `activeDeadlineSeconds` budget | 1200 s | 1200 s |
+
+## 4. Scope
+
+### 4.1 In Scope
+
+- **Bronze freshness SLA** — the `dbt source freshness` check against
+ `_airbyte_extracted_at` (incremental) or a business-date column
+ (windowed), with four threshold tiers (`default` 36/72 h, `event`
+ 96/168 h, `report` 72/120 h, `report_extended` 120/168 h). Source:
+ [`feature-bronze-freshness-sla/FEATURE.md`](feature-bronze-freshness-sla/FEATURE.md).
+ The per-source declarations themselves are owned by the connector-schema
+ work (EPIC #1321 / #1322 — PR #1346 + PR SharedQA/insight#1), not this module.
+- **Runtime trap detector** — heuristic full-reemit detection (no
+ config) plus opt-in `meta.bronze_business_date_col` divergence check;
+ advisory only, never affects workflow exit code. Source:
+ [`freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py).
+- **Multi-driver notification fan-out** — five drivers
+ (webhook / zulip / slack / teams / email) selectable via Helm; URL /
+ SMTP password sourced from a Kubernetes Secret so the credential
+ never appears in rendered manifests or Argo UI. Source: workflow
+ template lines 161–194 and 412–540.
+- **Identity labels in payload** — `cluster` and `tenant` carried into
+ every payload's summary prefix and JSON body so a shared bot can
+ route between installations.
+
+### 4.2 Out of Scope
+
+- **Deployment health** — "is a connector that should be deployed
+ actually deployed?" is a separate concern (Helm probes / sync
+ workflow status). Tracked as
+ [issue #272](https://github.com/cyberfabric/insight/issues/272). The
+ freshness workflow narrows its selector to bronze databases that
+ exist (workflow template lines 254–297) precisely so it does *not*
+ alert on connectors a tenant hasn't deployed.
+- **Volume baseline anomaly detection** — "API returned 50 rows
+ instead of 5000". Listed under "Open work" in
+ [`MONITORING.md`](../../../../src/ingestion/MONITORING.md).
+- **Source-vs-bronze attribution** — distinguishing "Airbyte sync
+ failed" from "Airbyte sync ran but pulled 0 rows" requires polling
+ Airbyte's Jobs API; listed under "Open work" in
+ [`MONITORING.md`](../../../../src/ingestion/MONITORING.md).
+- **Notification rotation / `CODEOWNERS`** — see
+ [`MONITORING.md` §"Rotation / ownership — not assigned"](../../../../src/ingestion/MONITORING.md).
+
+## 5. Functional Requirements
+
+### 5.1 Freshness Check
+
+#### Daily Argo CronWorkflow
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-daily-cronworkflow`
+
+The system **MUST** schedule a single Argo `CronWorkflow`
+(`dbt-source-freshness-check`) that runs once per day on the schedule
+supplied by `ingestion.freshness.schedule` (default `0 13 * * *`,
+charts/insight/values.yaml:151).
+
+#### Selector Narrowing
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-selector-narrowing`
+
+The workflow **MUST** narrow the dbt selector from `source:*` to the
+`bronze_*` databases that actually exist in `system.databases` (workflow
+template lines 262–297) — connectors a tenant did not deploy never
+produce a phantom ERROR.
+
+#### Stale Report Cleanup
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-stale-report-cleanup`
+
+The workflow **MUST** remove a stale `target/sources.json` before each
+run (workflow template line 252) so a mid-run dbt crash maps to exit
+code 2 rather than misclassifying using leftover data.
+
+#### Exit Code Re-Derivation
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-exit-rederivation`
+
+`dbt source freshness` is invoked with `--select ` and its
+exit code **MUST** be swallowed (line 305–307). The Python parser
+re-derives the workflow outcome from `target/sources.json`.
+
+### 5.2 Threshold Tiers
+
+#### Single Source of Truth
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-thresholds-sot`
+
+Each source's thresholds **MUST** be a single source of truth: literal
+`warn_after`/`error_after` values declared in that source's `schema.yml`.
+The CronWorkflow runs `dbt source freshness --select source:*` and reads
+exactly what each source declares — no Helm/env-var threshold layer, so there
+is no second place a value can drift.
+
+#### Four Tiers
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-four-tiers`
+
+The system **MUST** support exactly four tiers, each with documented
+rationale (applied as literal values per source):
+
+- `default` (36/72 h) — incremental connectors on daily cron cadence.
+- `event` (96/168 h) — natural-quiet-day connectors (Confluence
+ edits, Zoom meetings) where a zero-row weekend is normal.
+- `report` (72/120 h) — windowed vendor analytics with documented 24–48 h
+ publish lag (Microsoft Graph reports, ChatGPT/Claude Team baseline).
+- `report_extended` (120/168 h) — windowed vendor analytics with ~3-day
+ baseline lag (Slack admin.analytics typical).
+
+#### Tier Assignment
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-tier-assignment`
+
+Tier assignment **MUST** be per-connector and live as literal values in the
+connector's `schema.yml`. Re-tiering is an engineering change committed to the
+source declaration — there is no runtime tuning knob.
+
+### 5.3 Opt-Out Hygiene
+
+#### Per-Table Opt-Out
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-optout-pertable`
+
+Slow-moving roster / catalog tables (e.g. `cursor_members`,
+`wiki_spaces`) **MUST** be allowed to opt out via `freshness: null`
+per-table in `schema.yml`.
+
+#### Mandatory Rationale
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-optout-rationale`
+
+Every `freshness: null` opt-out **MUST** carry a one-line rationale comment
+beside it so the audit surface stays grep-able, and is reserved for
+*incremental* streams that legitimately go quiet (a full-refresh roster keeps
+`_airbyte_extracted_at` as a sync-liveness signal instead of opting out).
+
+#### Mandatory Anchor
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-mandatory-anchor`
+
+The lint **MUST** reject any `bronze_*` source with no reachable
+`loaded_at_field` (source-level, per-table, or explicit opt-out).
+
+### 5.4 Trap Detection
+
+#### Post-Parser Execution
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-trap-post-parser`
+
+After the freshness parser exits, the workflow **MUST** run
+[`freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py)
+(workflow template lines 564–579).
+
+#### Two Detection Modes
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-trap-two-modes`
+
+The detector **MUST** support two modes:
+
+- **Full re-emit (heuristic, no config)** — fires when ≥ 95 % of
+ rows have `_airbyte_extracted_at` within the last 30 h *and*
+ `_airbyte_extracted_at` covers ≤ 2 distinct calendar days *and*
+ the table has ≥ 100 rows (script constants
+ `SUSPECT_PCT_RECENT`, `SUSPECT_MAX_DISTINCT_DAYS`,
+ `RECENT_WINDOW_HOURS`, `MIN_ROWS`).
+- **Business-date divergence (opt-in)** — sources that declare
+ `meta.bronze_business_date_col: ` get
+ `MAX()` compared with `MAX(_airbyte_extracted_at)`; a
+ ≥ 24 h gap is flagged.
+
+#### Skip Annotation
+
+- [ ] `p2` - **ID**: `cpt-insightspec-fr-mon-trap-skip-annotation`
+
+A source **MUST** be allowed to opt out of mode 1 via
+`meta.bronze_freshness_trap_check: skip` at source or table level.
+
+#### Advisory-Only Verdict
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-trap-advisory`
+
+Findings **MUST** be advisory: `trap_exit=1` only logs a notice. The
+workflow's page/no-page decision remains owned by the freshness
+parser (workflow template lines 575–579).
+
+### 5.5 Notification Delivery
+
+#### Driver Selection
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-driver-selection`
+
+`ingestion.freshness.notification.driver` **MUST** select one of `""`,
+`webhook`, `zulip`, `slack`, `teams`, `email`
+(charts/insight/values.yaml:220, schema enum at
+charts/insight/values.schema.json:37).
+
+#### Credential Isolation
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-credential-isolation`
+
+Each URL-driven driver **MUST** read its credential from a
+`secretKeyRef` branched in the workflow template (lines 161–194). The
+URL is bound to `NOTIFICATION_URL` only when the driver is configured;
+email's SMTP password binds to `NOTIFICATION_EMAIL_SMTP_PASSWORD`.
+Plain settings (Zulip stream, Slack channel, SMTP host/port) flow
+through normal parameters because they are not credentials.
+
+#### Dispatch Implementation
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-dispatch-impl`
+
+Driver dispatch **MUST** happen in the inline parser via the
+`dispatchers` dict (workflow template lines 533–539).
+
+#### Delivery Failure Isolation
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-delivery-failure-isolation`
+
+Notification failures **MUST** be caught and logged (lines 550–554) —
+they never change the workflow's primary exit code (the breach signal
+is more important than the delivery success).
+
+### 5.6 Identity Labels
+
+#### Helm Plumbing
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-identity-helm`
+
+`ingestion.freshness.cluster` and `.tenant` (values.yaml:193–194)
+**MUST** flow into the parser as `CLUSTER` / `TENANT` env vars.
+
+#### Summary Prefix
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-identity-summary`
+
+The summary line **MUST** carry every set label as a
+`[cluster=…, tenant=…] ` prefix (workflow template lines 393–397);
+empty labels drop out so single-deployment installs see plain
+`N bronze source(s) breaching ...`.
+
+#### Payload Routing Fields
+
+- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-identity-payload`
+
+The webhook payload **MUST** include both `cluster` and `tenant` as
+raw fields so receivers can route by either dimension.
+
+## 6. Non-Functional Requirements
+
+### 6.1 NFR Inclusions
+
+#### Idempotency
+
+- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-idempotency`
+
+Re-running the workflow on the same data **MUST** produce the same exit
+code and the same payload (no stateful side-effects,
+`target/sources.json` is purged before each run).
+
+#### Daily Cadence
+
+- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-daily-cadence`
+
+The default cadence **MUST** be daily; the cron schedule is Helm-tunable
+but the parser is stateless and can run ad-hoc.
+
+#### Credential Isolation
+
+- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-credential-isolation`
+
+Webhook URL / SMTP password **MUST** be pulled from k8s Secrets via
+`secretKeyRef`; the rendered `WorkflowTemplate` / `CronWorkflow` YAML
+and the Argo UI never see the raw value.
+
+#### Deterministic Exit Codes
+
+- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-exit-codes`
+
+The parser **MUST** emit deterministic exit codes: `0` no breach or
+warn-only; `1` at least one `error` / `runtime error`; `2`
+`target/sources.json` missing (dbt crashed).
+
+#### CI-Gated Schema
+
+- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-ci-gated`
+
+Per-source freshness declaration coverage **MUST** be gated in CI by the
+QA-owned `scripts/ci/dbt_coverage.py` (EPIC #1321 / #1322) — this module does
+not own a connector-schema lint of its own.
+
+#### Advisory-Fail Mode for Traps
+
+- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-trap-advisory-mode`
+
+Trap detector findings **MUST NOT** override the freshness parser's
+verdict (page-worthy stays page-worthy; warn-only stays warn-only).
+
+#### Activation Deadline
+
+- [ ] `p2` - **ID**: `cpt-insightspec-nfr-mon-activation-deadline`
+
+The workflow **MUST** carry `activeDeadlineSeconds: 1200` (workflow
+template line 110); current connector set finishes in ~2 min on a
+warm CH.
+
+#### Empty-Table Sentinel
+
+- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-empty-sentinel`
+
+A source with zero rows yields `MAX(...) = NULL → 1970-01-01T…`; the
+parser **MUST** flag it explicitly as `(table is empty — no rows ever
+ingested)` rather than reporting ~500 000 h of lag (workflow template
+lines 348–360).
+
+### 6.2 NFR Exclusions
+
+- ClickHouse cluster performance tuning (infrastructure concern).
+- Argo Workflows controller HA (infrastructure concern).
+- Notification receiver SLAs (Zulip / Slack / Teams uptime is out of
+ this domain's control).
+- Vendor-side publish-lag changes (Microsoft Graph / Slack
+ admin.analytics window changes are tracked in `MONITORING.md`, not
+ enforced as NFR thresholds here).
+
+## 7. Public Library Interfaces
+
+### 7.1 Public API Surface
+
+Not applicable — this domain has no public library API. The monitoring
+surface is consumed via Helm values + Argo UI + dbt, not via a Rust /
+Python / TypeScript library exposed to other domains. See
+[DESIGN §3.3](DESIGN.md#33-api-contracts) for the workflow / dbt /
+notification protocol contracts.
+
+### 7.2 External Integration Contracts
+
+#### Webhook Driver Contract
+
+Generic JSON POST to `NOTIFICATION_URL`; canonical payload shape is
+documented in [DESIGN §3.4 / §3.6](DESIGN.md#34-driver-contracts).
+Carries `topic`, `cluster`, `tenant`, `summary`, `breaches[]`.
+
+#### Zulip Driver Contract
+
+POST to a Zulip incoming-webhook URL. The dispatcher rewrites
+`/api/v1/external/json` to `/api/v1/external/slack` so operators who
+pasted the JSON URL still get formatted output. Form-urlencoded body
+with Slack mrkdwn. See [DESIGN §3.4.2](DESIGN.md#342-zulip).
+
+#### Slack Driver Contract
+
+POST to a Slack incoming-webhook URL with `application/json`
+`{"text": ""}`, optionally with `channel` override. See
+[DESIGN §3.4.3](DESIGN.md#343-slack).
+
+#### Teams Driver Contract
+
+POST to a Microsoft Teams incoming webhook with `application/json`
+`MessageCard`. `themeColor` is `FF0000` (red) when any breach is
+page-worthy, `FFA500` (orange) otherwise. See
+[DESIGN §3.4.4](DESIGN.md#344-teams).
+
+#### Email Driver Contract
+
+`smtplib`-based dispatch (no extra deps). `SMTP_SSL` when port=465,
+otherwise plain `SMTP` with optional `STARTTLS`. Required Helm values:
+`notification.email.smtp.host`, `notification.email.from`,
+`notification.email.to`. See [DESIGN §3.4.5](DESIGN.md#345-email).
+
+## 8. Use Cases
+
+#### Use Case 1: Daily Freshness Check
+
+- [ ] `p1` - **ID**: `cpt-insightspec-usecase-mon-daily-check`
+
+**Actors**: `dbt-source-freshness-check` CronWorkflow,
+`dbt-source-freshness` WorkflowTemplate, ClickHouse, ingestion on-call.
+
+**Preconditions**: Helm values `ingestion.freshness.enabled: true`,
+the `bronze_*` databases exist in `system.databases`, the dbt project
+declares `loaded_at_field` for every source.
+
+**Main Flow**:
+
+1. Argo triggers `dbt-source-freshness-check` at the configured cron.
+2. The workflow narrows `source:*` to bronze databases that exist.
+3. `dbt source freshness` runs against ClickHouse and writes
+ `target/sources.json`.
+4. The inline Python parser classifies every source as PASS / WARN /
+ ERROR / RUNTIME ERROR.
+5. The trap detector runs and logs any heuristic findings without
+ changing the verdict.
+6. The driver dispatcher posts to the configured channel whenever the
+ `breaches` list is non-empty — including `warn`-only runs, so
+ on-call sees latency creeping up before it crosses into
+ page-worthy territory.
+7. The workflow exits with `1` only when at least one breach is
+ `error` or `runtime error`; `warn`-only runs exit `0` so the
+ payload is informational, not paging.
+
+**Postconditions**: A canonical breach record exists in the workflow
+log; if a driver is configured and any breach was page-worthy, the
+external sink received the payload.
+
+#### Use Case 2: Triage a Paging Breach
+
+- [ ] `p1` - **ID**: `cpt-insightspec-usecase-mon-triage-breach`
+
+**Actors**: ingestion on-call, connector owner.
+
+**Preconditions**: a notification carrying `[cluster=…, tenant=…] N
+bronze source(s) breaching freshness SLA` has fired.
+
+**Main Flow**:
+
+1. On-call opens the payload, identifies the breaching `source` and
+ `age_hours`.
+2. On-call cross-references the source's tier in
+ [DESIGN §3.5](DESIGN.md#35-sla-tier-mapping) to confirm it was
+ page-worthy (not warn-only).
+3. On-call inspects the connector's last sync in Argo / Airbyte UI to
+ distinguish "sync stopped" from "sync ran but produced no rows".
+4. On-call hands off to the connector owner with the payload as
+ evidence.
+5. Connector owner fixes the connector or revisits the SLA tier in
+ `schema.yml`.
+
+**Postconditions**: Either the connector is healthy on the next daily
+run, or the tier has been changed (with rationale in commit message).
+
+#### Use Case 3: Add a New Bronze Source
+
+- [ ] `p1` - **ID**: `cpt-insightspec-usecase-mon-add-bronze-source`
+
+**Actors**: connector author, CI lint.
+
+**Preconditions**: a new connector under
+`src/ingestion/connectors///` with a `dbt/schema.yml`.
+
+**Main Flow**:
+
+1. Author declares `loaded_at_field` at source or table level
+ (`_airbyte_extracted_at` for incremental, a
+ `parseDateTimeBestEffortOrNull()` expression for windowed).
+2. Author sets the literal tier `warn_after`/`error_after` on the source's
+ `freshness:` block (`default`/`report`/`report_extended`/`event`).
+3. Author opens a PR. The `dbt_coverage.py` gate (EPIC #1321) confirms the
+ source is declared; the windowed-vs-incremental call is confirmed against
+ live data (`ext_age` vs `biz_age`), not by a lint.
+4. PR merges; the next daily freshness run includes the new source.
+
+**Postconditions**: The new source is monitored on the same SLA tier
+as comparable connectors with no further plumbing.
+
+#### Use Case 4: Switch Notification Driver in a Tenant Overlay
+
+- [ ] `p2` - **ID**: `cpt-insightspec-usecase-mon-switch-driver`
+
+**Actors**: platform engineer.
+
+**Preconditions**: a Helm values overlay for the target tenant; a
+Kubernetes Secret carrying the new driver's URL or SMTP password.
+
+**Main Flow**:
+
+1. Engineer creates / updates the Secret in the same namespace as the
+ workflow.
+2. Engineer sets `ingestion.freshness.notification.driver: ` and
+ the driver's `urlSecret.{name,key}` (or `email.smtp.*`) in the
+ overlay.
+3. `helm upgrade` re-renders the `WorkflowTemplate`; the `secretKeyRef`
+ binding switches to the new driver's branch.
+4. The next scheduled run dispatches to the new channel; the previous
+ driver's secret reference is no longer rendered.
+
+**Postconditions**: The tenant's freshness payload now arrives in the
+new channel; rendered manifests and Argo UI carry the `secretKeyRef`,
+not the credential.
+
+## 9. Acceptance Criteria
+
+- [ ] Every `bronze_*` source under `src/ingestion/connectors/*/*/dbt/schema.yml` declares `loaded_at_field` + a literal `freshness:` block (source / table / opt-out) — counted by the QA-owned `dbt_coverage.py` gate (EPIC #1321).
+- [ ] Every `freshness: null` opt-out carries a one-line rationale comment and is an incremental (not full-refresh) stream.
+- [ ] Windowed connectors anchor on a business-date column, not `_airbyte_extracted_at` — confirmed by `ext_age` vs `biz_age` on live data where rows exist.
+- [ ] `dbt source freshness --select 'source:bronze_*'` from a clean checkout produces `target/sources.json` with one result per declared (source, table) pair, no `runtime error` due to missing `loaded_at_field`.
+- [ ] Each tier produces the expected verdict on its representative connectors:
+ - `default` — Bitbucket, Jira, BambooHR, Cursor (non-daily), GitHub, OpenAI, Claude (PASS within 30 h of last sync).
+ - `event` — Confluence `wiki_page_versions`, Zoom `meetings`/`participants` (PASS within 72 h, weekend-tolerant).
+ - `report` — M365 `*_activity` (PASS within 48 h of `reportRefreshDate`).
+ - `report_extended` — Slack `users_details` (PASS within 72 h of `date`).
+- [ ] A stale-data fixture (truncate or backdate `_airbyte_extracted_at` on one source) produces parser exit code 1, a log line naming the source / `max_loaded_at` / `age_hours`, and a notification payload (when a driver is configured).
+- [ ] The notification payload always carries both `cluster` and `tenant` keys (empty string when unset), matching the canonical webhook contract documented in [`MONITORING.md`](../../../../src/ingestion/MONITORING.md).
+- [ ] The configured driver's credential **does not appear** in `kubectl get workflowtemplate dbt-source-freshness -o yaml` or in `argo get` output — only the `secretKeyRef` does.
+- [ ] An empty `bronze_*` table is reported as `(table is empty)` instead of `~500000h` lag (workflow template lines 348–360).
+- [ ] Synthetic Confluence-style fixture (full table re-emit within 24 h, 1 distinct extract day, ≥ 100 rows) is flagged by the trap detector with `kind=full-reemit` (script lines 188–200).
+- [ ] Synthetic incremental-topup fixture (`MAX(_airbyte_extracted_at)` fresh, `MAX()` ≥ 24 h behind) is flagged with `kind=incremental-topup` (script lines 202–227).
+- [ ] `values.schema.json` rejects an unknown driver name (only `""`, `webhook`, `zulip`, `slack`, `teams`, `email` are accepted — schema line 37).
+
+## 10. Dependencies
+
+- **Argo Workflows** controller in the `argo` namespace — provides
+ `WorkflowTemplate` / `CronWorkflow` reconciliation and the
+ `failedJobsHistoryLimit` retention used as the no-fan-out fallback.
+- **dbt-clickhouse** adapter inside the `insight-toolbox` image —
+ provides `dbt source freshness` and `target/sources.json`. The
+ adapter does not support metadata-based freshness, so a missing
+ `loaded_at_field` produces `runtime error` rather than silent skip.
+- **ClickHouse** — readable `system.databases` (for selector
+ narrowing) and HTTP interface on port 8123 (used by the trap
+ detector).
+- **Kubernetes Secret access** — `secretKeyRef` for both
+ `clickhouse.passwordSecret` and the active driver's credential
+ Secret.
+- **Helm umbrella chart** — `ingestion.freshness.*` block in
+ `charts/insight/values.yaml`, validated by
+ `charts/insight/values.schema.json`.
+
+## 11. Assumptions
+
+- Every Airbyte-managed `bronze_*` source carries a usable
+ `_airbyte_extracted_at` column or a business-date column suitable as
+ a freshness anchor.
+- Vendor publish-lag windows (Microsoft Graph 24–48 h; Slack
+ admin.analytics ~3 d) remain within the configured tier bounds; if a
+ vendor changes its cadence, the affected source is re-tiered via a
+ schema.yml edit, not by widening the global tier.
+- Operators have access to a Kubernetes Secret store and accept
+ `secretKeyRef` as the credential carrier — no in-tree plaintext
+ fallback is provided.
+- Argo Workflows is healthy enough to schedule the daily run; if the
+ controller is down, monitoring goes silent (tracked under
+ "Open work" in `MONITORING.md`).
+
+## 12. Risks
+
+| Risk | Impact | Mitigation |
+|---|---|---|
+| Trap heuristic false positive (full-reemit mode 1) on a legitimate burst-load source | On-call sees a noisy advisory line | Findings are advisory only; opt out via `meta.bronze_freshness_trap_check: skip`. |
+| Driver receiver outage hides real breaches | Page-worthy breach silently dropped | Notification failure does not change the workflow's exit code; Argo's `failedJobsHistoryLimit` keeps the breach visible in the UI. |
+| Connector author picks the wrong anchor (uses `_airbyte_extracted_at` for a report-style source) | Trap-class silent green | CI lint requires an anchor; trap detector flags re-emit at runtime; PRD §1.2 documents the failure mode. |
+| Vendor changes its publish window | Tier bounds become wrong; pages too eagerly or too late | Re-tiering is a schema.yml edit; tier values are Helm-tunable as a safety valve. |
+| Multi-tenant fan-out from one bot mixes installations | On-call cannot route | `cluster` / `tenant` labels are mandatory in payload; receivers route by either dimension. |
diff --git a/docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md b/docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md
new file mode 100644
index 000000000..4d2e44dbb
--- /dev/null
+++ b/docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md
@@ -0,0 +1,502 @@
+---
+status: proposed
+date: 2026-05-04
+---
+
+# Feature: Bronze Freshness SLA
+
+
+
+- [1. Feature Context](#1-feature-context)
+ - [1.1 Overview](#11-overview)
+ - [1.2 Purpose](#12-purpose)
+ - [1.3 Actors](#13-actors)
+ - [1.4 References](#14-references)
+- [2. Actor Flows (CDSL)](#2-actor-flows-cdsl)
+ - [Trigger Freshness Check on Schedule](#trigger-freshness-check-on-schedule)
+ - [Triage a Page-Worthy Breach](#triage-a-page-worthy-breach)
+ - [Wire a New Connector into Freshness](#wire-a-new-connector-into-freshness)
+ - [Switch Notification Driver in a Tenant Overlay](#switch-notification-driver-in-a-tenant-overlay)
+ - [Override SLA Threshold for One Connector](#override-sla-threshold-for-one-connector)
+- [3. Processes / Business Logic (CDSL)](#3-processes--business-logic-cdsl)
+ - [Resolve dbt Selector to Existing Bronze Schemas](#resolve-dbt-selector-to-existing-bronze-schemas)
+ - [Classify Source Status (PASS / WARN / ERROR / RUNTIME ERROR)](#classify-source-status-pass--warn--error--runtime-error)
+ - [Render Notification Body per Driver](#render-notification-body-per-driver)
+ - [Detect Re-Emit Trap](#detect-re-emit-trap)
+ - [Choose SLA Tier per Source](#choose-sla-tier-per-source)
+- [4. States (CDSL)](#4-states-cdsl)
+ - [Source Freshness Status](#source-freshness-status)
+ - [Trap Suspect Lifecycle](#trap-suspect-lifecycle)
+- [5. Definitions of Done](#5-definitions-of-done)
+ - [Per-Source Thresholds](#per-source-thresholds)
+ - [Per-Table Opt-Out](#per-table-opt-out)
+ - [Workflow Execution](#workflow-execution)
+ - [Notification Delivery](#notification-delivery)
+ - [Trap Detection](#trap-detection)
+ - [Local Verification](#local-verification)
+- [6. Acceptance Criteria](#6-acceptance-criteria)
+
+
+
+Daily check that every Airbyte-managed `bronze_*` source has received fresh
+rows within the last warn-after window for its tier. The mechanism is a
+single dbt project policy plus an Argo `CronWorkflow` — no per-connector
+code changes are required for new sources to be covered.
+
+## 1. Feature Context
+
+### 1.1 Overview
+
+Today the ingestion pipeline has no signal for "data is missing".
+Connectors that silently stop emitting rows (expired tokens, upstream API
+outage, Argo CronWorkflow failures, sync ran but produced 0 rows) only
+become visible when a downstream metric goes flat days later.
+
+This feature wires `dbt source freshness` against the
+`_airbyte_extracted_at` column that every Airbyte ClickHouse destination
+writes per row, or against a business-date column for report-style
+connectors. A source is `pass` when `MAX()` is within the tier's
+warn-after window, `warn` between warn-after and error-after, `error` past
+error-after, and `runtime error` if the freshness query itself fails.
+
+### 1.2 Purpose
+
+Catch ingestion-layer breaches before they propagate to silver/gold
+metrics and to the dashboard. Keep the cost of adding new connectors low
+— a new connector is covered the moment it declares a `freshness:` block on
+its bronze source; the CronWorkflow picks it up via `source:*` with no
+per-connector plumbing.
+
+**Requirements**: `cpt-insightspec-fr-mon-daily-cronworkflow`,
+`cpt-insightspec-fr-mon-thresholds-sot`,
+`cpt-insightspec-fr-mon-four-tiers`,
+`cpt-insightspec-fr-mon-driver-selection`
+
+**Principles**: `cpt-insightspec-principle-mon-trap-advisory`,
+`cpt-insightspec-principle-mon-vendor-thresholds`
+
+### 1.3 Actors
+
+| Actor | Role in Feature |
+|---|---|
+| Ingestion on-call | Reads Argo workflow status; triages `error` runs |
+| Connector owner | Receives a follow-up issue from on-call when their connector breaches |
+| `dbt-source-freshness-check` CronWorkflow | Runs daily at 13:00 UTC, parses `target/sources.json`, fans the result out to the configured notification driver |
+| Notification consumer | Receives the webhook / Zulip / Slack / Teams / SMTP payload |
+
+### 1.4 References
+
+- Operational runbook: [`src/ingestion/MONITORING.md`](../../../../../src/ingestion/MONITORING.md) — verification steps, on-call matrix, parser exit codes, payload shape
+- Workflow template: [`charts/insight/templates/ingestion/dbt-source-freshness.yaml`](../../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml)
+- Threshold config: literal `warn_after`/`error_after` per source in each connector's `dbt/schema.yml` (declaration half tracked under EPIC #1321 / #1322 — PR #1346 + the per-connector business-date anchoring in PR SharedQA/insight#1)
+- PRD: [../PRD.md](../PRD.md)
+- DESIGN: [../DESIGN.md](../DESIGN.md)
+- Per-source declarations: every connector's `dbt/schema.yml` carries
+ `loaded_at_field` at source level (dbt does not propagate this
+ property from project config). Streaming connectors anchor on
+ `_airbyte_extracted_at`; report-style connectors (M365 Graph reports,
+ Slack admin.analytics) anchor on the report's own business-day column
+ wrapped in `parseDateTimeBestEffortOrNull(...)`.
+
+## 2. Actor Flows (CDSL)
+
+### Trigger Freshness Check on Schedule
+
+- [ ] `p1` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-trigger`
+
+**Actors**:
+- `dbt-source-freshness-check` CronWorkflow (Argo)
+- `dbt-source-freshness` WorkflowTemplate (Argo)
+- ClickHouse
+
+**Success Scenarios**:
+- Cron fires; selector narrowing finds at least one deployed `bronze_*`
+ database; `dbt source freshness` writes `target/sources.json`; parser
+ exits 0 (no breach) or 1 (breach) and dispatches when a driver is
+ configured.
+
+**Error Scenarios**:
+- `target/sources.json` missing because dbt crashed → parser exits 2.
+- ClickHouse unreachable → dbt fails; parser exits 2 because no report
+ was produced.
+
+**Steps**:
+1. [ ] - `p1` - Argo evaluates `ingestion.freshness.schedule` and spawns a Workflow from the template - `inst-cron-spawn`
+2. [ ] - `p1` - Workflow removes any stale `target/sources.json` (workflow template line 252) - `inst-clean-stale`
+3. [ ] - `p1` - Workflow queries `system.databases` for existing `bronze_*` schemas (lines 262–297) and builds the effective selector - `inst-narrow-selector`
+4. [ ] - `p1` - Workflow runs `dbt source freshness --select ` (lines 305–307) - `inst-run-dbt`
+5. [ ] - `p1` - Workflow invokes the inline Python parser - `inst-call-parser`
+6. [ ] - `p1` - Workflow invokes the trap detector after the parser; its exit code is captured but not used to override the parser's verdict - `inst-call-trap`
+7. [ ] - `p1` - Workflow exits with the parser's exit code - `inst-final-exit`
+
+### Triage a Page-Worthy Breach
+
+- [ ] `p1` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-triage`
+
+**Actors**:
+- Ingestion on-call
+- Connector owner
+
+**Success Scenarios**:
+- On-call reads the payload, identifies the source, hands off a tracked
+ issue to the connector owner; next run is green.
+
+**Error Scenarios**:
+- Payload missing `cluster` / `tenant` labels → on-call cannot route
+ multi-tenant fan-out; configuration bug to fix in Helm overlay.
+- Receiver outage hides the page → fall back to Argo
+ `failedJobsHistoryLimit` retention to find the breach in the UI.
+
+**Steps**:
+1. [ ] - `p1` - On-call receives the notification carrying `[cluster=…, tenant=…] N bronze source(s) breaching freshness SLA` - `inst-receive-page`
+2. [ ] - `p1` - On-call identifies the breaching `source` and `age_hours` from `breaches[]` - `inst-read-payload`
+3. [ ] - `p1` - On-call cross-references the source's tier in DESIGN to confirm page-worthiness - `inst-check-tier`
+4. [ ] - `p1` - On-call inspects last sync in Argo / Airbyte UI to distinguish "sync stopped" from "sync ran but produced no rows" - `inst-inspect-sync`
+5. [ ] - `p1` - On-call hands off to the connector owner with the payload as evidence - `inst-handoff`
+6. [ ] - `p1` - Connector owner fixes the connector or revisits the SLA tier in `schema.yml` - `inst-fix-connector`
+
+### Wire a New Connector into Freshness
+
+- [ ] `p1` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-wire`
+
+**Actors**:
+- Connector author
+- CI lint
+
+**Success Scenarios**:
+- New connector declares a `freshness:` block + `loaded_at_field` at source or
+ table level; the `dbt_coverage.py` gate counts it; next daily run includes it.
+
+**Error Scenarios**:
+- Author leaves a windowed connector on `_airbyte_extracted_at` → false-green
+ (no breach raised while data is stale). Caught by confirming `ext_age` vs
+ `biz_age` on live data, not by a lint.
+- Author anchors on a non-timestamp / wrong column → dbt emits `runtime error`
+ for that source (visible, paged), not a silent miss.
+
+**Steps**:
+1. [ ] - `p1` - Author adds a `sources:` entry with a literal `freshness:` block in the connector's `dbt/schema.yml` - `inst-add-source-entry`
+2. [ ] - `p1` - Author selects the anchor: `_airbyte_extracted_at` for incremental, a `parseDateTimeBestEffortOrNull()` expression for windowed - `inst-pick-anchor`
+3. [ ] - `p1` - Author sets the literal tier (`default`/`report`/`report_extended`/`event`) `warn_after`/`error_after` per source - `inst-pick-tier`
+4. [ ] - `p1` - Author opens a PR; the `dbt_coverage.py` gate (EPIC #1321) confirms the source is declared - `inst-coverage-gate`
+5. [ ] - `p1` - PR merges; the next daily freshness run includes the new source with no further plumbing - `inst-included-next-run`
+
+### Switch Notification Driver in a Tenant Overlay
+
+- [ ] `p2` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-switch-driver`
+
+**Actors**:
+- Platform engineer
+
+**Success Scenarios**:
+- Engineer flips the driver in a Helm overlay; `helm upgrade` re-renders
+ the WorkflowTemplate; next run dispatches to the new channel.
+
+**Error Scenarios**:
+- New driver's `urlSecret.name` references a missing Secret → workflow
+ pod fails to start; Argo controller logs surface the error; engineer
+ creates the Secret and retries.
+
+**Steps**:
+1. [ ] - `p2` - Engineer creates / updates the Secret carrying the new driver's URL or SMTP password - `inst-create-secret`
+2. [ ] - `p2` - Engineer sets `ingestion.freshness.notification.driver: ` and the driver's `urlSecret.{name,key}` (or `email.smtp.*`) in the overlay - `inst-edit-overlay`
+3. [ ] - `p2` - `helm upgrade` re-renders the WorkflowTemplate; the `secretKeyRef` binding switches to the new driver's branch - `inst-helm-upgrade`
+4. [ ] - `p2` - Engineer triggers an ad-hoc workflow run to confirm dispatch to the new channel - `inst-adhoc-run`
+
+### Override SLA Threshold for One Connector
+
+- [ ] `p2` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-override`
+
+**Actors**:
+- Connector author
+
+**Success Scenarios**:
+- Author re-tiers the connector by editing its `schema.yml`; next run
+ uses the new tier's warn / error pair.
+
+**Error Scenarios**:
+- Author tries to override thresholds at project level via
+ `+loaded_at_field` → silently ignored by dbt; PRD-level rationale +
+ CI lint redirect them to per-source declarations.
+
+**Steps**:
+1. [ ] - `p2` - Author updates the per-source `freshness:` block in `schema.yml` to reference a different tier env var (e.g. switch from `FRESHNESS_*_DEFAULT_H` to `FRESHNESS_*_REPORT_H`) - `inst-edit-tier`
+2. [ ] - `p2` - Author documents rationale in the commit message (vendor publish-lag evidence, observation window) - `inst-document-rationale`
+3. [ ] - `p2` - PR merges; the next run uses the new tier - `inst-take-effect`
+
+## 3. Processes / Business Logic (CDSL)
+
+### Resolve dbt Selector to Existing Bronze Schemas
+
+- [ ] `p1` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-resolve-selector`
+
+**Input**: configured `dbt_select` Helm value (default `source:*`),
+ClickHouse `system.databases` rows.
+
+**Output**: effective dbt selector string used in
+`dbt source freshness --select `.
+
+**Steps**:
+1. [ ] - `p1` - Read `dbt_select` parameter (default `source:*`) - `inst-read-select`
+2. [ ] - `p1` - Query ClickHouse: `SELECT name FROM system.databases WHERE name LIKE 'bronze_%'` - `inst-query-databases`
+3. [ ] - `p1` - **IF** caller supplied a more specific selector than `source:*`, pass it through verbatim - `inst-passthrough-specific`
+4. [ ] - `p1` - **ELSE** narrow `source:*` to `source: source: ...` for each existing bronze database - `inst-narrow-list`
+5. [ ] - `p1` - **RETURN** the joined selector - `inst-return-selector`
+
+### Classify Source Status (PASS / WARN / ERROR / RUNTIME ERROR)
+
+- [ ] `p1` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-classify`
+
+**Input**: `target/sources.json` results array.
+
+**Output**: list of `breaches` records `{source, status, max_loaded_at,
+age_hours, empty}`; workflow exit code (0 / 1 / 2).
+
+**Steps**:
+1. [ ] - `p1` - **IF** `target/sources.json` is missing → exit 2 (workflow template line 323) - `inst-missing-report`
+2. [ ] - `p1` - **FOR EACH** result in `results[]` - `inst-iter-results`
+ 1. [ ] - `p1` - Read `unique_id`, `status`, `max_loaded_at`, `max_loaded_at_time_ago_in_s` - `inst-read-fields`
+ 2. [ ] - `p1` - **IF** `max_loaded_at.startswith("1970-01-01")` → mark `empty=true`, set `max_loaded_at="(table is empty)"`, `age_hours=null` (lines 348–360) - `inst-empty-sentinel`
+ 3. [ ] - `p1` - **ELSE** convert `max_loaded_at_time_ago_in_s` to hours - `inst-compute-age`
+ 4. [ ] - `p1` - Append `{source, status, max_loaded_at, age_hours, empty}` to `breaches[]` if status != `pass` - `inst-append-breach`
+3. [ ] - `p1` - **IF** any `breaches[i].status ∈ {error, runtime error}` → exit 1 (page-worthy) - `inst-decide-page`
+4. [ ] - `p1` - **ELSE** exit 0 (line 559) - `inst-decide-clean`
+
+### Render Notification Body per Driver
+
+- [ ] `p1` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-render`
+
+**Input**: canonical breach payload `{topic, cluster, tenant, summary,
+breaches[]}`, active `driver`.
+
+**Output**: driver-shaped HTTP body or SMTP message.
+
+**Steps**:
+1. [ ] - `p1` - Build summary prefix: include `cluster=…` and `tenant=…` only when non-empty (workflow template lines 393–397) - `inst-build-prefix`
+2. [ ] - `p1` - **MATCH** driver - `inst-match-driver`
+ 1. [ ] - `p1` - `webhook` → JSON body verbatim, `Content-Type: application/json` - `inst-render-webhook`
+ 2. [ ] - `p1` - `zulip` → rewrite path `/external/json` → `/external/slack`; `application/x-www-form-urlencoded` with Slack-compat fields; Slack mrkdwn (lines 445–478) - `inst-render-zulip`
+ 3. [ ] - `p1` - `slack` → JSON `{text, channel?}`; GitHub-style mrkdwn (lines 480–487) - `inst-render-slack`
+ 4. [ ] - `p1` - `teams` → MessageCard JSON; `themeColor` red if any page-worthy, orange otherwise (lines 489–500) - `inst-render-teams`
+ 5. [ ] - `p1` - `email` → text/plain body; `Subject: ` (lines 502–531) - `inst-render-email`
+3. [ ] - `p1` - **RETURN** rendered body / message - `inst-return-body`
+
+### Detect Re-Emit Trap
+
+- [ ] `p2` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-trap`
+
+**Input**: bronze table rows (sampled), optional
+`meta.bronze_business_date_col` SQL expression.
+
+**Output**: list of trap suspects `{source, table, kind, evidence}`.
+
+**Steps**:
+1. [ ] - `p2` - **IF** `meta.bronze_freshness_trap_check == "skip"` → return [] for this source/table - `inst-honour-skip`
+2. [ ] - `p2` - **MODE 1** (heuristic, no config) - `inst-mode-fullreemit`
+ 1. [ ] - `p2` - Compute pct of rows with `_airbyte_extracted_at` in last 30 h - `inst-pct-recent`
+ 2. [ ] - `p2` - Compute distinct count of `toDate(_airbyte_extracted_at)` - `inst-distinct-days`
+ 3. [ ] - `p2` - **IF** pct ≥ 95 % AND distinct days ≤ 2 AND row count ≥ 100 → flag `kind=full-reemit` (script lines 188–200) - `inst-flag-fullreemit`
+3. [ ] - `p2` - **MODE 2** (opt-in) - `inst-mode-incremental-topup`
+ 1. [ ] - `p2` - **IF** source declares `meta.bronze_business_date_col` → compute `MAX()` - `inst-compute-bdate`
+ 2. [ ] - `p2` - Compute `MAX(_airbyte_extracted_at)` - `inst-compute-extracted`
+ 3. [ ] - `p2` - **IF** gap ≥ 24 h → flag `kind=incremental-topup` (lines 202–227) - `inst-flag-incremental`
+4. [ ] - `p2` - **RETURN** suspects (advisory only — does not change parser exit code) - `inst-return-suspects`
+
+### Choose SLA Tier per Source
+
+- [ ] `p2` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-tier`
+
+**Input**: connector cadence (streaming / event / report /
+report-extended), vendor-documented publish-lag evidence.
+
+**Output**: per-source `freshness:` block in `schema.yml` referencing
+the chosen tier's env vars.
+
+**Steps**:
+1. [ ] - `p2` - **IF** connector is streaming with daily cron and no natural quiet days → `default` (30/48 h) - `inst-tier-default`
+2. [ ] - `p2` - **ELSE IF** connector has natural quiet days (Confluence edits, Zoom meetings) → `event` (72/96 h) - `inst-tier-event`
+3. [ ] - `p2` - **ELSE IF** vendor documents 24–48 h publish lag (Microsoft Graph reports baseline) → `report` (48/96 h) - `inst-tier-report`
+4. [ ] - `p2` - **ELSE IF** vendor publishes with ~3-day baseline lag (Slack admin.analytics typical) → `report_extended` (72/120 h) - `inst-tier-report-extended`
+5. [ ] - `p2` - Encode the choice as a per-source `freshness:` block referencing the tier's env vars (`FRESHNESS_WARN_*_H` / `FRESHNESS_ERROR_*_H`) - `inst-encode-tier`
+
+## 4. States (CDSL)
+
+### Source Freshness Status
+
+- [ ] `p1` - **ID**: `cpt-insightspec-state-bronze-freshness-sla-source-status`
+
+A bronze source transitions between four classification states on each
+daily run. The state is recomputed from the anchor every time — there is
+no persistence between runs.
+
+| From | Event | To |
+|---|---|---|
+| (any) | `MAX()` within warn-after | `pass` |
+| `pass` | anchor ages past warn-after, before error-after | `warn` |
+| `warn` | anchor ages past error-after | `error` |
+| (any) | freshness query itself fails | `runtime error` |
+| `error` / `warn` | new rows land within warn-after | `pass` |
+| `pass` (zero rows ever) | empty-table sentinel detected | `pass` (with `empty=true` flag in payload) |
+
+`pass` does not page; `warn` shows up in the payload but is not
+page-worthy; `error` and `runtime error` page on the daily run.
+
+### Trap Suspect Lifecycle
+
+- [ ] `p2` - **ID**: `cpt-insightspec-state-bronze-freshness-sla-trap-suspect`
+
+A trap suspect has a single advisory-only state per run — it is logged
+but never persisted.
+
+| From | Event | To |
+|---|---|---|
+| (none) | mode-1 heuristic threshold crossed | `flagged-fullreemit` (advisory) |
+| (none) | mode-2 business-date divergence detected | `flagged-incremental-topup` (advisory) |
+| `flagged-*` | next run no longer matches | (none) |
+| (any) | `meta.bronze_freshness_trap_check: skip` | `suppressed` (no state machine entry) |
+
+## 5. Definitions of Done
+
+### Per-Source Thresholds
+
+- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-thresholds`
+
+The system **MUST** ensure every `bronze_*` source declared anywhere
+under `src/ingestion/connectors/` is included in `dbt source freshness`
+without per-connector wiring. Each source **MUST** declare its own
+`freshness:` block + `loaded_at_field` literally in its `schema.yml`
+(the `dbt_coverage.py` gate counts per-source declarations); the
+CronWorkflow runs `--select source:*` and needs no project default.
+
+**Implements**:
+- `cpt-insightspec-flow-bronze-freshness-sla-wire`
+- `cpt-insightspec-algo-bronze-freshness-sla-tier`
+
+**Covers (PRD)**:
+- `cpt-insightspec-fr-mon-thresholds-sot`
+- `cpt-insightspec-fr-mon-four-tiers`
+
+**Touches**:
+- `src/ingestion/connectors/*/*/dbt/schema.yml`
+
+### Per-Table Opt-Out
+
+- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-optout`
+
+`freshness: null` on a table **MUST** exclude it from the breach count
+without removing the source itself from the report. Every opt-out
+**MUST** carry a one-line rationale comment next to it, and is reserved
+for *incremental* streams that legitimately go quiet for days (a
+full-refresh roster/lookup keeps `_airbyte_extracted_at` as a
+sync-liveness signal instead).
+
+**Implements**:
+- `cpt-insightspec-flow-bronze-freshness-sla-wire`
+
+**Covers (PRD)**:
+- `cpt-insightspec-fr-mon-optout-pertable`
+- `cpt-insightspec-fr-mon-optout-rationale`
+
+**Touches**:
+- `src/ingestion/connectors/*/*/dbt/schema.yml`
+
+### Workflow Execution
+
+- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-workflow`
+
+The daily run **MUST** complete within `activeDeadlineSeconds: 1200` for
+the current connector set (~20 sources, ~80 tables; observed end-to-end
+~2 min on a warm CH). A stale-data fixture **MUST** produce parser
+exit code 1 with a log entry naming the source, max anchor, and lag in
+hours. A clean run **MUST** produce parser exit code 0 and a one-line
+"all sources within SLA" log.
+
+**Implements**:
+- `cpt-insightspec-flow-bronze-freshness-sla-trigger`
+- `cpt-insightspec-algo-bronze-freshness-sla-resolve-selector`
+- `cpt-insightspec-algo-bronze-freshness-sla-classify`
+
+**Covers (PRD)**:
+- `cpt-insightspec-fr-mon-daily-cronworkflow`
+- `cpt-insightspec-fr-mon-selector-narrowing`
+- `cpt-insightspec-fr-mon-stale-report-cleanup`
+- `cpt-insightspec-fr-mon-exit-rederivation`
+- `cpt-insightspec-nfr-mon-activation-deadline`
+- `cpt-insightspec-nfr-mon-exit-codes`
+
+**Touches**:
+- `charts/insight/templates/ingestion/dbt-source-freshness.yaml`
+
+### Notification Delivery
+
+- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-notification`
+
+When `notification.driver` is `""`, the workflow **MUST** succeed (or
+fail on `error`) without attempting a fan-out. When set, the rendered
+payload **MUST** be POSTed (or sent via SMTP). Delivery failures
+**MUST** be logged but **MUST NOT** change the workflow's primary exit
+code.
+
+**Implements**:
+- `cpt-insightspec-flow-bronze-freshness-sla-switch-driver`
+- `cpt-insightspec-algo-bronze-freshness-sla-render`
+
+**Covers (PRD)**:
+- `cpt-insightspec-fr-mon-driver-selection`
+- `cpt-insightspec-fr-mon-credential-isolation`
+- `cpt-insightspec-fr-mon-dispatch-impl`
+- `cpt-insightspec-fr-mon-delivery-failure-isolation`
+- `cpt-insightspec-fr-mon-identity-helm`
+- `cpt-insightspec-fr-mon-identity-summary`
+- `cpt-insightspec-fr-mon-identity-payload`
+
+**Touches**:
+- `charts/insight/templates/ingestion/dbt-source-freshness.yaml`
+- `charts/insight/values.yaml`
+- `charts/insight/values.schema.json`
+
+### Trap Detection
+
+- [ ] `p2` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-trap`
+
+The trap detector **MUST** run after the parser, support both heuristic
+full-reemit detection and opt-in business-date divergence, honour
+`meta.bronze_freshness_trap_check: skip`, and never override the
+parser's exit code.
+
+**Implements**:
+- `cpt-insightspec-algo-bronze-freshness-sla-trap`
+
+**Covers (PRD)**:
+- `cpt-insightspec-fr-mon-trap-post-parser`
+- `cpt-insightspec-fr-mon-trap-two-modes`
+- `cpt-insightspec-fr-mon-trap-skip-annotation`
+- `cpt-insightspec-fr-mon-trap-advisory`
+- `cpt-insightspec-nfr-mon-trap-advisory-mode`
+
+**Touches**:
+- `src/ingestion/scripts/freshness-trap-detect.py`
+- `charts/insight/templates/ingestion/dbt-source-freshness.yaml`
+
+### Local Verification
+
+- [ ] `p2` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-local-verify`
+
+`dbt source freshness --select 'source:bronze_*'` **MUST** run from the
+`insight-toolbox` image against `clickhouse-local` (HTTP `:8123`,
+`host.docker.internal`) using the workspace dbt profile, with no
+per-source environment-specific overrides.
+
+**Touches**:
+- `src/ingestion/dbt/profiles.yml`
+- `src/ingestion/MONITORING.md`
+
+## 6. Acceptance Criteria
+
+- [ ] Every bronze source under `src/ingestion/connectors/*/*/dbt/schema.yml` declares a reachable `loaded_at_field` (source / table / opt-out).
+- [ ] `dbt source freshness --select 'source:bronze_*'` from a clean checkout produces a non-empty `target/sources.json` with one result per declared (source, table) pair, no `runtime error` due to missing `loaded_at_field`.
+- [ ] CronWorkflow renders cleanly when `templates.enabled=true` and `toolboxImage` is pinned (no `:latest` defaults).
+- [ ] Stale-data fixture (truncate or backdate `_airbyte_extracted_at` on a single source) produces exit code 1 and the source appears in the log + payload.
+- [ ] On-call matrix and webhook payload contract documented in [`MONITORING.md`](../../../../../src/ingestion/MONITORING.md).
+- [ ] Synthetic re-emit fixture is flagged by the trap detector with `kind=full-reemit`; opt-in business-date divergence fixture is flagged with `kind=incremental-topup`.
+- [ ] Switching `notification.driver` in a Helm overlay re-points dispatch to the new channel after `helm upgrade`, with no Secret value visible in rendered manifests.
diff --git a/src/ingestion/MONITORING.md b/src/ingestion/MONITORING.md
new file mode 100644
index 000000000..4467774e2
--- /dev/null
+++ b/src/ingestion/MONITORING.md
@@ -0,0 +1,315 @@
+# Ingestion Monitoring — operator runbook
+
+Goal: detect "data is missing for several days" before users notice. Today's
+coverage is **freshness only** — newly arrived rows in bronze tables. Volume
+and Airbyte-job-level signals are listed under [Open work](#open-work).
+
+This file is the operator-facing runbook (verification steps, on-call
+matrix, parser exit codes, payload shape). The feature design itself —
+purpose, threshold inheritance, acceptance criteria — lives in
+[`docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md`](../../docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md).
+The companion PRD and DESIGN for the broader monitoring domain live next
+to it: [`PRD.md`](../../docs/domain/ingestion-monitoring/specs/PRD.md),
+[`DESIGN.md`](../../docs/domain/ingestion-monitoring/specs/DESIGN.md).
+
+## What's wired
+
+### Bronze freshness (live)
+
+Every bronze source declares its own `freshness:` block + `loaded_at_field`
+**per source in the connector's `schema.yml`** (not inherited from a project
+default — `dbt_coverage.py` counts per-source declarations, so each source
+declares explicitly). The declaration half is tracked under EPIC #1321 / #1322
+(PR #1346 + the per-connector anchoring in PR SharedQA/insight#1); this
+CronWorkflow only *runs* the check and acts on the verdict.
+
+The critical choice is **which column** `loaded_at_field` points at, and it
+depends on whether the connector is **incremental** or **windowed**:
+
+- **Incremental** (git, jira, youtrack — rows land as events happen) →
+ `loaded_at_field: _airbyte_extracted_at`. The technical extract timestamp
+ tracks reality because the cursor only advances when new data arrives.
+- **Windowed / vendor-analytics** (M365 Graph, ChatGPT/Claude Team, Slack
+ analytics, Cursor daily, OpenAI usage — the API re-emits a fixed reporting
+ window every sync) → anchor on the **business-date column**
+ (`parseDateTimeBestEffortOrNull(
)`). Here `_airbyte_extracted_at` is
+ re-stamped every run and would be **false-green**: verified on live data,
+ M365 business data was 4.5 days stale while `_airbyte_extracted_at` was 10.6h
+ fresh. See the "Trap to avoid" item in the new-connector checklist below.
+
+Examples of the two forms:
+
+```yaml
+# Streaming connector — Airbyte cursor follows business time. Rows land in
+# bronze approximately when they happen, so the technical extracted-at
+# timestamp tracks reality.
+sources:
+ - name: bronze_
+ schema: bronze_
+ loaded_at_field: _airbyte_extracted_at
+ tables: ...
+```
+
+```yaml
+# Report-style connector — Airbyte re-fetches a fixed window every run
+# (e.g. Microsoft Graph reports, Slack admin.analytics.getFile). Even when
+# the upstream has not advanced, the sync writes "fresh" rows for older
+# business days, so `_airbyte_extracted_at` stays green forever. Anchor on
+# the report's own business-day column instead. ISO-8601 strings sort
+# lexically, so wrapping in `parseDateTimeBestEffortOrNull(...)` works
+# directly inside `loaded_at_field`.
+sources:
+ - name: bronze_
+ schema: bronze_
+ loaded_at_field: parseDateTimeBestEffortOrNull(reportRefreshDate)
+ tables: ...
+```
+
+Active per-source assignments. The `warn_after`/`error_after` thresholds are
+**literal values in each source's `schema.yml`** (not Helm/env-var driven), so
+there is a single source of truth and the local `dbt source freshness` reads
+exactly what ships. Tiers used: **default** 36h/72h · **report** 72h/120h ·
+**report_extended** 120h/168h · **event** 96h/168h.
+
+| Source / table | `loaded_at_field` | Tier | Confidence |
+|---|---|---|---|
+| bronze_m365.*_activity | `parseDateTimeBestEffortOrNull(reportRefreshDate)` | report | **verified** (live) |
+| bronze_chatgpt_team.{chat_activity,codex_user_daily} | `parseDateTimeBestEffortOrNull(date)` | report | **verified** |
+| bronze_chatgpt_team.{subscription_usage,subscription_balance} | `parseDateTimeBestEffortOrNull(snapshot_date)` | report | inferred |
+| bronze_claude_team.code_metrics | `parseDateTimeBestEffortOrNull(metric_date)` | report | **verified** |
+| bronze_cursor.cursor_daily_usage | `parseDateTimeBestEffortOrNull(day)` | report | inferred (daily resync re-fetch) |
+| bronze_slack.users_details | `parseDateTimeBestEffortOrNull(date)` | report_extended | inferred (3–5d Slack lag) |
+| bronze_zoom.{meetings,participants} | `parseDateTimeBestEffortOrNull(end_time/join_time)` | event | inferred (30d window; quiet weekends real) |
+| bronze_openai.usage_*/costs | `parseDateTimeBestEffortOrNull(bucket_start_time)` | report | inferred |
+| bronze_claude_admin.{messages_usage,cost_report,code_usage} | `parseDateTimeBestEffortOrNull(date)` | report | inferred |
+| bronze_claude_enterprise.summaries | `parseDateTimeBestEffortOrNull(date)` | report | inferred |
+| bronze_github_copilot.{user,org}_metrics | `parseDateTimeBestEffortOrNull(day)` | report | inferred |
+| bronze_confluence.wiki_pages / bronze_outline.wiki_pages | `parseDateTimeBestEffortOrNull(updated_at)` | event | inferred |
+| git / jira / youtrack / figma / salesforce / hubspot / zulip_proxy | `_airbyte_extracted_at` | default | incremental — extracted_at tracks reality |
+| rosters & lookups (bamboohr, ms_entra, workday, *_members, *_seats, *_users, *_statuses…) | `_airbyte_extracted_at` | default | full-refresh → sync-liveness signal |
+
+"verified" rows are backed by measured `ext_age` vs `biz_age` on live data;
+"inferred" rows are anchored on the connector's cursor column (verified to
+exist) but their windowed behavior is not yet confirmed against data — they are
+marked inline in each `schema.yml` with `confirm once ingested`.
+
+Re-categorizing a connector across tiers is an engineering change (it usually
+comes with a `loaded_at_field` revisit), not an ops dial — that's why the
+mapping lives in connector `schema.yml`, literal per source.
+
+`loaded_at_field` is a dbt **property**, not a config — `+loaded_at_field`
+at project level is silently ignored. The dbt-clickhouse adapter does not
+support metadata-based freshness, so a source missing `loaded_at_field`
+fails with `runtime error` instead of falling back to a default.
+
+A single CronWorkflow `dbt-source-freshness-check` runs `dbt source freshness`
+daily at 13:00 UTC (after every connector's sync window of 02:00–11:00 UTC)
+and parses `target/sources.json`. Any source in `warn` or `error` is logged
+with name, max-loaded-at and lag; if a notification driver is configured
+under `ingestion.freshness.notification.driver` (one of `webhook`, `zulip`,
+`slack`, `teams`, `email`), the breach list is dispatched through the
+matching driver. Driver shapes and credential bindings are documented in
+[`docs/domain/ingestion-monitoring/specs/DESIGN.md` §3.3](../../docs/domain/ingestion-monitoring/specs/DESIGN.md#33-api-contracts).
+
+| Status | Meaning | Workflow exit | What to do |
+|--------|---------|---------------|------------|
+| `pass` | MAX(``) within the source's tier `warn_after` window | 0 | Nothing |
+| `warn` | between `warn_after` and `error_after` (one missed run for daily-cadence sources) | 0 (visible in log + payload) | Investigate during business hours |
+| `error` | past `error_after` for the source's tier | 1 (Argo Failed) | Page |
+| `runtime error` | dbt couldn't even check the source (CH down, schema drift, query failure) | 1 (Argo Failed) | Page — investigate before trusting other sources |
+
+Concrete tier values are the literal `warn_after`/`error_after` in the source's
+`schema.yml`. Tiers: `default` 36h/72h, `report` 72h/120h, `report_extended`
+120h/168h, `event` 96h/168h.
+
+`error` and `runtime error` flip the workflow to Failed so Argo retains the
+run in `failedJobsHistoryLimit`. Warn-only runs stay Successful — the breach
+is still printed to the workflow log and POSTed in the notification payload,
+but on-call doesn't get paged on a single missed sync.
+
+### How it stays generic
+
+The CronWorkflow is connector-agnostic: it runs `dbt source freshness --select
+source:*` and acts on whatever every `bronze_*` source declares. A new
+connector is covered the moment its `schema.yml` carries a `freshness:` block —
+no per-connector pipeline plumbing. The declaration (which column, which tier)
+is the connector author's call, captured per source in `schema.yml`.
+
+## Adding a new connector — freshness checklist
+
+1. In the new `connectors///dbt/schema.yml`, declare the
+ `bronze_*` source with a `freshness:` block + `loaded_at_field`.
+2. Pick the `loaded_at_field` by sync shape:
+ - **Incremental / event-cursor** (commits, issues, edits) —
+ `loaded_at_field: _airbyte_extracted_at`, `default` tier (36h/72h). The
+ extract timestamp tracks reality because the cursor only advances on new data.
+ - **Windowed / vendor-analytics** (re-fetches a fixed reporting window every
+ run — Graph reports, Slack/Cursor/OpenAI daily) —
+ `loaded_at_field: parseDateTimeBestEffortOrNull()` with the
+ `report` (or `report_extended` for 3–5d lag) tier. **Do not** leave it on
+ `_airbyte_extracted_at` — it re-stamps every sync and goes false-green.
+ - **Event-style with quiet weekends** (Confluence/Zoom — a zero-row Saturday is
+ normal) — anchor on the event timestamp with the `event` tier (96h/168h):
+
+ ```yaml
+ freshness:
+ warn_after: { count: 96, period: hour }
+ error_after: { count: 168, period: hour }
+ loaded_at_field: parseDateTimeBestEffortOrNull()
+ ```
+
+3. Confirm the windowed-vs-incremental call against live data when the connector
+ has rows: compare `now() - max(_airbyte_extracted_at)` against
+ `now() - max()`. A large gap = windowed (anchor on the business
+ date); roughly equal = incremental (`_airbyte_extracted_at` is fine).
+4. Roster/lookup tables (full-refresh, rarely-changing) keep `_airbyte_extracted_at`
+ on the `default` tier — the re-stamped extract time is a useful "sync alive"
+ signal. Use a per-table `freshness: null` opt-out only for streams that are
+ *incremental* and legitimately go quiet for days (where `_airbyte_extracted_at`
+ would false-alarm).
+5. Coverage is measured by the QA-owned `dbt_coverage.py` gate (EPIC #1321),
+ which counts per-source declarations — which is why each source declares
+ explicitly rather than inheriting a project default.
+
+6. **Trap to avoid**: if your connector re-emits a fixed window every run
+ (`SELECT count(), max(_airbyte_extracted_at) - min(_airbyte_extracted_at)
+ FROM bronze_.
` shows all rows extracted within the last 24h
+ even though the table covers many days of history), `_airbyte_extracted_at`
+ will look fresh forever (the false-green failure mode). Anchor on the
+ business-date column instead. This is a judgment call about the source
+ shape — confirm it against live data and record it in the assignment table
+ above.
+
+## Who consumes the signal
+
+Per-environment ownership matrix until the delivery channel is wired (see
+[Open work](#open-work)).
+
+Ownership for "ingestion on-call" and "connector owner" is not assigned yet
+(no rotation document, no `CODEOWNERS` for `src/ingestion/connectors/*` as
+of this commit). The matrix below describes the *roles* the freshness
+signal expects to land on; see [Open work](#open-work) for the rotation gap.
+
+| Role | What they read | When | Action |
+|---|---|---|---|
+| Ingestion on-call (TBD) | Argo UI / `kubectl get workflows -n argo --sort-by=.metadata.creationTimestamp` for the `dbt-source-freshness-check` runs | Daily, after the 13:00 UTC run | Triage `error` / `runtime error` runs |
+| `constructorfabric/insight` repo Issues | One issue per persistent breach (>2 consecutive runs) opened by the on-call | Within 1 business day of the breach | Hand off to the connector owner |
+| Connector owner (TBD per connector) | The issue body — includes the failing source, max-loaded-at, lag in hours | On issue assignment | Fix the connector or update the SLA |
+| Tenant on-call (post-MVP) | Webhook payload (Zulip / email / generic POST) routed by `cluster` field | Real-time | Same triage as above, scoped to one deployment |
+
+Until the webhook channel lands, the **only** push mechanism is Argo's
+failed-runs list — on-call must check `kubectl get workflows -n argo
+--sort-by=.metadata.creationTimestamp` at least once per business day. The
+`failedJobsHistoryLimit: 5` ensures the latest five breaching runs are
+retained for inspection.
+
+`pass`-only runs leave no trace beyond Argo's success history (kept by
+`successfulJobsHistoryLimit: 3`) — silent green is the desired steady state.
+
+## Open work
+
+### Rotation / ownership — not assigned
+
+The matrix above describes roles, not people. There is no documented
+ingestion on-call rotation as of this commit, and `src/ingestion/connectors/`
+has no `CODEOWNERS` entries assigning per-connector owners. Until that
+lands, the freshness signal lives in Argo's failed-runs list with no
+named consumer.
+
+Action: agree on an on-call rotation (or a single owner during MVP) and
+add a `CODEOWNERS` block listing the per-connector owner so the breach
+hand-off has a real target.
+
+### Delivery channel — driver-based
+
+`charts/insight/values.yaml` exposes notification routing under
+`ingestion.freshness.notification.*` (driver selector + per-driver
+subblock). When `driver: ""` (the default), breaches surface only via:
+
+- Argo UI / `kubectl get workflows -n argo -l app.kubernetes.io/component=ingestion-monitoring`
+- Workflow exit status (failed runs accumulate in
+ `failedJobsHistoryLimit: 5`)
+
+Setting `driver` to one of `webhook`, `zulip`, `slack`, `teams`, `email`
+activates the matching driver's dispatcher; the URL or SMTP password is
+read from a Kubernetes Secret on a `secretKeyRef` env binding so the
+credential never lands in rendered manifests or Argo UI. Driver shapes,
+secret bindings, and observed quirks (Zulip endpoint rewrite, Cloudflare
+UA, Slack mrkdwn vs Zulip native) are documented in
+[`docs/domain/ingestion-monitoring/specs/DESIGN.md` §3.3](../../docs/domain/ingestion-monitoring/specs/DESIGN.md#33-api-contracts).
+
+The webhook driver's body remains provider-agnostic so a custom relay
+can sit in front of any of the above:
+
+```json
+{
+ "topic": "ingestion-freshness",
+ "cluster": "prod",
+ "tenant": "acme-co",
+ "summary": "[cluster=prod, tenant=acme-co] 3 bronze source(s) breaching freshness SLA",
+ "breaches": [
+ {
+ "source": "source.ingestion.bronze_jira.jira_issue",
+ "status": "error",
+ "max_loaded_at": "2026-04-28T03:14:21Z",
+ "age_hours": 51.2,
+ "empty": false
+ }
+ ]
+}
+```
+
+`cluster` and `tenant` come from `ingestion.freshness.{cluster,tenant}`
+in values overrides — both empty by default; the `summary` drops empty
+labels from its prefix.
+
+### Volume baseline (next iteration)
+
+Freshness catches "no rows arrived" but misses "API returned 50 rows instead
+of 5000". Plan: a singular SQL test (or dbt operation) that compares today's
+row count per stream against a 14-day median, alerts on <30%. Reuses the same
+`dbt-source-freshness` workflow shell — just a different selector.
+
+### Source vs bronze attribution
+
+Today the freshness check flags "no rows in bronze" but cannot tell whether
+the upstream Airbyte sync ran. To distinguish:
+
+- *source/credential issue*: Airbyte sync ✅, bronze pulled 0 rows
+- *pipeline issue*: Airbyte sync ❌ (didn't run / errored)
+
+We'd need a sidecar that polls the Airbyte Jobs API after each sync and
+appends a row to `staging.airbyte_runs (connection_id, status,
+records_emitted, started_at, ended_at)`. Then the freshness step can JOIN
+against that table and label each breach with a root cause. Future PR.
+
+## How to verify locally
+
+```bash
+# After dev-up.sh + at least one successful sync
+kubectl create -n argo -f - <<'EOF'
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ generateName: freshness-adhoc-
+ namespace: argo
+spec:
+ workflowTemplateRef:
+ name: dbt-source-freshness
+ arguments:
+ parameters:
+ - name: dbt_select
+ value: "source:bronze_jira"
+ - name: toolbox_image
+ value: "insight-toolbox:local"
+ - name: clickhouse_host
+ value: "insight-clickhouse.insight.svc.cluster.local"
+ - name: clickhouse_port
+ value: "8123"
+ - name: clickhouse_user
+ value: "default"
+EOF
+
+# Watch the run
+kubectl logs -n argo -l workflows.argoproj.io/workflow= -f
+```
diff --git a/src/ingestion/scripts/freshness-trap-detect.py b/src/ingestion/scripts/freshness-trap-detect.py
new file mode 100755
index 000000000..ce3fab882
--- /dev/null
+++ b/src/ingestion/scripts/freshness-trap-detect.py
@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+"""Detect bronze sources whose `_airbyte_extracted_at` anchor is masking a
+trap that the dbt source freshness check cannot see — every nightly sync
+makes the anchor look fresh even when the upstream has stopped publishing.
+
+Two detection modes:
+
+1. Full re-emit (heuristic, no per-source config required).
+ Fires when ≥ 95% of rows have `_airbyte_extracted_at` within the last
+ 30h *and* `_airbyte_extracted_at` covers ≤ 2 distinct calendar days
+ *and* the table has ≥ 100 rows. Catches connectors that overwrite the
+ entire bronze table every run (Confluence-style).
+
+2. Business-date divergence (explicit opt-in via `meta`).
+ When a source declares
+ meta:
+ bronze_business_date_col:
+ the script compares `MAX()` against `MAX(_airbyte_extracted_at)`
+ and warns when the business date lags by ≥ 24h while extracted-at is
+ fresh. Catches connectors that incrementally top up the latest day's
+ row (M365 / Slack / Cursor-style) — the heuristic above can't see these
+ because most rows really are old, just `MAX(extract)` happens to be
+ today's because of the new top-up row.
+
+A source can opt out of mode 1 (e.g. genuine append-only full-refresh
+rosters where neither the heuristic nor a business-date anchor applies)
+via `meta: { bronze_freshness_trap_check: skip }` at source or table
+level.
+
+Connection params: `CLICKHOUSE_{HOST,PORT,USER,PASSWORD}` env vars (same
+as the freshness CronWorkflow). Defaults match the local toolbox setup.
+
+Exit codes:
+ 0 — no trap suspects
+ 1 — at least one suspect found (warn, not page)
+ 2 — script failure (schema parse, etc.)
+"""
+from __future__ import annotations
+
+import os
+import sys
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+try:
+ import yaml
+except ImportError:
+ print("PyYAML required: pip install pyyaml", file=sys.stderr)
+ sys.exit(2)
+
+EXTRACTED_AT_LITERAL = "_airbyte_extracted_at"
+RECENT_WINDOW_HOURS = 30
+SUSPECT_PCT_RECENT = 95.0
+SUSPECT_MAX_DISTINCT_DAYS = 2
+MIN_ROWS = 100
+
+
+class CHQueryError(Exception):
+ """Raised when ClickHouse rejects a query (4xx / 5xx). Distinguished from
+ transport errors so the caller can decide whether to silently skip
+ (database missing in fixture) or escalate."""
+
+
+CH_BASE_URL: str | None = None
+
+
+def _resolve_ch_base_url() -> str:
+ """Resolve `http://host:port/?user=…&password=…` once at startup.
+
+ Defaults match the local toolbox so the script is runnable from a
+ developer shell without exporting anything; in CI / Argo every var is
+ set explicitly. Missing vars fall back to defaults — the script logs
+ which params came from defaults so a half-configured prod env is
+ obvious in workflow logs."""
+ host_default, port_default = "host.docker.internal", "8123"
+ user_default, password_default = "default", "clickhouse"
+ host = os.environ.get("CLICKHOUSE_HOST") or host_default
+ port = os.environ.get("CLICKHOUSE_PORT") or port_default
+ user = os.environ.get("CLICKHOUSE_USER") or user_default
+ password = os.environ.get("CLICKHOUSE_PASSWORD") or password_default
+ using_defaults = [
+ name for name, used_default in [
+ ("CLICKHOUSE_HOST", host == host_default and not os.environ.get("CLICKHOUSE_HOST")),
+ ("CLICKHOUSE_PORT", port == port_default and not os.environ.get("CLICKHOUSE_PORT")),
+ ("CLICKHOUSE_USER", user == user_default and not os.environ.get("CLICKHOUSE_USER")),
+ ("CLICKHOUSE_PASSWORD", password == password_default and not os.environ.get("CLICKHOUSE_PASSWORD")),
+ ] if used_default
+ ]
+ if using_defaults:
+ print(
+ f"[freshness-trap-detect] using local defaults for: {', '.join(using_defaults)}",
+ file=sys.stderr,
+ )
+ creds = urllib.parse.urlencode({"user": user, "password": password})
+ return f"http://{host}:{port}/?{creds}"
+
+
+def ch_query(sql: str) -> str:
+ global CH_BASE_URL
+ if CH_BASE_URL is None:
+ CH_BASE_URL = _resolve_ch_base_url()
+ url = f"{CH_BASE_URL}&{urllib.parse.urlencode({'query': sql})}"
+ try:
+ with urllib.request.urlopen(url, timeout=30) as resp:
+ return resp.read().decode("utf-8").strip()
+ except urllib.error.HTTPError as e:
+ body = e.read().decode("utf-8", errors="replace")
+ raise CHQueryError(f"HTTP {e.code}: {body[:200]}") from None
+ except urllib.error.URLError as e:
+ # DNS / refused / TLS / socket-level failures land here. Treat as
+ # transport errors so the caller skips the table without crashing
+ # the whole script.
+ raise CHQueryError(f"URLError: {e.reason}") from None
+ except (TimeoutError, OSError) as e:
+ # `socket.timeout` was aliased to `TimeoutError` in Py 3.10+;
+ # `OSError` covers connection-reset / broken-pipe at the socket
+ # layer that bypass the URLError translation.
+ raise CHQueryError(f"transport: {e}") from None
+
+
+def candidate_tables(connectors_root: Path) -> list[dict]:
+ """Return one record per bronze table that should be checked.
+
+ Each record has: database, table, source_meta_business_date (str|None),
+ table_meta_business_date (str|None), skip_heuristic (bool).
+
+ A table is included if its effective `loaded_at_field` is
+ `_airbyte_extracted_at` (no business-date anchor was deliberately
+ chosen). Tables opted out via `meta.bronze_freshness_trap_check: skip`
+ are excluded entirely. Tables that declare
+ `meta.bronze_business_date_col` get the divergence check even when
+ they would otherwise be skipped by the heuristic.
+ """
+ out: list[dict] = []
+ for schema_file in sorted(connectors_root.glob("**/dbt/schema.yml")):
+ try:
+ doc = yaml.safe_load(schema_file.read_text())
+ except yaml.YAMLError as e:
+ print(f"[freshness-trap-detect] skipping malformed yaml {schema_file}: {e}", file=sys.stderr)
+ continue
+ if not isinstance(doc, dict):
+ print(f"[freshness-trap-detect] skipping non-mapping yaml root {schema_file}", file=sys.stderr)
+ continue
+ for source in doc.get("sources") or []:
+ name = source.get("name", "")
+ # dbt convention: if a source omits `schema:`, the source `name`
+ # is used as the ClickHouse database. Mirror that here so we
+ # query the same DB dbt does.
+ schema = source.get("schema", name)
+ # Match bronze sources by either dbt name or CH schema (bamboohr
+ # uses `name: bamboohr, schema: bronze_bamboohr`).
+ is_bronze = (
+ isinstance(name, str) and name.startswith("bronze_")
+ ) or (
+ isinstance(schema, str) and schema.startswith("bronze_")
+ )
+ if not is_bronze:
+ continue
+ source_anchor = source.get("loaded_at_field", "").strip()
+ source_meta = source.get("meta") or {}
+ for table in source.get("tables") or []:
+ tname = table.get("name")
+ if not tname:
+ continue
+ if "freshness" in table and table["freshness"] is None:
+ continue
+ anchor = table.get("loaded_at_field", "").strip() or source_anchor
+ if anchor != EXTRACTED_AT_LITERAL:
+ continue
+ meta = table.get("meta") or {}
+ skip = (
+ (meta.get("bronze_freshness_trap_check") or "").lower() == "skip"
+ or (source_meta.get("bronze_freshness_trap_check") or "").lower() == "skip"
+ )
+ business = (
+ meta.get("bronze_business_date_col")
+ or source_meta.get("bronze_business_date_col")
+ )
+ # If the table is opted-out and has no business-date hint,
+ # there is nothing to check.
+ if skip and not business:
+ continue
+ out.append({
+ "database": schema,
+ "table": tname,
+ "skip_heuristic": skip,
+ "business_date_col": business,
+ })
+ return out
+
+
+def check_table(candidate: dict) -> dict | None:
+ """Return a finding dict if the table is a trap suspect, else None."""
+ database = candidate["database"]
+ table = candidate["table"]
+ business = candidate["business_date_col"]
+ skip_heuristic = candidate["skip_heuristic"]
+
+ select_parts = [
+ "count() AS rows",
+ f"countIf(_airbyte_extracted_at >= now() - INTERVAL {RECENT_WINDOW_HOURS} HOUR) AS recent",
+ "uniqExact(toDate(_airbyte_extracted_at)) AS distinct_days",
+ "max(_airbyte_extracted_at) AS last_extract",
+ ]
+ if business:
+ select_parts.append(f"max({business}) AS last_business")
+ sql = (
+ "SELECT " + ", ".join(select_parts)
+ + f" FROM `{database}`.`{table}` "
+ "FORMAT TabSeparated"
+ )
+ try:
+ raw = ch_query(sql)
+ except CHQueryError as e:
+ # Common in fixtures: bronze_ database does not exist locally.
+ # In prod the same response means a connector was renamed / dropped
+ # without a schema.yml update — surface it on stderr so workflow
+ # logs catch the misconfig instead of swallowing it.
+ if "Code: 81" in str(e) or "UNKNOWN_DATABASE" in str(e) or "HTTP 404" in str(e):
+ print(f" [{database}.{table}] missing in ClickHouse (skipping): {e}", file=sys.stderr)
+ return None
+ print(f" [{database}.{table}] query failed: {e}", file=sys.stderr)
+ return None
+ if not raw:
+ return None
+ fields = raw.split("\t")
+ rows = int(fields[0])
+ if rows < MIN_ROWS:
+ return None
+ recent = int(fields[1])
+ distinct_days = int(fields[2])
+ last_extract = fields[3]
+ pct_recent = 100.0 * recent / rows
+ last_business = fields[4] if business and len(fields) >= 5 else None
+
+ finding: dict | None = None
+
+ if not skip_heuristic and (
+ pct_recent >= SUSPECT_PCT_RECENT
+ and distinct_days <= SUSPECT_MAX_DISTINCT_DAYS
+ ):
+ finding = {
+ "kind": "full-reemit",
+ "database": database,
+ "table": table,
+ "rows": rows,
+ "pct_recent": round(pct_recent, 1),
+ "distinct_days": distinct_days,
+ "last_extract": last_extract,
+ }
+
+ if business and last_business and last_business not in ("\\N", "1970-01-01 00:00:00"):
+ # Compare gap. `_airbyte_extracted_at` arrives with milliseconds
+ # (e.g. '2026-05-04 02:01:58.805') which `toDateTime` cannot parse —
+ # use `parseDateTime64BestEffortOrNull` for both sides.
+ diff_sql = (
+ "SELECT dateDiff('hour', "
+ f"parseDateTime64BestEffortOrNull('{last_business}'), "
+ f"parseDateTime64BestEffortOrNull('{last_extract}'))"
+ " FORMAT TabSeparated"
+ )
+ try:
+ diff_h = int(ch_query(diff_sql))
+ except (CHQueryError, ValueError):
+ diff_h = 0
+ if diff_h >= 24:
+ finding = {
+ "kind": "incremental-topup",
+ "database": database,
+ "table": table,
+ "rows": rows,
+ "last_extract": last_extract,
+ "last_business": last_business,
+ "extract_minus_business_h": diff_h,
+ "business_date_col": business,
+ }
+ return finding
+
+
+def main(argv: list[str]) -> int:
+ root = Path(argv[1]) if len(argv) >= 2 else Path("src/ingestion/connectors")
+ if not root.is_dir():
+ print(f"connectors root not found: {root}", file=sys.stderr)
+ return 2
+
+ candidates = candidate_tables(root)
+ if not candidates:
+ print("no candidate sources (every bronze source already uses a non-extracted-at anchor or is opted out)")
+ return 0
+
+ print(f"checking {len(candidates)} bronze table(s) anchored on `_airbyte_extracted_at`...")
+ suspects: list[dict] = []
+ for c in candidates:
+ result = check_table(c)
+ if result:
+ suspects.append(result)
+
+ if not suspects:
+ print("no trap suspects")
+ return 0
+
+ full_reemit = [s for s in suspects if s["kind"] == "full-reemit"]
+ incremental = [s for s in suspects if s["kind"] == "incremental-topup"]
+
+ if full_reemit:
+ print(f"\n{len(full_reemit)} full re-emit suspect(s):")
+ print(
+ f" ≥ {SUSPECT_PCT_RECENT:.0f}% of rows have `_airbyte_extracted_at` within the "
+ f"last {RECENT_WINDOW_HOURS}h, across ≤ {SUSPECT_MAX_DISTINCT_DAYS} distinct day(s). "
+ "The connector is rewriting the entire table on every run — "
+ "`_airbyte_extracted_at` will look fresh forever, even when the "
+ "upstream stops publishing. Switch `loaded_at_field` to a real "
+ "business-date column."
+ )
+ for s in full_reemit:
+ print(
+ f" - {s['database']}.{s['table']}: "
+ f"rows={s['rows']}, pct_recent={s['pct_recent']}%, "
+ f"distinct_extract_days={s['distinct_days']}, "
+ f"last_extract={s['last_extract']}"
+ )
+
+ if incremental:
+ print(f"\n{len(incremental)} incremental top-up suspect(s):")
+ print(
+ " `MAX(_airbyte_extracted_at)` is fresh but the declared "
+ "business-date column is at least 24h behind. The connector is "
+ "writing today's row but the upstream's reported business date "
+ "has not advanced — the freshness check sees a green anchor that "
+ "has nothing to do with reality. Switch `loaded_at_field` to the "
+ "business-date expression you used in `meta`."
+ )
+ for s in incremental:
+ print(
+ f" - {s['database']}.{s['table']}: "
+ f"last_extract={s['last_extract']}, "
+ f"last_business={s['last_business']}, "
+ f"gap={s['extract_minus_business_h']}h "
+ f"(business_date_col={s['business_date_col']})"
+ )
+
+ print(
+ "\nLegitimate full-refresh roster tables (no daily cadence, no "
+ "business-date column) can opt out per-table:\n"
+ " meta:\n"
+ " bronze_freshness_trap_check: skip"
+ )
+ return 1
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))