diff --git a/charts/insight/templates/ingestion/dbt-source-freshness.yaml b/charts/insight/templates/ingestion/dbt-source-freshness.yaml new file mode 100644 index 000000000..a2f2dc9f9 --- /dev/null +++ b/charts/insight/templates/ingestion/dbt-source-freshness.yaml @@ -0,0 +1,632 @@ +{{- if and .Values.ingestion.templates.enabled .Values.ingestion.freshness.enabled }} +{{- /* +Bronze freshness check: runs `dbt source freshness` against ClickHouse, reads +the resulting `target/sources.json`, and emits a notification when any source +is `warn` or `error`. Single workflow covers every connector — new bronze +sources are picked up automatically as soon as they appear in any +`schema.yml` (defaults inherited from `dbt_project.yml`). + +How to reason about it operationally: + - `pass` : MAX(_airbyte_extracted_at) within the warn window. Sync OK. + - `warn` : older than warn_after but younger than error_after. One missed + run; alert lower priority. + - `error` : older than error_after. Multiple missed runs / source dead. + Page on these. + +Two failure modes this catches: + 1. Argo CronWorkflow didn't run / couldn't talk to Airbyte → no new rows. + 2. Sync ran but source API returned 0 rows (token expired, API outage). + +It does NOT distinguish (1) from (2) — for that we need an independent +"last successful Airbyte job" signal (future work, see MONITORING.md). +*/}} +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: dbt-source-freshness + labels: + {{- include "insight.labels" . | nindent 4 }} + app.kubernetes.io/component: ingestion-template +spec: + serviceAccountName: argo-workflow + entrypoint: run + templates: + - name: run + inputs: + parameters: + # Selector applied to `dbt source freshness`. Default covers every + # bronze_* source declared anywhere in the project. Override to + # narrow scope (e.g., per-tenant smoke check). + - name: dbt_select + value: "source:*" + - name: toolbox_image + - name: clickhouse_host + - name: clickhouse_port + - name: clickhouse_user + # Identification labels for routing when one bot fans out to + # multiple deployments. `cluster` = installation tier; + # `tenant` = customer/workspace. Both empty by default. + - name: cluster + value: "" + - name: tenant + value: "" + # Notification delivery. `driver` selects the parser branch + # ("" / webhook / zulip / slack / email / future). Credential + # material (webhook URL, SMTP password) is **not** a workflow + # parameter — it is read from a k8s Secret via `secretKeyRef` + # on the env block below so the token never lands in rendered + # manifests or the Argo UI. Plain per-driver settings + # (channel, stream, smtp host/port, from/to) come through as + # ordinary parameters because they aren't credentials. + - name: notification_driver + value: "" + # zulip driver + - name: notification_zulip_stream + value: "" + - name: notification_zulip_topic + value: "" + # slack driver + - name: notification_slack_channel + value: "" + # email driver + - name: notification_email_smtp_host + value: "" + - name: notification_email_smtp_port + value: "587" + - name: notification_email_smtp_username + value: "" + - name: notification_email_smtp_starttls + value: "true" + - name: notification_email_from + value: "" + - name: notification_email_to + value: "" + - name: notification_email_subject_prefix + value: "[ingestion-freshness]" + # NOTE: SLA thresholds (warn_after / error_after) are NOT passed + # from here. They live as literals on each bronze source's + # `freshness:` block in the connector `schema.yml` files (see + # constructorfabric/insight#1346 / EPIC #1321). This workflow only + # *runs* the check and acts on the verdict in `target/sources.json`; + # it does not own the threshold values. Recommended per-tier + # thresholds (default / event / report) are documented in + # `src/ingestion/MONITORING.md` for whoever sets the declarations. + # 20 min: dbt source freshness issues one CH query per source. With + # the current connector set this finishes in ~2 min; the headroom + # is for connector growth and CH cold-start cases. + activeDeadlineSeconds: 1200 + script: + image: {{ `"{{inputs.parameters.toolbox_image}}"` }} + imagePullPolicy: Always + command: [bash] + env: + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ required "clickhouse.passwordSecret.name is required" .Values.clickhouse.passwordSecret.name | quote }} + key: {{ required "clickhouse.passwordSecret.key is required" .Values.clickhouse.passwordSecret.key | quote }} + - name: DBT_SELECT + value: {{ `"{{inputs.parameters.dbt_select}}"` }} + - name: CLICKHOUSE_HOST + value: {{ `"{{inputs.parameters.clickhouse_host}}"` }} + - name: CLICKHOUSE_PORT + value: {{ `"{{inputs.parameters.clickhouse_port}}"` }} + - name: CLICKHOUSE_USER + value: {{ `"{{inputs.parameters.clickhouse_user}}"` }} + - name: CLUSTER + value: {{ `"{{inputs.parameters.cluster}}"` }} + - name: TENANT + value: {{ `"{{inputs.parameters.tenant}}"` }} + - name: NOTIFICATION_DRIVER + value: {{ `"{{inputs.parameters.notification_driver}}"` }} + - name: NOTIFICATION_ZULIP_STREAM + value: {{ `"{{inputs.parameters.notification_zulip_stream}}"` }} + - name: NOTIFICATION_ZULIP_TOPIC + value: {{ `"{{inputs.parameters.notification_zulip_topic}}"` }} + - name: NOTIFICATION_SLACK_CHANNEL + value: {{ `"{{inputs.parameters.notification_slack_channel}}"` }} + - name: NOTIFICATION_EMAIL_SMTP_HOST + value: {{ `"{{inputs.parameters.notification_email_smtp_host}}"` }} + - name: NOTIFICATION_EMAIL_SMTP_PORT + value: {{ `"{{inputs.parameters.notification_email_smtp_port}}"` }} + - name: NOTIFICATION_EMAIL_SMTP_USERNAME + value: {{ `"{{inputs.parameters.notification_email_smtp_username}}"` }} + - name: NOTIFICATION_EMAIL_SMTP_STARTTLS + value: {{ `"{{inputs.parameters.notification_email_smtp_starttls}}"` }} + - name: NOTIFICATION_EMAIL_FROM + value: {{ `"{{inputs.parameters.notification_email_from}}"` }} + - name: NOTIFICATION_EMAIL_TO + value: {{ `"{{inputs.parameters.notification_email_to}}"` }} + - name: NOTIFICATION_EMAIL_SUBJECT_PREFIX + value: {{ `"{{inputs.parameters.notification_email_subject_prefix}}"` }} + # Credential material — sourced from the k8s Secret of the + # active driver. Rendered conditionally so an unconfigured + # cluster (driver="") does not try to resolve a non-existent + # Secret. The parser treats a missing env var as "log only". + # `NOTIFICATION_URL` for URL-based drivers (webhook/zulip/slack); + # `NOTIFICATION_EMAIL_SMTP_PASSWORD` for the email driver. + {{- $n := .Values.ingestion.freshness.notification }} + {{- if eq $n.driver "webhook" }} + - name: NOTIFICATION_URL + valueFrom: + secretKeyRef: + name: {{ required "ingestion.freshness.notification.webhook.urlSecret.name is required when driver=webhook" $n.webhook.urlSecret.name | quote }} + key: {{ ($n.webhook.urlSecret.key | default "url") | quote }} + {{- else if eq $n.driver "zulip" }} + - name: NOTIFICATION_URL + valueFrom: + secretKeyRef: + name: {{ required "ingestion.freshness.notification.zulip.urlSecret.name is required when driver=zulip" $n.zulip.urlSecret.name | quote }} + key: {{ ($n.zulip.urlSecret.key | default "url") | quote }} + {{- else if eq $n.driver "slack" }} + - name: NOTIFICATION_URL + valueFrom: + secretKeyRef: + name: {{ required "ingestion.freshness.notification.slack.urlSecret.name is required when driver=slack" $n.slack.urlSecret.name | quote }} + key: {{ ($n.slack.urlSecret.key | default "url") | quote }} + {{- else if eq $n.driver "teams" }} + - name: NOTIFICATION_URL + valueFrom: + secretKeyRef: + name: {{ required "ingestion.freshness.notification.teams.urlSecret.name is required when driver=teams" $n.teams.urlSecret.name | quote }} + key: {{ ($n.teams.urlSecret.key | default "url") | quote }} + {{- else if eq $n.driver "email" }} + {{- if $n.email.smtp.passwordSecret.name }} + - name: NOTIFICATION_EMAIL_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $n.email.smtp.passwordSecret.name | quote }} + key: {{ ($n.email.smtp.passwordSecret.key | default "password") | quote }} + {{- end }} + {{- end }} + source: | + # SECURITY: every parameter reaches us via env vars only — no + # interpolation into shell or YAML source. Same convention as + # dbt-run.yaml. + set -euo pipefail + cd /ingestion/dbt + python3 - <<'PY' + import os, yaml + profile = { + "ingestion": { + "target": "k8s", + "outputs": { + "k8s": { + "type": "clickhouse", + "host": os.environ["CLICKHOUSE_HOST"], + "port": int(os.environ["CLICKHOUSE_PORT"]), + # Sentinel: source freshness reads each source's + # declared schema (`bronze_*`), never the profile + # default. Pick a name that won't collide if dbt + # ever falls back here. + "schema": "_freshness_check", + "user": os.environ["CLICKHOUSE_USER"], + "password": os.environ["CLICKHOUSE_PASSWORD"], + "secure": False, + "send_receive_timeout": 600, + "query_limit": 0, + "connect_timeout": 30, + } + } + } + } + with open("profiles.yml", "w") as f: + yaml.safe_dump(profile, f) + PY + + # Drop any stale freshness report first. If dbt crashes mid-run we + # want exit code 2 below, not a misclassification using a leftover + # `target/sources.json` from a prior invocation that still happens + # to live in the toolbox image's working directory. + rm -f target/sources.json + + # Narrow the freshness selector to bronze_* databases that actually + # exist in this deployment. The freshness check answers "is the + # data we ARE collecting stale?" — it deliberately does not detect + # "is a connector that should be deployed actually deployed?". + # That is a separate deployment-health concern (Helm probes / sync + # workflow status) tracked under a separate issue. Without this + # narrowing, a tenant that doesn't run, say, the OpenAI connector + # would see one ERROR per OpenAI source on every freshness run. + if [ "$DBT_SELECT" = "source:*" ]; then + EXISTING_DBS=$(python3 - <<'PY' + import os, sys, urllib.parse, urllib.request + url = "http://" + os.environ["CLICKHOUSE_HOST"] + ":" + os.environ["CLICKHOUSE_PORT"] + "/" + # ClickHouse LIKE has no ESCAPE clause; `startsWith` avoids the + # underscore-as-wildcard pitfall. We don't currently have any + # database whose name begins with `bronze` but isn't a bronze + # source, so the prefix is precise enough. + sql = "SELECT name FROM system.databases WHERE startsWith(name, 'bronze_') FORMAT TabSeparated" + qs = urllib.parse.urlencode({ + "user": os.environ["CLICKHOUSE_USER"], + "password": os.environ["CLICKHOUSE_PASSWORD"], + "query": sql, + }) + try: + with urllib.request.urlopen(url + "?" + qs, timeout=30) as resp: + print(resp.read().decode("utf-8")) + except Exception as e: + print(f"freshness: failed to enumerate bronze databases ({e}) — falling back to full source:* selector", file=sys.stderr) + sys.exit(0) + PY + ) + if [ -n "$EXISTING_DBS" ]; then + EFFECTIVE_SELECT="" + for db in $EXISTING_DBS; do + EFFECTIVE_SELECT="$EFFECTIVE_SELECT source:$db" + done + EFFECTIVE_SELECT="${EFFECTIVE_SELECT# }" + echo "freshness: narrowed selector to deployed bronze databases — $EFFECTIVE_SELECT" + else + EFFECTIVE_SELECT="$DBT_SELECT" + fi + else + # User-supplied selector is already specific; honor it verbatim. + EFFECTIVE_SELECT="$DBT_SELECT" + fi + + # `dbt source freshness` exits non-zero when any source breaches + # SLA. We swallow the exit code here and re-derive the workflow + # outcome from `target/sources.json` below — dbt does not + # distinguish warn from error in its exit code, but we want to. + # SLA thresholds come from each source's `freshness:` block in the + # connector schema.yml (constructorfabric/insight#1346) — this + # workflow does not pass or override them. + set +e + dbt source freshness --profiles-dir . --select "$EFFECTIVE_SELECT" + set -e + + # Parse target/sources.json. Exit semantics (set by the script): + # 0 — no breaches, OR only `warn` (visible in log; one missed run) + # 1 — at least one `error` or `runtime error` (page-worthy) + # 2 — `target/sources.json` missing (dbt crashed before checking) + # Argo's `failedJobsHistoryLimit` retains the breaching runs. + # `set +e` so we keep going to the trap detector even when the + # freshness parser exits non-zero (which is the page-worthy case). + set +e + python3 - <<'PY' + import json, os, sys, urllib.request + + path = "target/sources.json" + if not os.path.exists(path): + print(f"freshness: no {path} produced — dbt likely crashed before checking sources", file=sys.stderr) + sys.exit(2) + + with open(path) as f: + report = json.load(f) + + # `runtime error` = dbt couldn't check the source at all (CH down, + # schema drift, query failure). Treated as page-worthy alongside + # `error`, since "we don't know" is worse than "we know it's stale". + PAGE_STATUSES = ("error", "runtime error") + BREACH_STATUSES = ("warn",) + PAGE_STATUSES + + breaches = [] + for r in report.get("results", []): + status = r.get("status") + if status in BREACH_STATUSES: + unique_id = r.get("unique_id", "?") + max_loaded_at = r.get("max_loaded_at") + age_seconds = r.get("max_loaded_at_time_ago_in_s") + # Empty-table sentinel: dbt-clickhouse computes + # `MAX()` over zero rows as NULL, which serialises + # as the Unix epoch (`1970-01-01T00:00:00+00:00`) plus + # an enormous `age_in_s`. Raw output reads as ~500k + # hours of lag, which looks like a parser bug rather + # than the real signal ("the source has never received + # a single row"). Flag it explicitly. + empty = ( + isinstance(max_loaded_at, str) + and max_loaded_at.startswith("1970-01-01") + ) + breaches.append({ + "source": unique_id, + "status": status, + "max_loaded_at": "(table is empty)" if empty else max_loaded_at, + "age_hours": None if empty else ( + round(age_seconds / 3600, 1) if age_seconds is not None else None + ), + "empty": empty, + }) + + if not breaches: + print("freshness: all sources within SLA") + sys.exit(0) + + print(f"freshness: {len(breaches)} source(s) breaching SLA:") + for b in breaches: + if b.get("empty"): + detail = "(table is empty — no rows ever ingested)" + else: + age = f"{b['age_hours']}h" if b['age_hours'] is not None else "n/a" + detail = f"max_loaded_at={b['max_loaded_at']}, age={age}" + print(f" [{b['status']:>5}] {b['source']} {detail}") + + # Notification fan-out. The driver selects the body format and + # any driver-specific transport. URL comes from a k8s Secret + # (resolved by the workflow YAML, not by us) and lands in + # `NOTIFICATION_URL` only when a driver is configured. + # + # Adding a new driver is one match arm here + one Helm subblock + # in `values.yaml` + one secretKeyRef branch in the workflow + # YAML. Nothing else changes. + import urllib.parse + driver = os.environ.get("NOTIFICATION_DRIVER", "").strip() + url = os.environ.get("NOTIFICATION_URL", "").strip() + cluster = os.environ.get("CLUSTER", "").strip() + tenant = os.environ.get("TENANT", "").strip() + # Summary prefix: every label that's set, in a single bracket, + # so receivers handling multi-tenant / multi-cluster bots can + # route on either dimension without parsing the breach list. + # Empty labels drop out entirely — single-deployment installs + # see plain "N bronze source(s) breaching ...". + _labels = [] + if cluster: _labels.append(f"cluster={cluster}") + if tenant: _labels.append(f"tenant={tenant}") + _prefix = f"[{', '.join(_labels)}] " if _labels else "" + summary = f"{_prefix}{len(breaches)} bronze source(s) breaching freshness SLA" + has_page_status = any(b["status"] in PAGE_STATUSES for b in breaches) + + def _detail(b): + if b.get("empty"): + return "table is empty (no rows ingested)" + age = f"{b['age_hours']}h" if b['age_hours'] is not None else "n/a" + return f"max_loaded_at={b['max_loaded_at']}, age={age}" + + def _line_md(b): + return f"- `[{b['status']}]` `{b['source']}` — {_detail(b)}" + + def _line_plain(b): + return f" [{b['status']}] {b['source']} — {_detail(b)}" + + def _post(req_url, body, headers): + # Some receivers sit behind Cloudflare / WAF and reject the + # default `Python-urllib/3.x` UA as a bot signature. Set an + # explicit, identifying UA so the request is allowed. + headers = {**headers, "User-Agent": "insight-freshness-monitor/1.0"} + req = urllib.request.Request(req_url, data=body, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=10) as resp: + print(f"freshness: notification posted via {driver} ({resp.status})") + except urllib.error.HTTPError as e: + # Surface the upstream's response body — `urllib` only + # exposes the status line by default and 4xx responses + # from Zulip / Slack / Teams typically carry a JSON + # error message that explains the rejection (wrong + # API key, stream not subscribed, channel-locked, etc.). + reason = "" + try: + reason = e.read().decode("utf-8", errors="replace")[:400] + except Exception: + pass + raise RuntimeError(f"HTTP {e.code}: {reason}") from None + + def _send_webhook(): + payload = { + "topic": "ingestion-freshness", + "cluster": cluster, + "tenant": tenant, + "summary": summary, + "breaches": breaches, + } + _post(url, json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}) + + def _send_zulip(): + # Zulip exposes two incoming-webhook integrations: + # /api/v1/external/json — dumps the body as a JSON code + # block (a debugging/logging integration, no markdown). + # /api/v1/external/slack — Slack-compatible, takes + # `{"text": ""}` and renders properly. + # The chart's URL is whatever the operator configured. If + # they pointed at `.../external/json` we silently rewrite + # the path to `.../external/slack` and use Slack mrkdwn so + # the message arrives formatted instead of dumped as JSON. + stream = os.environ.get("NOTIFICATION_ZULIP_STREAM", "").strip() + topic = os.environ.get("NOTIFICATION_ZULIP_TOPIC", "").strip() + target = url.replace("/api/v1/external/json", "/api/v1/external/slack", 1) + extras = [] + if stream: extras.append("stream=" + urllib.parse.quote(stream)) + if topic: extras.append("topic=" + urllib.parse.quote(topic)) + if extras: + target += ("&" if "?" in target else "?") + "&".join(extras) + # Slack mrkdwn: single `*` for bold (vs `**` in Zulip + # native), backtick spans and bullet lists work the same. + def _line_slack(b): + return f"• `[{b['status']}]` `{b['source']}` — {_detail(b)}" + md = [f"*{summary}*", "", *(_line_slack(b) for b in breaches)] + # Zulip's Slack-compatible incoming webhook expects + # legacy-Slack form fields: `user_name`, `text`, + # `channel_name`. URL query carries stream/topic; the + # form fields carry the message itself. + body = urllib.parse.urlencode({ + "user_name": "ingestion-freshness", + "channel_name": "freshness", + "text": "\n".join(md), + }) + _post(target, body.encode("utf-8"), + {"Content-Type": "application/x-www-form-urlencoded"}) + + def _send_slack(): + channel = os.environ.get("NOTIFICATION_SLACK_CHANNEL", "").strip() + text = "\n".join([f"*{summary}*", *(_line_md(b) for b in breaches)]) + payload = {"text": text} + if channel: + payload["channel"] = channel + _post(url, json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}) + + def _send_teams(): + theme = "FF0000" if has_page_status else "FFA500" + payload = { + "@type": "MessageCard", + "@context": "https://schema.org/extensions", + "summary": summary, + "themeColor": theme, + "title": summary, + "text": "\n\n".join(_line_md(b) for b in breaches), + } + _post(url, json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}) + + def _send_email(): + import smtplib + from email.message import EmailMessage + host = os.environ.get("NOTIFICATION_EMAIL_SMTP_HOST", "").strip() + port = int(os.environ.get("NOTIFICATION_EMAIL_SMTP_PORT", "587") or "587") + username = os.environ.get("NOTIFICATION_EMAIL_SMTP_USERNAME", "").strip() + password = os.environ.get("NOTIFICATION_EMAIL_SMTP_PASSWORD", "") + starttls = os.environ.get("NOTIFICATION_EMAIL_SMTP_STARTTLS", "true").lower() != "false" + sender = os.environ.get("NOTIFICATION_EMAIL_FROM", "").strip() + to_csv = os.environ.get("NOTIFICATION_EMAIL_TO", "").strip() + prefix = os.environ.get("NOTIFICATION_EMAIL_SUBJECT_PREFIX", "").strip() + if not (host and sender and to_csv): + raise RuntimeError( + "email driver requires notification.email.smtp.host, " + "notification.email.from and notification.email.to" + ) + recipients = [r.strip() for r in to_csv.split(",") if r.strip()] + msg = EmailMessage() + msg["Subject"] = (f"{prefix} {summary}" if prefix else summary).strip() + msg["From"] = sender + msg["To"] = ", ".join(recipients) + msg.set_content("\n".join([summary, "", *(_line_plain(b) for b in breaches)])) + smtp_cls = smtplib.SMTP_SSL if port == 465 else smtplib.SMTP + with smtp_cls(host, port, timeout=15) as s: + if starttls and port != 465: + s.starttls() + if username: + s.login(username, password) + s.send_message(msg) + print(f"freshness: notification posted via email ({len(recipients)} recipient(s))") + + dispatchers = { + "webhook": _send_webhook, + "zulip": _send_zulip, + "slack": _send_slack, + "teams": _send_teams, + "email": _send_email, + } + + if not driver: + print("freshness: notification driver is empty — log only") + elif driver not in dispatchers: + print(f"freshness: unknown driver '{driver}' — log only", file=sys.stderr) + elif driver != "email" and not url: + print(f"freshness: driver={driver} but NOTIFICATION_URL is empty — log only", file=sys.stderr) + else: + try: + dispatchers[driver]() + except Exception as e: + # Notification failure does not change the run's primary + # exit code — we'd rather have a noisy log + breached SLA + # in workflow status than swallow the upstream failure. + print(f"freshness: {driver} notification failed: {e}", file=sys.stderr) + + # Page only on `error` / `runtime error`. Warn-only runs stay + # exit 0 so they don't drown on-call in noise; they're still + # printed above and POSTed in the payload for triage. + sys.exit(1 if any(b["status"] in PAGE_STATUSES for b in breaches) else 0) + PY + freshness_exit=$? + set -e + + # Trap detector — runs after the freshness check itself. Catches + # bronze sources whose `_airbyte_extracted_at` anchor is masking + # a re-emit / incremental-topup pattern (the freshness check + # returns PASS even though the upstream has stopped publishing). + # Its findings are *warnings*: logged for visibility, but the + # workflow's page/no-page decision is owned by the freshness + # check itself — we do not override it. + set +e + python3 /ingestion/scripts/freshness-trap-detect.py /ingestion/connectors + trap_exit=$? + set -e + if [ "$trap_exit" -eq 1 ]; then + echo "freshness-trap-detect: found suspect(s) — review the connector schema.yml for those tables (see src/ingestion/MONITORING.md)" + elif [ "$trap_exit" -ne 0 ]; then + echo "freshness-trap-detect: script error (exit $trap_exit) — not affecting workflow status" >&2 + fi + + exit "$freshness_exit" +--- +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: dbt-source-freshness-check + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: ingestion-monitoring + {{- include "insight.labels" . | nindent 4 }} + {{- with .Values.ingestion.reconcile.argoInstanceId }} + # When the cluster's Argo workflow-controller runs with `instanceID:` + # set, it only schedules CronWorkflows carrying a matching + # controller-instanceid label. Without this the cron silently never + # ticks (lastScheduled stays `never`). Reuses the reconcile-loop key. + workflows.argoproj.io/controller-instanceid: {{ . | quote }} + {{- end }} +spec: + schedules: + - {{ .Values.ingestion.freshness.schedule | quote }} + timezone: UTC + concurrencyPolicy: Forbid + startingDeadlineSeconds: 600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + workflowSpec: + serviceAccountName: argo-workflow + podGC: + strategy: OnPodCompletion + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 6 }} + {{- end }} + entrypoint: run + templates: + - name: run + steps: + - - name: check + templateRef: + name: dbt-source-freshness + template: run + arguments: + parameters: + - name: dbt_select + value: {{ .Values.ingestion.freshness.dbtSelect | quote }} + - name: toolbox_image + value: {{ required "ingestion.toolboxImage is required for the freshness CronWorkflow — pin a tag in values overrides" .Values.ingestion.toolboxImage | quote }} + - name: clickhouse_host + value: {{ include "insight.clickhouse.host" . | quote }} + - name: clickhouse_port + value: {{ include "insight.clickhouse.port" . | quote }} + - name: clickhouse_user + value: {{ .Values.clickhouse.username | quote }} + - name: cluster + value: {{ .Values.ingestion.freshness.cluster | quote }} + - name: tenant + value: {{ .Values.ingestion.freshness.tenant | default "" | quote }} + {{- $n := .Values.ingestion.freshness.notification }} + - name: notification_driver + value: {{ $n.driver | default "" | quote }} + - name: notification_zulip_stream + value: {{ $n.zulip.stream | default "" | quote }} + - name: notification_zulip_topic + value: {{ $n.zulip.topic | default "" | quote }} + - name: notification_slack_channel + value: {{ $n.slack.channel | default "" | quote }} + - name: notification_email_smtp_host + value: {{ $n.email.smtp.host | default "" | quote }} + - name: notification_email_smtp_port + value: {{ ($n.email.smtp.port | default 587) | quote }} + - name: notification_email_smtp_username + value: {{ $n.email.smtp.username | default "" | quote }} + - name: notification_email_smtp_starttls + value: {{ ($n.email.smtp.starttls | default true) | quote }} + - name: notification_email_from + value: {{ $n.email.from | default "" | quote }} + - name: notification_email_to + value: {{ $n.email.to | default "" | quote }} + - name: notification_email_subject_prefix + value: {{ $n.email.subjectPrefix | default "[ingestion-freshness]" | quote }} +{{- end }} diff --git a/charts/insight/values.schema.json b/charts/insight/values.schema.json index 9529843f6..b22e5663f 100644 --- a/charts/insight/values.schema.json +++ b/charts/insight/values.schema.json @@ -50,6 +50,63 @@ } } } + }, + "freshness": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "schedule": { "type": "string", "default": "0 13 * * *" }, + "dbtSelect": { "type": "string", "default": "source:*" }, + "cluster": { "type": "string" }, + "tenant": { "type": "string" }, + "notification": { + "type": "object", + "properties": { + "driver": { "enum": ["", "webhook", "zulip", "slack", "teams", "email"] }, + "webhook": { + "type": "object", + "properties": { "urlSecret": { "$ref": "#/definitions/urlSecret" } } + }, + "zulip": { + "type": "object", + "properties": { + "urlSecret": { "$ref": "#/definitions/urlSecret" }, + "stream": { "type": "string" }, + "topic": { "type": "string" } + } + }, + "slack": { + "type": "object", + "properties": { + "urlSecret": { "$ref": "#/definitions/urlSecret" }, + "channel": { "type": "string" } + } + }, + "teams": { + "type": "object", + "properties": { "urlSecret": { "$ref": "#/definitions/urlSecret" } } + }, + "email": { + "type": "object", + "properties": { + "smtp": { + "type": "object", + "properties": { + "host": { "type": "string" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "username": { "type": "string" }, + "passwordSecret": { "$ref": "#/definitions/urlSecret" }, + "starttls": { "type": "boolean" } + } + }, + "from": { "type": "string" }, + "to": { "type": "string" }, + "subjectPrefix": { "type": "string" } + } + } + } + } + } } } }, @@ -140,6 +197,13 @@ }, "definitions": { + "urlSecret": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "key": { "type": "string", "minLength": 1 } + } + }, "infraDep": { "type": "object", "properties": { diff --git a/charts/insight/values.yaml b/charts/insight/values.yaml index c6cf558f9..14efbd1f7 100644 --- a/charts/insight/values.yaml +++ b/charts/insight/values.yaml @@ -236,6 +236,122 @@ ingestion: limits: cpu: "500m" memory: "512Mi" + + # ─── bronze source freshness (ingestion-monitoring) ────────────────────── + # Runtime half of source-freshness monitoring. The per-source `freshness:` + # blocks (loaded_at_field + warn_after/error_after) live in each connector + # `schema.yml` — see constructorfabric/insight#1346 / EPIC #1321. This block + # only governs the CronWorkflow that *runs* `dbt source freshness` on a + # schedule, reads `target/sources.json`, and fans the verdict out to a + # notification channel. It does NOT own threshold values. + freshness: + enabled: true + # Cron expression for the freshness check. Daily at 13:00 UTC sits past + # all current connector sync slots (02:00–11:00 UTC) plus a 2h grace. + schedule: "0 13 * * *" + # dbt selector for the freshness check. `source:*` covers every declared + # source. Narrow this for smoke-testing in dev. + dbtSelect: "source:*" + # Identification labels included in every notification payload so + # receivers can route alerts when one bot is shared across multiple + # deployments / customers. Both default to empty — leaving them out + # of the summary entirely. + # cluster — deployment / installation tier (e.g. "prod", "staging"). + # tenant — customer / workspace identifier. Set when one Insight + # installation serves multiple tenants. + cluster: "" + tenant: "" + # Notification delivery. Each driver has its own subblock with the + # settings it needs; `driver` selects which one is active and the + # rest are inert. New drivers drop in by adding a sibling block + + # extending the parser branch in `dbt-source-freshness.yaml` — no + # other Helm surface needs to change. + # + # Every driver that needs an outgoing URL reads it from a Kubernetes + # Secret (mirrors the `clickhouse.passwordSecret` pattern) so the + # token is not rendered into manifests and not visible in the Argo UI. + notification: + # "" — no fan-out (default). Breaches still surface via Argo run + # status / `failedJobsHistoryLimit`. Use this for clusters + # that don't have an outgoing channel wired yet. + # "webhook" — generic JSON POST. Body shape: + # { topic, cluster, summary, breaches: [...] } + # Suitable for Mattermost / a custom relay / opsgenie ingest. + # "zulip" — Zulip incoming-webhook integration. + # "slack" — Slack incoming-webhook. + # "teams" — Microsoft Teams incoming-webhook (MessageCard). + # "email" — SMTP via stdlib smtplib (no extra deps). + driver: "" + + # ─── webhook driver ───────────────────────────────────────────── + webhook: + # Required when driver=webhook. The full POST URL lives in + # this Secret under `key`. + urlSecret: + name: "" + key: "url" + + # ─── zulip driver ─────────────────────────────────────────────── + zulip: + # Required when driver=zulip. The integration URL Zulip's UI + # gives you when you add an "Incoming webhook" integration + # already encodes stream/topic in its query string — paste it + # verbatim here. The parser silently rewrites the path from + # `.../external/json` (which dumps the body as a JSON code + # block) to `.../external/slack` (Zulip's Slack-compatible + # webhook, which renders markdown), so a JSON-style URL still + # produces a properly-formatted message. + urlSecret: + name: "" + key: "url" + # Routing overrides. Empty → use whatever's in the integration + # URL's query string. Note Zulip stream names can contain a + # trailing zero-width space (U+200B); copy-pasting from the UI + # is safer than retyping. + stream: "" + topic: "" + + # ─── slack driver ─────────────────────────────────────────────── + slack: + # Required when driver=slack. Slack incoming-webhook URL. + urlSecret: + name: "" + key: "url" + # Optional channel override. Slack incoming webhooks are bound + # to a default channel at creation; setting this overrides + # only if the workspace allows it (admin policy). + channel: "" + + # ─── teams driver ─────────────────────────────────────────────── + teams: + # Required when driver=teams. Microsoft Teams incoming-webhook + # URL. The URL is bound to a single channel at creation; there + # is no override knob. + urlSecret: + name: "" + key: "url" + + # ─── email driver ─────────────────────────────────────────────── + email: + smtp: + host: "" + port: 587 # 587 = STARTTLS, 465 = SMTPS, 25 = plain + username: "" # SMTP AUTH user; password lives in Secret + # SMTP password — required when driver=email and the SMTP + # server requires AUTH. Put the password in a k8s Secret. + passwordSecret: + name: "" + key: "password" + # Use STARTTLS on connect. Ignored when port=465 (already + # implicit TLS). + starttls: true + # Envelope. + from: "" # e.g. "ingestion-monitor@example.com" + # Comma-separated list of recipient addresses. Helm values do + # not support arrays through to env vars cleanly without a + # template loop; a single string is the simplest shape. + to: "" # e.g. "oncall@example.com,sre@example.com" + subjectPrefix: "[ingestion-freshness]" # ═══════════════════════════════════════════════════════════════════════════ # INFRASTRUCTURE # ═══════════════════════════════════════════════════════════════════════════ diff --git a/docs/domain/README.md b/docs/domain/README.md index dde5fe40f..002dcd284 100644 --- a/docs/domain/README.md +++ b/docs/domain/README.md @@ -7,5 +7,6 @@ Domain-level specifications for the Insight platform. Each domain represents a b | Domain | Description | Status | |---|---|---| | [`ingestion/`](ingestion/) | Data pipeline from source APIs to Silver step 1 (Airbyte + Argo Workflows + dbt) | Accepted | +| [`ingestion-monitoring/`](ingestion-monitoring/) | Bronze freshness SLA, trap detection, multi-driver alert delivery | Proposed | | [`connector/`](connector/) | Connector development: Insight Connector packages, nocode and CDK patterns, packaging, debugging | Accepted | | [`identity-resolution/`](identity-resolution/) | Person identity matching and resolution across sources | Proposed | diff --git a/docs/domain/ingestion-monitoring/specs/DESIGN.md b/docs/domain/ingestion-monitoring/specs/DESIGN.md new file mode 100644 index 000000000..f47c5d988 --- /dev/null +++ b/docs/domain/ingestion-monitoring/specs/DESIGN.md @@ -0,0 +1,874 @@ +--- +status: proposed +date: 2026-05-04 +--- + +# Technical Design — Ingestion Monitoring + + + +- [1. Architecture Overview](#1-architecture-overview) + - [1.1 Architectural Vision](#11-architectural-vision) + - [1.2 Architecture Drivers](#12-architecture-drivers) + - [1.3 Architecture Layers](#13-architecture-layers) +- [2. Principles & Constraints](#2-principles--constraints) + - [2.1 Design Principles](#21-design-principles) + - [2.2 Constraints](#22-constraints) +- [3. Technical Architecture](#3-technical-architecture) + - [3.1 Domain Model](#31-domain-model) + - [3.2 Component Model](#32-component-model) + - [3.3 API Contracts](#33-api-contracts) + - [3.4 Internal Dependencies](#34-internal-dependencies) + - [3.5 External Dependencies](#35-external-dependencies) + - [3.6 Interactions & Sequences](#36-interactions--sequences) + - [3.7 Database schemas & tables](#37-database-schemas--tables) + - [3.8 Deployment Topology](#38-deployment-topology) +- [4. Additional context](#4-additional-context) + - [4.1 Why two report tiers (`report`, `report_extended`) instead of one](#41-why-two-report-tiers-report-reportextended-instead-of-one) + - [4.2 Why `_airbyte_extracted_at` for streaming and a business date for re-emit](#42-why-airbyteextractedat-for-streaming-and-a-business-date-for-re-emit) + - [4.3 Why per-driver subblocks instead of a flat `notification.url`](#43-why-per-driver-subblocks-instead-of-a-flat-notificationurl) + - [4.4 Why the Zulip dispatcher silently rewrites the endpoint](#44-why-the-zulip-dispatcher-silently-rewrites-the-endpoint) + - [4.5 Why narrow the selector at runtime](#45-why-narrow-the-selector-at-runtime) + - [4.6 Why the trap detector is advisory](#46-why-the-trap-detector-is-advisory) + - [4.7 Why every `freshness: null` opt-out needs a rationale](#47-why-every-freshness-null-opt-out-needs-a-rationale) +- [5. Traceability](#5-traceability) + + + +## 1. Architecture Overview + +### 1.1 Architectural Vision + +Ingestion Monitoring is a single daily-cadence Argo `CronWorkflow` +(`dbt-source-freshness-check`) that delegates to one stateless +`WorkflowTemplate` (`dbt-source-freshness`). The template runs `dbt +source freshness` against ClickHouse, parses `target/sources.json` with +an inline Python step, and dispatches breaches to one of five +notification drivers (webhook / zulip / slack / teams / email). Two +scripts sit at the edges: a CI lint that prevents bad anchors from +landing, and a runtime trap detector that flags re-emit patterns the +freshness check itself cannot see. + +The design's defining property is **everything is config-driven from a +single Helm block** (`ingestion.freshness.*` in +[`charts/insight/values.yaml`](../../../../charts/insight/values.yaml)). +Adding a connector is a `schema.yml` edit; switching notification +channels is a `values.yaml` edit; tuning thresholds is a `values.yaml` +edit. No Rust services, no extra Deployments, no per-connector pipeline +plumbing. + +The second defining property is **credential isolation by +construction**. The driver's URL or SMTP password is bound to the pod's +env from a `secretKeyRef` resolved server-side by Argo; the rendered +manifest, the Argo UI, and the workflow-controller logs only ever see +the Secret reference, never the raw value +([`charts/insight/templates/ingestion/dbt-source-freshness.yaml:161-194`](../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml)). + +### 1.2 Architecture Drivers + +#### Functional Drivers + +| PRD Requirement | Design Response | +|---|---| +| `cpt-insightspec-fr-mon-daily-cronworkflow` | Argo `CronWorkflow` `dbt-source-freshness-check`, one daily run at the Helm-configured cron. | +| `cpt-insightspec-fr-mon-selector-narrowing` | Pre-step queries `system.databases`, builds the effective dbt selector. | +| `cpt-insightspec-fr-mon-stale-report-cleanup` | `rm -f target/sources.json` before every dbt invocation. | +| `cpt-insightspec-fr-mon-exit-rederivation` | Inline Python parser re-derives outcome from `target/sources.json`; dbt's exit code is swallowed. | +| `cpt-insightspec-fr-mon-thresholds-sot` | Literal `warn_after`/`error_after` per source in `schema.yml`; the CronWorkflow reads them via `--select source:*`. No Helm/env-var layer. | +| `cpt-insightspec-fr-mon-four-tiers` | Per-source `freshness:` blocks with literal tier values `default` / `event` / `report` / `report_extended`. | +| `cpt-insightspec-fr-mon-tier-assignment` | Tier choice lives as literal values in connector `schema.yml`; no runtime tuning knob. | +| `cpt-insightspec-fr-mon-optout-pertable` | Per-table `freshness: null` in `schema.yml`. | +| `cpt-insightspec-fr-mon-optout-rationale` | One-line rationale comment beside each `freshness: null`; reserved for incremental streams. | +| `cpt-insightspec-fr-mon-mandatory-anchor` | Per-source `loaded_at_field`; coverage counted by the QA `dbt_coverage.py` gate (#1321). A missing/invalid anchor surfaces as a dbt `runtime error`. | +| `cpt-insightspec-fr-mon-trap-post-parser` | Workflow runs `freshness-trap-detect.py` after the parser, advisory-only. | +| `cpt-insightspec-fr-mon-trap-two-modes` | Heuristic full-reemit + opt-in business-date divergence in the same script. | +| `cpt-insightspec-fr-mon-trap-skip-annotation` | `meta.bronze_freshness_trap_check: skip` honoured by detector. | +| `cpt-insightspec-fr-mon-trap-advisory` | Detector exit only writes a notice; freshness parser owns the page-worthy verdict. | +| `cpt-insightspec-fr-mon-driver-selection` | `notification.driver` enum validated by `values.schema.json`. | +| `cpt-insightspec-fr-mon-credential-isolation` | Per-driver `secretKeyRef` branches in the workflow template. | +| `cpt-insightspec-fr-mon-dispatch-impl` | `dispatchers` dict in the inline parser. | +| `cpt-insightspec-fr-mon-delivery-failure-isolation` | `try/except` around dispatch; primary exit code untouched. | +| `cpt-insightspec-fr-mon-identity-helm` | `cluster` / `tenant` Helm values → `CLUSTER` / `TENANT` env vars. | +| `cpt-insightspec-fr-mon-identity-summary` | Summary prefix builder drops empty labels. | +| `cpt-insightspec-fr-mon-identity-payload` | `cluster` / `tenant` written as raw JSON keys. | + +#### NFR Allocation + +| NFR | Component(s) | Verification | +|---|---|---| +| `cpt-insightspec-nfr-mon-idempotency` | Inline parser + workflow template (no persistent state) | Re-run; verify identical exit + payload. | +| `cpt-insightspec-nfr-mon-daily-cadence` | `CronWorkflow.schedule` | Inspect rendered CronWorkflow; manual ad-hoc submission still works. | +| `cpt-insightspec-nfr-mon-credential-isolation` | `secretKeyRef` branches in workflow template | `kubectl get workflowtemplate ... -o yaml` carries no raw URLs/passwords. | +| `cpt-insightspec-nfr-mon-exit-codes` | Inline parser exit logic | Inject fixtures; assert exits 0 / 1 / 2. | +| `cpt-insightspec-nfr-mon-ci-gated` | QA-owned `dbt_coverage.py` gate (#1321) | Open PR with an undeclared `bronze_*` source; coverage gate fails. | +| `cpt-insightspec-nfr-mon-trap-advisory-mode` | Trap detector exit handling | Force trap finding; verify primary exit code unchanged. | +| `cpt-insightspec-nfr-mon-activation-deadline` | `WorkflowTemplate.activeDeadlineSeconds: 1200` | Inspect rendered template. | +| `cpt-insightspec-nfr-mon-empty-sentinel` | Inline parser `1970-01-01` detection | Empty bronze table fixture; verify `(table is empty)` line. | + +### 1.3 Architecture Layers + +``` +ingestion.freshness.* (charts/insight/values.yaml) + │ + ▼ +CronWorkflow.spec.workflowSpec parameters +(charts/insight/templates/ingestion/dbt-source-freshness.yaml:583-662) + │ + ▼ +WorkflowTemplate inputs.parameters → env vars + secretKeyRef +(.yaml:34-212) + │ + ▼ +dbt source freshness ─► target/sources.json +(reads literal warn_after/error_after per source in schema.yml) + │ + ▼ +Inline Python parser (.yaml:317-562) + │ │ + │ └─► driver dispatcher (webhook / zulip / slack / teams / email) + │ │ + │ └─► external sink (Zulip / Slack / Teams / SMTP / generic) + ▼ +freshness-trap-detect.py (advisory) +(src/ingestion/scripts/freshness-trap-detect.py) +``` + +## 2. Principles & Constraints + +### 2.1 Design Principles + +#### Advisory failure mode for traps + +- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-trap-advisory` + +The trap detector logs but never overrides the freshness parser's exit +code (workflow template lines 575–579). Page-worthy decisions belong to +one place — making heuristic checks page-worthy would erode trust in +the whole monitoring domain. + +#### Log loud on failure + +- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-log-loud` + +Notification HTTP errors surface the upstream's body (workflow template +lines 421–432) — the `Python-urllib` default exposes only the status +line, which hides Zulip / Slack / Teams JSON error payloads. Loud logs +are the difference between "fix in 5 min" and "fix in 5 hours". + +#### Scope freshness to what's collected + +- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-scope-collected` + +Selector is narrowed from `source:*` to `bronze_*` databases that +exist in `system.databases` (lines 262–297); a tenant that didn't +deploy OpenAI doesn't see phantom OpenAI ERRORs. Deployment health is +the deployment-health workstream +([#272](https://github.com/cyberfabric/insight/issues/272)), not this +domain. + +#### Vendor-documented thresholds, not gut feeling + +- [ ] `p1` - **ID**: `cpt-insightspec-principle-mon-vendor-thresholds` + +Each tier's warn / error pair sits at the upper edge of the vendor's +documented normal cadence (M365 24–48 h → warn 48 h; Slack ~3 d → +warn 72 h). Documented in `values.yaml` lines 156–183. Re-tiering +requires either a vendor change or evidence from a multi-week +observation window. + +### 2.2 Constraints + +#### dbt-clickhouse needs explicit `loaded_at_field` + +- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-explicit-anchor` + +The dbt-clickhouse adapter does not support metadata-based freshness; +a missing field → `runtime error`. `+loaded_at_field` at project level +is silently ignored (`loaded_at_field` is a *property*, not a config). +Hence every source declares its own `loaded_at_field`, and coverage is +counted by the QA `dbt_coverage.py` gate (#1321). + +#### Thresholds are literal per source, not inherited + +- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-freshness-propagates` + +Each source carries its own literal `freshness:` block (`warn_after` / +`error_after`) and `loaded_at_field` directly in `schema.yml`. There is no +project-level `+freshness` default and no Helm/env-var threshold layer — the +`dbt_coverage.py` gate counts per-source declarations, so inheritance would not +register as coverage anyway. Tier values live with the source. + +#### Zulip's `/external/json` is not a markdown sender + +- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-zulip-endpoint` + +`/api/v1/external/json` dumps the request body as a JSON code block. +The Slack-compatible endpoint (`/api/v1/external/slack`) renders +Slack-style markdown. The dispatcher silently rewrites the path so an +operator who pasted the JSON URL still gets formatted output (workflow +template lines 445–478). + +#### Cloudflare-managed receivers reject the default UA + +- [ ] `p2` - **ID**: `cpt-insightspec-constraint-mon-explicit-ua` + +A bare `Python-urllib/3.x` looks like a bot. The dispatcher sets an +explicit `User-Agent: insight-freshness-monitor/1.0` (workflow +template line 416). + +#### Empty table sentinel must not read as 500 000 h lag + +- [ ] `p1` - **ID**: `cpt-insightspec-constraint-mon-empty-sentinel` + +`MAX()` over zero rows is `NULL`. dbt-clickhouse serialises it as +the Unix epoch with an enormous `age_in_s`, which reads as ~500 000 h +of lag. The parser detects the sentinel +(`max_loaded_at.startswith("1970-01-01")`) and surfaces "table is +empty (no rows ingested)" instead (workflow template lines 348–360). + +## 3. Technical Architecture + +### 3.1 Domain Model + +| Concept | Where it's defined | +|---|---| +| **Bronze source** | `sources:` entry in any connector's `dbt/schema.yml`; identified by `name: bronze_*` / `schema: bronze_*`. | +| **Freshness anchor** | `loaded_at_field` property at source or table level. Two valid forms: `_airbyte_extracted_at` (incremental) or a business-date expression (`parseDateTimeBestEffortOrNull(...)`, etc.) for windowed connectors. | +| **Opt-out** | Per-table `freshness: null` plus a one-line rationale comment; reserved for incremental streams that legitimately go quiet. | +| **SLA tier** | One of `default`, `event`, `report`, `report_extended`; chosen by the literal `warn_after`/`error_after` values in the connector's source-level `freshness:` block. | +| **Breach** | A row in the parser's `breaches` list: `{source, status, max_loaded_at, age_hours, empty}` (workflow template lines 352–360). | +| **Trap suspect** | A finding from the trap detector: `kind ∈ {full-reemit, incremental-topup}` plus row-level evidence (`freshness-trap-detect.py:186-227`). | +| **Driver** | A parser dispatch arm + Helm subblock + workflow-yaml `secretKeyRef` branch. Five today (`webhook`, `zulip`, `slack`, `teams`, `email`); `""` = log-only. | + +### 3.2 Component Model + +#### Workflow Template (`dbt-source-freshness`) + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-workflow-template` + +##### Why this component exists + +Provides the stateless Argo `WorkflowTemplate` that any caller +(scheduled or ad-hoc) can invoke with a single set of parameters. +Centralises the dbt invocation, parser, dispatcher, and trap detector +so that switching tenants or drivers does not duplicate workflow +plumbing. + +##### Responsibility scope + +- Accept every per-deployment parameter via `inputs.parameters` + (charts/insight/templates/ingestion/dbt-source-freshness.yaml:34-106). +- Bind credentials via `secretKeyRef` only when the matching driver is + configured (lines 161–194). +- Run `dbt source freshness` with the effective selector (lines + 254–307). +- Hand off to the inline parser, then the trap detector, with the + shell wrapper preserving the parser's exit code (lines 561–581). + +##### Responsibility boundaries + +- Does NOT decide *when* to run — that is the `CronWorkflow`'s job. +- Does NOT own credential storage — Kubernetes Secrets do. +- Does NOT classify breaches — the inline parser does (see + `cpt-insightspec-component-mon-parser`). + +##### Related components (by ID) + +- `cpt-insightspec-component-mon-cronworkflow` — schedules this template. +- `cpt-insightspec-component-mon-parser` — invoked inline. +- `cpt-insightspec-component-mon-trap-detector` — invoked after the parser. +- `cpt-insightspec-component-mon-helm-surface` — supplies parameters. + +#### CronWorkflow (`dbt-source-freshness-check`) + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-cronworkflow` + +##### Why this component exists + +Wraps the workflow template with a daily schedule and feeds Helm values +into its parameters. Argo handles cron parsing, missed-fire policy, +and `failedJobsHistoryLimit` retention. + +##### Responsibility scope + +- `spec.schedule` from `ingestion.freshness.schedule`. +- `spec.workflowSpec` parameters from the Helm block (lines 583–662). +- Default `concurrencyPolicy: Forbid` so a long run cannot be doubled. + +##### Responsibility boundaries + +- Does NOT execute logic itself — it only delegates to the + `WorkflowTemplate`. +- Does NOT carry credentials or notification config beyond what the + template parameters require. + +##### Related components (by ID) + +- `cpt-insightspec-component-mon-workflow-template` — the delegate. +- `cpt-insightspec-component-mon-helm-surface` — Helm values it + flattens into parameters. + +#### Inline Freshness Parser + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-parser` + +##### Why this component exists + +dbt's exit code does not distinguish PASS / WARN / ERROR / RUNTIME +ERROR; we need a deterministic verdict. The parser reads +`target/sources.json` and computes the canonical breach list and exit +code that the dispatcher uses. + +##### Responsibility scope + +- Read `target/sources.json`; if missing, exit 2 (line 323). +- Classify each source as `pass` / `warn` / `error` / `runtime error`. +- Detect the empty-table sentinel and emit `(table is empty)` instead + of ~500 000 h lag (lines 348–360). +- Build the canonical breach record `{source, status, max_loaded_at, + age_hours, empty}`. +- Decide page vs no-page (`sys.exit(1)` if any breach is `error` or + `runtime error`, else `0`; line 559). +- Call the dispatcher when a driver is configured. + +##### Responsibility boundaries + +- Does NOT run dbt itself. +- Does NOT format driver-specific payloads — that is the dispatcher's + job. +- Does NOT write any state to disk other than the dbt-produced + `target/sources.json` (already there). + +##### Related components (by ID) + +- `cpt-insightspec-component-mon-workflow-template` — invokes it. +- `cpt-insightspec-component-mon-driver-dispatcher` — invoked from inside + the parser. +- `cpt-insightspec-component-mon-trap-detector` — runs after. + +#### Driver Dispatcher + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-driver-dispatcher` + +##### Why this component exists + +Different sinks (Zulip, Slack, Teams, SMTP, generic webhook) need +different payload shapes, content types, and quirks. A single +dispatcher dict (workflow template lines 533–539) keeps the parser +simple and makes adding a new driver a one-arm change. + +##### Responsibility scope + +- Map driver name → format function → `_post`/SMTP transport. +- Set `User-Agent: insight-freshness-monitor/1.0` (line 416). +- Surface upstream HTTP body on `HTTPError` (lines 421–432). +- Rewrite Zulip's `/external/json` → `/external/slack` for the Zulip + driver (line 457). +- Catch and log every dispatch failure without changing the workflow's + primary exit code (lines 550–554). + +##### Responsibility boundaries + +- Does NOT classify breaches. +- Does NOT decide whether to dispatch — the parser does. +- Does NOT manage Secret content; only consumes env vars bound from + Secrets. + +##### Related components (by ID) + +- `cpt-insightspec-component-mon-parser` — caller. +- `cpt-insightspec-component-mon-helm-surface` — driver enum + per-driver + Helm subblocks. + +#### Trap Detector + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-trap-detector` + +##### Why this component exists + +The freshness check on `_airbyte_extracted_at` is fooled by re-emit / +incremental-topup patterns. A second, independent script flags those +suspects so authors and operators see them — without overriding the +freshness verdict. + +##### Responsibility scope + +- Mode 1 (heuristic full-reemit): ≥ 95 % rows within last 30 h, ≤ 2 + distinct extract days, ≥ 100 rows + (`freshness-trap-detect.py:188-200`). +- Mode 2 (opt-in business-date divergence): compare + `MAX()` to `MAX(_airbyte_extracted_at)`; + flag ≥ 24 h gap (lines 202–227). +- Honour `meta.bronze_freshness_trap_check: skip` per source/table. +- Print finding lists to stdout; per-table query failures and bootstrap + errors go to stderr. Exit code is captured by the workflow but not + used to override the parser's verdict. + +##### Responsibility boundaries + +- Does NOT page anyone. +- Does NOT mutate ClickHouse data. +- Does NOT read driver Helm config. + +##### Related components (by ID) + +- `cpt-insightspec-component-mon-workflow-template` — invokes it after the + parser. +- `cpt-insightspec-component-mon-parser` — its verdict is preserved. + +#### Per-source declaration coverage (external — QA gate) + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-ci-lint` + +##### Why this is not a component of this module + +Catching an undeclared / mis-anchored `bronze_*` source at PR time is real, +but it is owned by the QA `scripts/ci/dbt_coverage.py` gate (EPIC #1321 / +#1322), which counts per-source `loaded_at_field` + `freshness` declarations. +This module does not ship its own connector-schema lint. The two failure +modes that gate does not cover — choosing the *wrong* anchor (windowed source +left on `_airbyte_extracted_at` → false-green) or a non-timestamp column +(→ dbt `runtime error`) — are caught by confirming `ext_age` vs `biz_age` +against live data, not statically. + +##### Related components (by ID) + +- `cpt-insightspec-component-mon-workflow-template` — runtime counterpart. + +#### Thresholds are literal per source (no env-var layer) + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-threshold-env` + +##### Why there is no threshold component + +Earlier drafts routed tier values through Helm → workflow params → +`FRESHNESS_*_H` env → dbt `env_var(...)`. That was dropped: the +`dbt_coverage.py` gate counts per-source declarations, so thresholds live as +**literal `warn_after`/`error_after` values in each source's `schema.yml`** — +a single source of truth with no second place to drift and no Helm/env layer. +The CronWorkflow simply runs `dbt source freshness --select source:*` and acts +on whatever each source declares. + +#### Helm Surface + +- [ ] `p1` - **ID**: `cpt-insightspec-component-mon-helm-surface` + +##### Why this component exists + +Operators need a single tunable surface. `ingestion.freshness.*` in +`values.yaml` is the documented configuration entrypoint; +`values.schema.json` validates it. + +##### Responsibility scope + +- `ingestion.freshness.{enabled,schedule,dbtSelect,cluster,tenant}` + top-level keys. (No `thresholds.*` block — thresholds are literal per + source in `schema.yml`, not Helm-tuned.) +- `ingestion.freshness.notification.{driver, webhook, zulip, slack, + teams, email}` per-driver subblocks with `urlSecret` references. +- `values.schema.json` enforces the driver enum (line 37) and the + shape of each subblock. + +##### Responsibility boundaries + +- Does NOT carry credentials directly — only `secretKeyRef` pointers. +- Does NOT decide tier choice for connectors. + +##### Related components (by ID) + +- `cpt-insightspec-component-mon-cronworkflow` — flattens this into + parameters. +- `cpt-insightspec-component-mon-driver-dispatcher` — reads driver + selection. + +### 3.3 API Contracts + +The driver dispatcher is the `dispatchers` dict at workflow template +lines 533–539. Common preamble: every dispatch goes through `_post` +(lines 412–432), which sets `User-Agent: insight-freshness-monitor/1.0` +and surfaces the upstream's response body on `HTTPError`. + +#### 3.3.1 `webhook` + +- Endpoint: whatever is in `NOTIFICATION_URL` (verbatim). +- Body shape (workflow template lines 434–443): + +```json +{ + "topic": "ingestion-freshness", + "cluster": "", + "tenant": "", + "summary": "[cluster=…, tenant=…] N bronze source(s) breaching freshness SLA", + "breaches": [ + { + "source": "source.ingestion.bronze_jira.jira_issue", + "status": "error", + "max_loaded_at": "2026-04-28T03:14:21Z", + "age_hours": 51.2, + "empty": false + } + ] +} +``` + +- Header: `Content-Type: application/json`. +- Secret binding: + `ingestion.freshness.notification.webhook.urlSecret.{name,key}` + (default `key: url`), workflow YAML lines 162–167. + +#### 3.3.2 `zulip` + +- Endpoint: `NOTIFICATION_URL` with the path `/api/v1/external/json` + rewritten to `/api/v1/external/slack` (line 457). The two paths are + different Zulip integrations — `/external/json` dumps the body as a + JSON code block (debugging integration); `/external/slack` is the + Slack-compatible incoming webhook that renders Slack mrkdwn. + Operators who pasted the JSON URL still get a formatted message. +- Routing: `stream` and `topic` from `NOTIFICATION_ZULIP_STREAM` / + `NOTIFICATION_ZULIP_TOPIC` are URL-encoded and appended to the query + string (lines 458–462). +- Body: `application/x-www-form-urlencoded` carrying + `user_name=ingestion-freshness`, `channel_name=freshness`, + `text=` — the Slack-compatible webhook expects these + legacy-Slack form fields (lines 472–478). +- Markdown: single `*` for bold (Slack mrkdwn), bullet `•` (line 466). +- Secret binding: + `ingestion.freshness.notification.zulip.urlSecret.{name,key}` + (workflow YAML lines 168–173). +- Quirk: Zulip stream names can include trailing zero-width-space + U+200B; `values.yaml:243-249` warns operators to copy from Zulip's + UI rather than retype. + +#### 3.3.3 `slack` + +- Endpoint: `NOTIFICATION_URL` (Slack incoming-webhook URL). +- Body: `application/json` with `{"text": ""}`, optionally + `channel: ` if `NOTIFICATION_SLACK_CHANNEL` is set and + workspace policy allows it (lines 480–487). +- Secret binding: + `ingestion.freshness.notification.slack.urlSecret.{name,key}` + (workflow YAML lines 174–179). + +#### 3.3.4 `teams` + +- Endpoint: `NOTIFICATION_URL` (Microsoft Teams incoming webhook). +- Body: `application/json`, `MessageCard` schema (lines 489–500). + `themeColor` is `FF0000` (red) when any breach is page-worthy, + `FFA500` (orange) otherwise. +- No channel override — Teams binds the URL to a single channel at + creation. +- Secret binding: + `ingestion.freshness.notification.teams.urlSecret.{name,key}` + (workflow YAML lines 180–185). + +#### 3.3.5 `email` + +- Transport: `smtplib` from the standard library (no extra deps). + `SMTP_SSL` when port=465, otherwise plain `SMTP` with optional + `STARTTLS` (lines 502–531). +- Required Helm values: `notification.email.smtp.host`, + `notification.email.from`, `notification.email.to`. The dispatcher + raises `RuntimeError` if any is missing (lines 513–517). +- Subject: ` ` when prefix is non-empty. +- Recipients: comma-separated `NOTIFICATION_EMAIL_TO` split on `,`, + whitespace-stripped. +- Secret binding: + `ingestion.freshness.notification.email.smtp.passwordSecret.{name,key}` + (workflow YAML lines 187–193, rendered only when + `$n.email.smtp.passwordSecret.name` is set). + +### 3.4 Internal Dependencies + +| From | To | Mechanism | +|---|---|---| +| CronWorkflow | WorkflowTemplate | Argo `templateRef` (same namespace). | +| WorkflowTemplate | dbt-clickhouse | Container exec inside `insight-toolbox`. | +| WorkflowTemplate | ClickHouse | dbt profile points at CH HTTP endpoint. | +| WorkflowTemplate | Trap detector | Container exec inside `insight-toolbox` after parser. | +| Trap detector | ClickHouse | HTTP queries on port 8123. | +| Inline parser | Driver dispatcher | In-process Python call. | +| Driver dispatcher | Kubernetes Secret | `secretKeyRef` env binding (server-side resolution). | + +### 3.5 External Dependencies + +#### Argo Workflows controller + +Provides `WorkflowTemplate` / `CronWorkflow` reconciliation. Argo is +**not** a chart dependency of the umbrella — operators install it out +of band (see [`scripts/install-argo.sh`](../../../../scripts/install-argo.sh) +referenced in `charts/insight/Chart.yaml`'s preamble). Tested against +v3.5.10 and v3.6.x; minimum CRDs needed are `workflows.argoproj.io`, +`workflowtemplates.argoproj.io`, and `cronworkflows.argoproj.io`. + +#### dbt-clickhouse adapter + +Bundled in `insight-toolbox`. Required because the adapter does not +support metadata-based freshness — `loaded_at_field` is the only +supported anchor expression. + +#### ClickHouse + +Provides `system.databases` for selector narrowing and the HTTP +endpoint used by the trap detector. + +#### Kubernetes Secret store + +`secretKeyRef` resolution for `clickhouse.passwordSecret` and the +active driver's URL / SMTP password Secret. + +#### Notification receivers + +Zulip / Slack / Microsoft Teams incoming-webhook endpoints, or any +generic JSON webhook, or an SMTP relay. Treated as black-box; their +uptime is out of this domain's NFR scope. + +### 3.6 Interactions & Sequences + +#### Daily freshness happy path + +**ID**: `cpt-insightspec-seq-mon-happy-path` + +```mermaid +sequenceDiagram + participant Cron as CronWorkflow + participant WT as WorkflowTemplate + participant CH as ClickHouse + participant DBT as dbt + participant P as Inline Parser + participant D as Driver Dispatcher + participant Sink as External Sink + participant T as Trap Detector + + Cron->>WT: spawn (params) + WT->>WT: rm -f target/sources.json + WT->>CH: SELECT name FROM system.databases + CH-->>WT: existing bronze_* list + WT->>DBT: dbt source freshness --select + DBT->>CH: per-source freshness queries + CH-->>DBT: anchors + DBT-->>WT: target/sources.json + WT->>P: python parser + P->>P: classify breaches, build payload + alt driver != "" + P->>D: dispatch(driver, payload) + D->>Sink: HTTP POST / SMTP + Sink-->>D: 200 / 250 + D-->>P: ok + end + P-->>WT: exit code (0 or 1) + WT->>T: freshness-trap-detect.py + T->>CH: trap-mode queries + CH-->>T: row-level evidence + T-->>WT: stderr findings (advisory) + WT-->>Cron: exit = parser exit +``` + +#### Failure path: dbt crash + +**ID**: `cpt-insightspec-seq-mon-dbt-crash` + +```mermaid +sequenceDiagram + participant WT as WorkflowTemplate + participant DBT as dbt + participant P as Inline Parser + + WT->>WT: rm -f target/sources.json + WT->>DBT: dbt source freshness ... + DBT--xWT: crash (no target/sources.json written) + WT->>P: python parser + P->>P: target/sources.json missing → exit 2 + P-->>WT: exit 2 + WT-->>WT: trap detector still runs (advisory) +``` + +#### Failure path: notification delivery error + +**ID**: `cpt-insightspec-seq-mon-delivery-error` + +```mermaid +sequenceDiagram + participant P as Inline Parser + participant D as Driver Dispatcher + participant Sink as External Sink + + P->>D: dispatch(driver, payload) + D->>Sink: HTTP POST + Sink--xD: 500 Internal Server Error (body: "...") + D-->>D: log status + body to stderr + D-->>P: caught (no re-raise) + P-->>P: exit code unchanged (still page-worthy if any error breach) +``` + +### 3.7 Database schemas & tables + +#### `target/sources.json` (dbt artefact) + +This is not a ClickHouse table; it is the dbt-produced JSON artefact +that the parser consumes. Shape (subset): + +| Path | Type | Purpose | +|---|---|---| +| `results[i].unique_id` | string | dbt source unique id (`source.ingestion.bronze_*.`). | +| `results[i].status` | enum | `pass` / `warn` / `error` / `runtime error`. | +| `results[i].max_loaded_at` | string | ISO-8601 of `MAX()` (or `1970-01-01T...` sentinel for empty tables). | +| `results[i].max_loaded_at_time_ago_in_s` | number | Seconds since the anchor (parser converts to hours). | +| `results[i].criteria.{warn_after,error_after}` | object | The active tier values for this source. | + +#### ClickHouse `system.databases` (read-only system table) + +Used by the workflow's selector-narrowing step to enumerate which +`bronze_*` databases actually exist on this deployment. No writes. + +#### ClickHouse `bronze_.` (bronze tables) + +The freshness check reads `MAX()` and `count()` from +each declared bronze source. The trap detector additionally reads +`_airbyte_extracted_at` distribution and the optional +`` column. No writes from this domain. + +### 3.8 Deployment Topology + +#### Argo namespace + +The `WorkflowTemplate`, `CronWorkflow`, and the workflow pods live in +the `argo` namespace. The `insight-toolbox` image carries dbt, +dbt-clickhouse, and the freshness scripts. + +#### Secret access + +Driver Secrets and `clickhouse.passwordSecret` live in the same +namespace as the workflow pods so `secretKeyRef` resolves +namespace-locally. Cross-namespace Secret access is out of scope. + +#### Kind / prod parity + +Local Kind clusters use the same Helm chart + values as production — +the only differences are the per-deployment `cluster` / `tenant` +labels and the driver selection (typically `""` locally, an actual +driver in production). This keeps the dev/prod-parity NFR honest. + +| Helm path | Type | Default | Purpose | +|---|---|---|---| +| `ingestion.freshness.enabled` | bool | `true` | Top-level kill switch. | +| `ingestion.freshness.schedule` | string | `"0 13 * * *"` | Cron — sits past every connector sync window (02:00–11:00 UTC) plus 2 h grace. | +| `ingestion.freshness.dbtSelect` | string | `"source:*"` | Default selector. Narrowed at runtime to deployed bronze databases. | +| `ingestion.freshness.cluster` | string | `""` | Identity label — installation tier. | +| `ingestion.freshness.tenant` | string | `""` | Identity label — customer / workspace. | +| _(thresholds)_ | — | — | Not a Helm value — `warn_after`/`error_after` are literal per source in each connector's `schema.yml`. | +| `ingestion.freshness.notification.driver` | enum | `""` | One of `""`, `webhook`, `zulip`, `slack`, `teams`, `email`. Schema-validated. | +| `ingestion.freshness.notification.webhook.urlSecret.{name,key}` | secretRef | — | Required when driver=webhook. | +| `ingestion.freshness.notification.zulip.urlSecret.{name,key}` | secretRef | — | Required when driver=zulip. | +| `ingestion.freshness.notification.zulip.stream` | string | `""` | Optional override; else inherits from URL query. | +| `ingestion.freshness.notification.zulip.topic` | string | `""` | Optional override. | +| `ingestion.freshness.notification.slack.urlSecret.{name,key}` | secretRef | — | Required when driver=slack. | +| `ingestion.freshness.notification.slack.channel` | string | `""` | Optional channel override. | +| `ingestion.freshness.notification.teams.urlSecret.{name,key}` | secretRef | — | Required when driver=teams. | +| `ingestion.freshness.notification.email.smtp.host` | string | `""` | Required when driver=email. | +| `ingestion.freshness.notification.email.smtp.port` | int | `587` | 587 STARTTLS / 465 SMTPS / 25 plain. | +| `ingestion.freshness.notification.email.smtp.username` | string | `""` | Optional SMTP AUTH user. | +| `ingestion.freshness.notification.email.smtp.passwordSecret.{name,key}` | secretRef | — | SMTP AUTH password (rendered only when set). | +| `ingestion.freshness.notification.email.smtp.starttls` | bool | `true` | Ignored when port=465. | +| `ingestion.freshness.notification.email.from` | string | `""` | Required when driver=email. | +| `ingestion.freshness.notification.email.to` | string | `""` | Required when driver=email — comma-separated. | +| `ingestion.freshness.notification.email.subjectPrefix` | string | `"[ingestion-freshness]"` | Prefix prepended to `summary`. | + +## 4. Additional context + +### 4.1 Why two report tiers (`report`, `report_extended`) instead of one + +Microsoft Graph reports document a 24–48 h publish lag — sliding the +warn to 48 h is just "the upper edge of stated normal". Slack +admin.analytics is observed and documented to lag ~3 days, with +stretches up to 5 days during Slack maintenance — applying the +report-tier (96 h error) to Slack would page on every Slack +maintenance window. They needed different bands, but lumping every +3-day vendor into "event" would lose the semantic signal that `event` +is for natural quiet days, not for vendor lag. + +### 4.2 Why `_airbyte_extracted_at` for streaming and a business date for re-emit + +Streaming connectors land rows in bronze approximately when they +happen — `_airbyte_extracted_at` tracks reality. Report-style +connectors re-fetch a fixed window every run: even when the upstream +stops publishing, Airbyte still writes "fresh" rows for older business +days, so `_airbyte_extracted_at` keeps advancing forever. The +`feature-bronze-freshness-sla/FEATURE.md` example (M365 9 h fresh +extracted-at vs 96 h-stale `reportRefreshDate` on 2026-05-04) is the +canonical evidence. Anchoring on the business-date column flips the +verdict to ERROR, which is correct. + +### 4.3 Why per-driver subblocks instead of a flat `notification.url` + +Different drivers need different *non-credential* settings (Zulip +stream/topic, Slack channel, SMTP host/port/from/to/subjectPrefix). A +flat `notification.url` would force these into either the URL itself +(Zulip's `?stream=...` works; Slack's channel override does not always) +or into an out-of-band lookup table. Per-driver subblocks let each +driver own its shape; the dispatcher just reads the env vars it cares +about. Adding a sixth driver is one Helm subblock + one match arm + +one `secretKeyRef` branch — no other Helm surface changes (workflow +template line 380–382, `values.yaml:195-201`). + +### 4.4 Why the Zulip dispatcher silently rewrites the endpoint + +Zulip exposes two incoming-webhook integrations under the same UI +("Incoming webhook"): `/api/v1/external/json` (intended for debugging, +dumps body as JSON code block) and `/api/v1/external/slack` +(Slack-compatible, renders mrkdwn). The UI does not warn that the +former is the wrong integration for routine notifications. Rather +than require operators to know the Zulip-internal distinction, the +dispatcher rewrites the path on the way out (workflow template line +457). This is documented in the chart values comment +(`values.yaml:230-239`) and in the dispatcher's docstring (lines +445–478). + +### 4.5 Why narrow the selector at runtime + +`source:*` against a partial deployment (e.g. a tenant who hasn't +deployed OpenAI) would emit one ERROR per missing source on every run, +flooding on-call with noise about something they deliberately didn't +deploy. Narrowing to `bronze_*` databases that exist in +`system.databases` (workflow template lines 262–297) keeps the check +scoped to "what we collect", and explicitly leaves "what should be +collected" to the deployment-health workstream +([#272](https://github.com/cyberfabric/insight/issues/272)). + +### 4.6 Why the trap detector is advisory + +The freshness check is the page-worthy signal — its threshold tiers are +documented per source, its semantics are deterministic, its output is +the canonical payload. The trap detector is heuristic — its thresholds +(`SUSPECT_PCT_RECENT = 95.0`, `SUSPECT_MAX_DISTINCT_DAYS = 2`, +`MIN_ROWS = 100`, `RECENT_WINDOW_HOURS = 30`) are hardcoded constants +in [`freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py) +rather than Helm-tunable values, so retuning is a code change with a +PR review attached. False positives on a heuristic that pages on-call +would erode trust in the whole monitoring domain. So the trap detector +logs, the freshness parser owns the verdict (workflow template lines +575–579). + +### 4.7 Why every `freshness: null` opt-out needs a rationale + +Bare `freshness: null` is too easy to leave in by accident. Once it +lands, the table is invisible to monitoring forever, with no comment +explaining why. The convention is a one-line rationale comment beside +each opt-out, kept grep-friendly: `grep -A1 'freshness: null'` shows +what every opt-out is for. Opt-out is reserved for incremental streams +that legitimately go quiet — a full-refresh roster/lookup keeps +`_airbyte_extracted_at` as a sync-liveness signal instead. + +## 5. Traceability + +- Feature spec: [`feature-bronze-freshness-sla/FEATURE.md`](feature-bronze-freshness-sla/FEATURE.md) +- Operator runbook: [`src/ingestion/MONITORING.md`](../../../../src/ingestion/MONITORING.md) +- Implementation is split across two PRs into `constructorfabric/insight`: + - **Runtime / alerting** (this module — branch `feat/ingestion-freshness-runtime`, closes #949, refs #1321): + - Workflow + parser: [`charts/insight/templates/ingestion/dbt-source-freshness.yaml`](../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml) + - Helm surface: [`charts/insight/values.yaml`](../../../../charts/insight/values.yaml) + - Schema: [`charts/insight/values.schema.json`](../../../../charts/insight/values.schema.json) + - Trap detector: [`src/ingestion/scripts/freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py) + - **Per-source declarations** (EPIC #1321 / #1322): PR #1346 (base freshness blocks) + PR SharedQA/insight#1 (business-date anchors + tiers) — `src/ingestion/connectors/*/*/dbt/schema.yml`. +- Future work cross-references: + - Deployment health — [issue #272](https://github.com/constructorfabric/insight/issues/272). + - Volume baseline / source-vs-bronze attribution / ownership — "Open work" sections of [`MONITORING.md`](../../../../src/ingestion/MONITORING.md). diff --git a/docs/domain/ingestion-monitoring/specs/PRD.md b/docs/domain/ingestion-monitoring/specs/PRD.md new file mode 100644 index 000000000..feeeea154 --- /dev/null +++ b/docs/domain/ingestion-monitoring/specs/PRD.md @@ -0,0 +1,730 @@ +--- +status: proposed +date: 2026-05-04 +--- + +# PRD — Ingestion Monitoring + + + +- [1. Overview](#1-overview) + - [1.1 Purpose](#11-purpose) + - [1.2 Background / Problem Statement](#12-background--problem-statement) + - [1.3 Goals (Business Outcomes)](#13-goals-business-outcomes) + - [1.4 Glossary](#14-glossary) +- [2. Actors](#2-actors) + - [2.1 Human Actors](#21-human-actors) + - [2.2 System Actors](#22-system-actors) +- [3. Operational Concept & Environment](#3-operational-concept--environment) + - [3.1 Module-Specific Environment Constraints](#31-module-specific-environment-constraints) + - [3.2 Expected Scale](#32-expected-scale) +- [4. Scope](#4-scope) + - [4.1 In Scope](#41-in-scope) + - [4.2 Out of Scope](#42-out-of-scope) +- [5. Functional Requirements](#5-functional-requirements) + - [5.1 Freshness Check](#51-freshness-check) + - [5.2 Threshold Tiers](#52-threshold-tiers) + - [5.3 Opt-Out Hygiene](#53-opt-out-hygiene) + - [5.4 Trap Detection](#54-trap-detection) + - [5.5 Notification Delivery](#55-notification-delivery) + - [5.6 Identity Labels](#56-identity-labels) +- [6. Non-Functional Requirements](#6-non-functional-requirements) + - [6.1 NFR Inclusions](#61-nfr-inclusions) + - [6.2 NFR Exclusions](#62-nfr-exclusions) +- [7. Public Library Interfaces](#7-public-library-interfaces) + - [7.1 Public API Surface](#71-public-api-surface) + - [7.2 External Integration Contracts](#72-external-integration-contracts) +- [8. Use Cases](#8-use-cases) +- [9. Acceptance Criteria](#9-acceptance-criteria) +- [10. Dependencies](#10-dependencies) +- [11. Assumptions](#11-assumptions) +- [12. Risks](#12-risks) + + + +## 1. Overview + +### 1.1 Purpose + +Catch ingestion-layer breaches — bronze tables that have stopped receiving +fresh rows, or whose freshness anchor is masking a re-emit pattern — before +the silence propagates to silver/gold metrics and to the dashboard. The +domain owns observability *of what we collect*; it deliberately does not +own deployment health, volume baselines, or vendor-job attribution (see +[§4.2](#42-out-of-scope)). + +### 1.2 Background / Problem Statement + +The ingestion pipeline previously had no signal for "data is missing". +Connectors that silently stopped emitting rows (expired tokens, upstream +API outages, Argo CronWorkflow failures, syncs that ran but produced 0 +rows) only became visible when a downstream metric went flat days later. + +Two classes of silent failure motivated the domain: + +1. **Sync stops, anchor goes stale.** The straightforward case — Airbyte + connector dies, no new rows land, `MAX(_airbyte_extracted_at)` ages + out of the warn window. +2. **Sync runs, anchor stays green, upstream has stopped publishing.** + The trap case. Some connectors (Microsoft Graph reports, Slack + admin.analytics, Cursor daily_usage, Confluence page versions) + re-fetch a fixed window every run. Even when the upstream advances + no new business days, Airbyte keeps writing rows for older days, so + `_airbyte_extracted_at` looks fresh forever. The local CH dump on + 2026-05-04 had M365 9 hours fresh on `_airbyte_extracted_at` but + `reportRefreshDate` 96 hours behind reality (see + [feature-bronze-freshness-sla/FEATURE.md](feature-bronze-freshness-sla/FEATURE.md)). + +### 1.3 Goals (Business Outcomes) + +- Every Airbyte-managed `bronze_*` source has a verifiable + PASS/WARN/ERROR freshness verdict produced once per day. +- Adding a new connector gains monitoring without per-pipeline + plumbing — a `freshness:` block in the connector's `schema.yml` is + enough. +- Operators receive breaches via a notification driver of their choice + (webhook, Zulip, Slack, Teams, email) — without exposing the channel's + credentials in rendered manifests or in the Argo UI. +- Connector authors who pick the wrong anchor are caught either at PR + time (CI lint) or at runtime (trap detector), not by silent green + dashboards. + +### 1.4 Glossary + +| Term | Meaning | +|---|---| +| Bronze | The first ClickHouse-side schema written by Airbyte: `bronze_.`. | +| Freshness anchor | The SQL expression in `loaded_at_field` against which `dbt source freshness` measures lag. | +| Streaming connector | Connector whose Airbyte cursor follows business time; rows land in bronze approximately when they happen. Anchor = `_airbyte_extracted_at`. | +| Report-style connector | Connector that re-fetches a fixed window every run (Microsoft Graph reports, Slack admin.analytics). Anchor = a business-date column. | +| Event-style connector | Streaming connector with legitimate quiet days (Confluence edits, Zoom meetings). Uses a wider tier. | +| SLA tier | A pair `(warn_after, error_after)` shared by connectors with similar cadence: `default`, `event`, `report`, `report_extended`. | +| Trap | A bronze table whose `_airbyte_extracted_at` looks fresh while the upstream has stopped publishing — the freshness check passes for the wrong reason. | +| Driver | A notification dispatch backend selected via `ingestion.freshness.notification.driver`: `webhook` / `zulip` / `slack` / `teams` / `email`. | + +## 2. Actors + +### 2.1 Human Actors + +#### Ingestion On-Call + +Triages freshness breaches. Reads Argo workflow status / notification +payloads. Acts on `error` and `runtime error` runs. + +#### Connector Owner + +Owns a specific `bronze_*` source. Receives an issue handed off by +on-call when their connector breaches. Fixes the connector or revisits +the SLA tier. + +#### Tenant On-Call (post-MVP) + +Per-deployment triage when one bot fans out to multiple installations. +Reads the notification payload's `cluster` / `tenant` labels. Same +triage as ingestion on-call, scoped to one deployment. + +Rotation / `CODEOWNERS` for `src/ingestion/connectors/` is **not yet +assigned** — see +[`src/ingestion/MONITORING.md` §"Rotation / ownership — not assigned"](../../../../src/ingestion/MONITORING.md). + +### 2.2 System Actors + +#### `dbt-source-freshness` WorkflowTemplate (Argo) + +Stateless template — runs `dbt source freshness`, parses +`target/sources.json`, dispatches to the active driver. Defined in +[`charts/insight/templates/ingestion/dbt-source-freshness.yaml`](../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml). + +#### `dbt-source-freshness-check` CronWorkflow (Argo) + +Schedules the template once a day at 13:00 UTC; carries the +per-deployment Helm config into the template's parameters. Same file. + +#### Inline freshness parser (Python) + +Reads `target/sources.json`, classifies breaches, calls the driver +dispatcher, sets the workflow exit code. Inline in the workflow +template, lines 317–562. + +#### Trap detector + +[`src/ingestion/scripts/freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py) +— advisory script that spots full-reemit and incremental-topup patterns +the dbt freshness check cannot see. Runs after the parser. + +#### Coverage gate (QA-owned, out of this module's scope) + +Per-source declaration coverage is measured by the QA-owned +`scripts/ci/dbt_coverage.py` gate (EPIC #1321 / #1322), which counts every +`bronze_*` source that declares `loaded_at_field` + `freshness`. This module +relies on that gate rather than shipping its own connector-schema lint. + +## 3. Operational Concept & Environment + +### 3.1 Module-Specific Environment Constraints + +- Production: Argo Workflows + ClickHouse in a Kubernetes cluster; the + `dbt-source-freshness-check` CronWorkflow runs once per day at 13:00 + UTC, sitting past every connector's sync window (02:00–11:00 UTC) plus + a 2 h grace. +- Local development: Kind K8s cluster (`insight`) with the same Helm + values; ClickHouse reachable on `host.docker.internal:8123` for the + trap detector. +- The `insight-toolbox` image must carry `dbt`, `dbt-clickhouse`, and + the freshness trap detector (`freshness-trap-detect.py`). +- Notification driver credentials must live in Kubernetes Secrets + referenced by `secretKeyRef`; the rendered `WorkflowTemplate` / + `CronWorkflow` YAML must never carry the raw URL or password. + +### 3.2 Expected Scale + +| Dimension | Current | Projected | +|---|---|---| +| Bronze sources monitored | 12 (one per declared `bronze_*` schema) | 50+ | +| Bronze tables monitored | ~80 (82 entries across 12 sources today) | 200+ | +| Notification drivers supported | 5 | 5 (stable) | +| SLA tiers | 4 | 4 (stable) | +| Workflow runtime (warm CH) | ~2 min | < 5 min | +| `activeDeadlineSeconds` budget | 1200 s | 1200 s | + +## 4. Scope + +### 4.1 In Scope + +- **Bronze freshness SLA** — the `dbt source freshness` check against + `_airbyte_extracted_at` (incremental) or a business-date column + (windowed), with four threshold tiers (`default` 36/72 h, `event` + 96/168 h, `report` 72/120 h, `report_extended` 120/168 h). Source: + [`feature-bronze-freshness-sla/FEATURE.md`](feature-bronze-freshness-sla/FEATURE.md). + The per-source declarations themselves are owned by the connector-schema + work (EPIC #1321 / #1322 — PR #1346 + PR SharedQA/insight#1), not this module. +- **Runtime trap detector** — heuristic full-reemit detection (no + config) plus opt-in `meta.bronze_business_date_col` divergence check; + advisory only, never affects workflow exit code. Source: + [`freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py). +- **Multi-driver notification fan-out** — five drivers + (webhook / zulip / slack / teams / email) selectable via Helm; URL / + SMTP password sourced from a Kubernetes Secret so the credential + never appears in rendered manifests or Argo UI. Source: workflow + template lines 161–194 and 412–540. +- **Identity labels in payload** — `cluster` and `tenant` carried into + every payload's summary prefix and JSON body so a shared bot can + route between installations. + +### 4.2 Out of Scope + +- **Deployment health** — "is a connector that should be deployed + actually deployed?" is a separate concern (Helm probes / sync + workflow status). Tracked as + [issue #272](https://github.com/cyberfabric/insight/issues/272). The + freshness workflow narrows its selector to bronze databases that + exist (workflow template lines 254–297) precisely so it does *not* + alert on connectors a tenant hasn't deployed. +- **Volume baseline anomaly detection** — "API returned 50 rows + instead of 5000". Listed under "Open work" in + [`MONITORING.md`](../../../../src/ingestion/MONITORING.md). +- **Source-vs-bronze attribution** — distinguishing "Airbyte sync + failed" from "Airbyte sync ran but pulled 0 rows" requires polling + Airbyte's Jobs API; listed under "Open work" in + [`MONITORING.md`](../../../../src/ingestion/MONITORING.md). +- **Notification rotation / `CODEOWNERS`** — see + [`MONITORING.md` §"Rotation / ownership — not assigned"](../../../../src/ingestion/MONITORING.md). + +## 5. Functional Requirements + +### 5.1 Freshness Check + +#### Daily Argo CronWorkflow + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-daily-cronworkflow` + +The system **MUST** schedule a single Argo `CronWorkflow` +(`dbt-source-freshness-check`) that runs once per day on the schedule +supplied by `ingestion.freshness.schedule` (default `0 13 * * *`, +charts/insight/values.yaml:151). + +#### Selector Narrowing + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-selector-narrowing` + +The workflow **MUST** narrow the dbt selector from `source:*` to the +`bronze_*` databases that actually exist in `system.databases` (workflow +template lines 262–297) — connectors a tenant did not deploy never +produce a phantom ERROR. + +#### Stale Report Cleanup + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-stale-report-cleanup` + +The workflow **MUST** remove a stale `target/sources.json` before each +run (workflow template line 252) so a mid-run dbt crash maps to exit +code 2 rather than misclassifying using leftover data. + +#### Exit Code Re-Derivation + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-exit-rederivation` + +`dbt source freshness` is invoked with `--select ` and its +exit code **MUST** be swallowed (line 305–307). The Python parser +re-derives the workflow outcome from `target/sources.json`. + +### 5.2 Threshold Tiers + +#### Single Source of Truth + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-thresholds-sot` + +Each source's thresholds **MUST** be a single source of truth: literal +`warn_after`/`error_after` values declared in that source's `schema.yml`. +The CronWorkflow runs `dbt source freshness --select source:*` and reads +exactly what each source declares — no Helm/env-var threshold layer, so there +is no second place a value can drift. + +#### Four Tiers + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-four-tiers` + +The system **MUST** support exactly four tiers, each with documented +rationale (applied as literal values per source): + +- `default` (36/72 h) — incremental connectors on daily cron cadence. +- `event` (96/168 h) — natural-quiet-day connectors (Confluence + edits, Zoom meetings) where a zero-row weekend is normal. +- `report` (72/120 h) — windowed vendor analytics with documented 24–48 h + publish lag (Microsoft Graph reports, ChatGPT/Claude Team baseline). +- `report_extended` (120/168 h) — windowed vendor analytics with ~3-day + baseline lag (Slack admin.analytics typical). + +#### Tier Assignment + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-tier-assignment` + +Tier assignment **MUST** be per-connector and live as literal values in the +connector's `schema.yml`. Re-tiering is an engineering change committed to the +source declaration — there is no runtime tuning knob. + +### 5.3 Opt-Out Hygiene + +#### Per-Table Opt-Out + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-optout-pertable` + +Slow-moving roster / catalog tables (e.g. `cursor_members`, +`wiki_spaces`) **MUST** be allowed to opt out via `freshness: null` +per-table in `schema.yml`. + +#### Mandatory Rationale + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-optout-rationale` + +Every `freshness: null` opt-out **MUST** carry a one-line rationale comment +beside it so the audit surface stays grep-able, and is reserved for +*incremental* streams that legitimately go quiet (a full-refresh roster keeps +`_airbyte_extracted_at` as a sync-liveness signal instead of opting out). + +#### Mandatory Anchor + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-mandatory-anchor` + +The lint **MUST** reject any `bronze_*` source with no reachable +`loaded_at_field` (source-level, per-table, or explicit opt-out). + +### 5.4 Trap Detection + +#### Post-Parser Execution + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-trap-post-parser` + +After the freshness parser exits, the workflow **MUST** run +[`freshness-trap-detect.py`](../../../../src/ingestion/scripts/freshness-trap-detect.py) +(workflow template lines 564–579). + +#### Two Detection Modes + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-trap-two-modes` + +The detector **MUST** support two modes: + +- **Full re-emit (heuristic, no config)** — fires when ≥ 95 % of + rows have `_airbyte_extracted_at` within the last 30 h *and* + `_airbyte_extracted_at` covers ≤ 2 distinct calendar days *and* + the table has ≥ 100 rows (script constants + `SUSPECT_PCT_RECENT`, `SUSPECT_MAX_DISTINCT_DAYS`, + `RECENT_WINDOW_HOURS`, `MIN_ROWS`). +- **Business-date divergence (opt-in)** — sources that declare + `meta.bronze_business_date_col: ` get + `MAX()` compared with `MAX(_airbyte_extracted_at)`; a + ≥ 24 h gap is flagged. + +#### Skip Annotation + +- [ ] `p2` - **ID**: `cpt-insightspec-fr-mon-trap-skip-annotation` + +A source **MUST** be allowed to opt out of mode 1 via +`meta.bronze_freshness_trap_check: skip` at source or table level. + +#### Advisory-Only Verdict + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-trap-advisory` + +Findings **MUST** be advisory: `trap_exit=1` only logs a notice. The +workflow's page/no-page decision remains owned by the freshness +parser (workflow template lines 575–579). + +### 5.5 Notification Delivery + +#### Driver Selection + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-driver-selection` + +`ingestion.freshness.notification.driver` **MUST** select one of `""`, +`webhook`, `zulip`, `slack`, `teams`, `email` +(charts/insight/values.yaml:220, schema enum at +charts/insight/values.schema.json:37). + +#### Credential Isolation + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-credential-isolation` + +Each URL-driven driver **MUST** read its credential from a +`secretKeyRef` branched in the workflow template (lines 161–194). The +URL is bound to `NOTIFICATION_URL` only when the driver is configured; +email's SMTP password binds to `NOTIFICATION_EMAIL_SMTP_PASSWORD`. +Plain settings (Zulip stream, Slack channel, SMTP host/port) flow +through normal parameters because they are not credentials. + +#### Dispatch Implementation + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-dispatch-impl` + +Driver dispatch **MUST** happen in the inline parser via the +`dispatchers` dict (workflow template lines 533–539). + +#### Delivery Failure Isolation + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-delivery-failure-isolation` + +Notification failures **MUST** be caught and logged (lines 550–554) — +they never change the workflow's primary exit code (the breach signal +is more important than the delivery success). + +### 5.6 Identity Labels + +#### Helm Plumbing + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-identity-helm` + +`ingestion.freshness.cluster` and `.tenant` (values.yaml:193–194) +**MUST** flow into the parser as `CLUSTER` / `TENANT` env vars. + +#### Summary Prefix + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-identity-summary` + +The summary line **MUST** carry every set label as a +`[cluster=…, tenant=…] ` prefix (workflow template lines 393–397); +empty labels drop out so single-deployment installs see plain +`N bronze source(s) breaching ...`. + +#### Payload Routing Fields + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-mon-identity-payload` + +The webhook payload **MUST** include both `cluster` and `tenant` as +raw fields so receivers can route by either dimension. + +## 6. Non-Functional Requirements + +### 6.1 NFR Inclusions + +#### Idempotency + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-idempotency` + +Re-running the workflow on the same data **MUST** produce the same exit +code and the same payload (no stateful side-effects, +`target/sources.json` is purged before each run). + +#### Daily Cadence + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-daily-cadence` + +The default cadence **MUST** be daily; the cron schedule is Helm-tunable +but the parser is stateless and can run ad-hoc. + +#### Credential Isolation + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-credential-isolation` + +Webhook URL / SMTP password **MUST** be pulled from k8s Secrets via +`secretKeyRef`; the rendered `WorkflowTemplate` / `CronWorkflow` YAML +and the Argo UI never see the raw value. + +#### Deterministic Exit Codes + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-exit-codes` + +The parser **MUST** emit deterministic exit codes: `0` no breach or +warn-only; `1` at least one `error` / `runtime error`; `2` +`target/sources.json` missing (dbt crashed). + +#### CI-Gated Schema + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-ci-gated` + +Per-source freshness declaration coverage **MUST** be gated in CI by the +QA-owned `scripts/ci/dbt_coverage.py` (EPIC #1321 / #1322) — this module does +not own a connector-schema lint of its own. + +#### Advisory-Fail Mode for Traps + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-trap-advisory-mode` + +Trap detector findings **MUST NOT** override the freshness parser's +verdict (page-worthy stays page-worthy; warn-only stays warn-only). + +#### Activation Deadline + +- [ ] `p2` - **ID**: `cpt-insightspec-nfr-mon-activation-deadline` + +The workflow **MUST** carry `activeDeadlineSeconds: 1200` (workflow +template line 110); current connector set finishes in ~2 min on a +warm CH. + +#### Empty-Table Sentinel + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-mon-empty-sentinel` + +A source with zero rows yields `MAX(...) = NULL → 1970-01-01T…`; the +parser **MUST** flag it explicitly as `(table is empty — no rows ever +ingested)` rather than reporting ~500 000 h of lag (workflow template +lines 348–360). + +### 6.2 NFR Exclusions + +- ClickHouse cluster performance tuning (infrastructure concern). +- Argo Workflows controller HA (infrastructure concern). +- Notification receiver SLAs (Zulip / Slack / Teams uptime is out of + this domain's control). +- Vendor-side publish-lag changes (Microsoft Graph / Slack + admin.analytics window changes are tracked in `MONITORING.md`, not + enforced as NFR thresholds here). + +## 7. Public Library Interfaces + +### 7.1 Public API Surface + +Not applicable — this domain has no public library API. The monitoring +surface is consumed via Helm values + Argo UI + dbt, not via a Rust / +Python / TypeScript library exposed to other domains. See +[DESIGN §3.3](DESIGN.md#33-api-contracts) for the workflow / dbt / +notification protocol contracts. + +### 7.2 External Integration Contracts + +#### Webhook Driver Contract + +Generic JSON POST to `NOTIFICATION_URL`; canonical payload shape is +documented in [DESIGN §3.4 / §3.6](DESIGN.md#34-driver-contracts). +Carries `topic`, `cluster`, `tenant`, `summary`, `breaches[]`. + +#### Zulip Driver Contract + +POST to a Zulip incoming-webhook URL. The dispatcher rewrites +`/api/v1/external/json` to `/api/v1/external/slack` so operators who +pasted the JSON URL still get formatted output. Form-urlencoded body +with Slack mrkdwn. See [DESIGN §3.4.2](DESIGN.md#342-zulip). + +#### Slack Driver Contract + +POST to a Slack incoming-webhook URL with `application/json` +`{"text": ""}`, optionally with `channel` override. See +[DESIGN §3.4.3](DESIGN.md#343-slack). + +#### Teams Driver Contract + +POST to a Microsoft Teams incoming webhook with `application/json` +`MessageCard`. `themeColor` is `FF0000` (red) when any breach is +page-worthy, `FFA500` (orange) otherwise. See +[DESIGN §3.4.4](DESIGN.md#344-teams). + +#### Email Driver Contract + +`smtplib`-based dispatch (no extra deps). `SMTP_SSL` when port=465, +otherwise plain `SMTP` with optional `STARTTLS`. Required Helm values: +`notification.email.smtp.host`, `notification.email.from`, +`notification.email.to`. See [DESIGN §3.4.5](DESIGN.md#345-email). + +## 8. Use Cases + +#### Use Case 1: Daily Freshness Check + +- [ ] `p1` - **ID**: `cpt-insightspec-usecase-mon-daily-check` + +**Actors**: `dbt-source-freshness-check` CronWorkflow, +`dbt-source-freshness` WorkflowTemplate, ClickHouse, ingestion on-call. + +**Preconditions**: Helm values `ingestion.freshness.enabled: true`, +the `bronze_*` databases exist in `system.databases`, the dbt project +declares `loaded_at_field` for every source. + +**Main Flow**: + +1. Argo triggers `dbt-source-freshness-check` at the configured cron. +2. The workflow narrows `source:*` to bronze databases that exist. +3. `dbt source freshness` runs against ClickHouse and writes + `target/sources.json`. +4. The inline Python parser classifies every source as PASS / WARN / + ERROR / RUNTIME ERROR. +5. The trap detector runs and logs any heuristic findings without + changing the verdict. +6. The driver dispatcher posts to the configured channel whenever the + `breaches` list is non-empty — including `warn`-only runs, so + on-call sees latency creeping up before it crosses into + page-worthy territory. +7. The workflow exits with `1` only when at least one breach is + `error` or `runtime error`; `warn`-only runs exit `0` so the + payload is informational, not paging. + +**Postconditions**: A canonical breach record exists in the workflow +log; if a driver is configured and any breach was page-worthy, the +external sink received the payload. + +#### Use Case 2: Triage a Paging Breach + +- [ ] `p1` - **ID**: `cpt-insightspec-usecase-mon-triage-breach` + +**Actors**: ingestion on-call, connector owner. + +**Preconditions**: a notification carrying `[cluster=…, tenant=…] N +bronze source(s) breaching freshness SLA` has fired. + +**Main Flow**: + +1. On-call opens the payload, identifies the breaching `source` and + `age_hours`. +2. On-call cross-references the source's tier in + [DESIGN §3.5](DESIGN.md#35-sla-tier-mapping) to confirm it was + page-worthy (not warn-only). +3. On-call inspects the connector's last sync in Argo / Airbyte UI to + distinguish "sync stopped" from "sync ran but produced no rows". +4. On-call hands off to the connector owner with the payload as + evidence. +5. Connector owner fixes the connector or revisits the SLA tier in + `schema.yml`. + +**Postconditions**: Either the connector is healthy on the next daily +run, or the tier has been changed (with rationale in commit message). + +#### Use Case 3: Add a New Bronze Source + +- [ ] `p1` - **ID**: `cpt-insightspec-usecase-mon-add-bronze-source` + +**Actors**: connector author, CI lint. + +**Preconditions**: a new connector under +`src/ingestion/connectors///` with a `dbt/schema.yml`. + +**Main Flow**: + +1. Author declares `loaded_at_field` at source or table level + (`_airbyte_extracted_at` for incremental, a + `parseDateTimeBestEffortOrNull()` expression for windowed). +2. Author sets the literal tier `warn_after`/`error_after` on the source's + `freshness:` block (`default`/`report`/`report_extended`/`event`). +3. Author opens a PR. The `dbt_coverage.py` gate (EPIC #1321) confirms the + source is declared; the windowed-vs-incremental call is confirmed against + live data (`ext_age` vs `biz_age`), not by a lint. +4. PR merges; the next daily freshness run includes the new source. + +**Postconditions**: The new source is monitored on the same SLA tier +as comparable connectors with no further plumbing. + +#### Use Case 4: Switch Notification Driver in a Tenant Overlay + +- [ ] `p2` - **ID**: `cpt-insightspec-usecase-mon-switch-driver` + +**Actors**: platform engineer. + +**Preconditions**: a Helm values overlay for the target tenant; a +Kubernetes Secret carrying the new driver's URL or SMTP password. + +**Main Flow**: + +1. Engineer creates / updates the Secret in the same namespace as the + workflow. +2. Engineer sets `ingestion.freshness.notification.driver: ` and + the driver's `urlSecret.{name,key}` (or `email.smtp.*`) in the + overlay. +3. `helm upgrade` re-renders the `WorkflowTemplate`; the `secretKeyRef` + binding switches to the new driver's branch. +4. The next scheduled run dispatches to the new channel; the previous + driver's secret reference is no longer rendered. + +**Postconditions**: The tenant's freshness payload now arrives in the +new channel; rendered manifests and Argo UI carry the `secretKeyRef`, +not the credential. + +## 9. Acceptance Criteria + +- [ ] Every `bronze_*` source under `src/ingestion/connectors/*/*/dbt/schema.yml` declares `loaded_at_field` + a literal `freshness:` block (source / table / opt-out) — counted by the QA-owned `dbt_coverage.py` gate (EPIC #1321). +- [ ] Every `freshness: null` opt-out carries a one-line rationale comment and is an incremental (not full-refresh) stream. +- [ ] Windowed connectors anchor on a business-date column, not `_airbyte_extracted_at` — confirmed by `ext_age` vs `biz_age` on live data where rows exist. +- [ ] `dbt source freshness --select 'source:bronze_*'` from a clean checkout produces `target/sources.json` with one result per declared (source, table) pair, no `runtime error` due to missing `loaded_at_field`. +- [ ] Each tier produces the expected verdict on its representative connectors: + - `default` — Bitbucket, Jira, BambooHR, Cursor (non-daily), GitHub, OpenAI, Claude (PASS within 30 h of last sync). + - `event` — Confluence `wiki_page_versions`, Zoom `meetings`/`participants` (PASS within 72 h, weekend-tolerant). + - `report` — M365 `*_activity` (PASS within 48 h of `reportRefreshDate`). + - `report_extended` — Slack `users_details` (PASS within 72 h of `date`). +- [ ] A stale-data fixture (truncate or backdate `_airbyte_extracted_at` on one source) produces parser exit code 1, a log line naming the source / `max_loaded_at` / `age_hours`, and a notification payload (when a driver is configured). +- [ ] The notification payload always carries both `cluster` and `tenant` keys (empty string when unset), matching the canonical webhook contract documented in [`MONITORING.md`](../../../../src/ingestion/MONITORING.md). +- [ ] The configured driver's credential **does not appear** in `kubectl get workflowtemplate dbt-source-freshness -o yaml` or in `argo get` output — only the `secretKeyRef` does. +- [ ] An empty `bronze_*` table is reported as `(table is empty)` instead of `~500000h` lag (workflow template lines 348–360). +- [ ] Synthetic Confluence-style fixture (full table re-emit within 24 h, 1 distinct extract day, ≥ 100 rows) is flagged by the trap detector with `kind=full-reemit` (script lines 188–200). +- [ ] Synthetic incremental-topup fixture (`MAX(_airbyte_extracted_at)` fresh, `MAX()` ≥ 24 h behind) is flagged with `kind=incremental-topup` (script lines 202–227). +- [ ] `values.schema.json` rejects an unknown driver name (only `""`, `webhook`, `zulip`, `slack`, `teams`, `email` are accepted — schema line 37). + +## 10. Dependencies + +- **Argo Workflows** controller in the `argo` namespace — provides + `WorkflowTemplate` / `CronWorkflow` reconciliation and the + `failedJobsHistoryLimit` retention used as the no-fan-out fallback. +- **dbt-clickhouse** adapter inside the `insight-toolbox` image — + provides `dbt source freshness` and `target/sources.json`. The + adapter does not support metadata-based freshness, so a missing + `loaded_at_field` produces `runtime error` rather than silent skip. +- **ClickHouse** — readable `system.databases` (for selector + narrowing) and HTTP interface on port 8123 (used by the trap + detector). +- **Kubernetes Secret access** — `secretKeyRef` for both + `clickhouse.passwordSecret` and the active driver's credential + Secret. +- **Helm umbrella chart** — `ingestion.freshness.*` block in + `charts/insight/values.yaml`, validated by + `charts/insight/values.schema.json`. + +## 11. Assumptions + +- Every Airbyte-managed `bronze_*` source carries a usable + `_airbyte_extracted_at` column or a business-date column suitable as + a freshness anchor. +- Vendor publish-lag windows (Microsoft Graph 24–48 h; Slack + admin.analytics ~3 d) remain within the configured tier bounds; if a + vendor changes its cadence, the affected source is re-tiered via a + schema.yml edit, not by widening the global tier. +- Operators have access to a Kubernetes Secret store and accept + `secretKeyRef` as the credential carrier — no in-tree plaintext + fallback is provided. +- Argo Workflows is healthy enough to schedule the daily run; if the + controller is down, monitoring goes silent (tracked under + "Open work" in `MONITORING.md`). + +## 12. Risks + +| Risk | Impact | Mitigation | +|---|---|---| +| Trap heuristic false positive (full-reemit mode 1) on a legitimate burst-load source | On-call sees a noisy advisory line | Findings are advisory only; opt out via `meta.bronze_freshness_trap_check: skip`. | +| Driver receiver outage hides real breaches | Page-worthy breach silently dropped | Notification failure does not change the workflow's exit code; Argo's `failedJobsHistoryLimit` keeps the breach visible in the UI. | +| Connector author picks the wrong anchor (uses `_airbyte_extracted_at` for a report-style source) | Trap-class silent green | CI lint requires an anchor; trap detector flags re-emit at runtime; PRD §1.2 documents the failure mode. | +| Vendor changes its publish window | Tier bounds become wrong; pages too eagerly or too late | Re-tiering is a schema.yml edit; tier values are Helm-tunable as a safety valve. | +| Multi-tenant fan-out from one bot mixes installations | On-call cannot route | `cluster` / `tenant` labels are mandatory in payload; receivers route by either dimension. | diff --git a/docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md b/docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md new file mode 100644 index 000000000..4d2e44dbb --- /dev/null +++ b/docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md @@ -0,0 +1,502 @@ +--- +status: proposed +date: 2026-05-04 +--- + +# Feature: Bronze Freshness SLA + + + +- [1. Feature Context](#1-feature-context) + - [1.1 Overview](#11-overview) + - [1.2 Purpose](#12-purpose) + - [1.3 Actors](#13-actors) + - [1.4 References](#14-references) +- [2. Actor Flows (CDSL)](#2-actor-flows-cdsl) + - [Trigger Freshness Check on Schedule](#trigger-freshness-check-on-schedule) + - [Triage a Page-Worthy Breach](#triage-a-page-worthy-breach) + - [Wire a New Connector into Freshness](#wire-a-new-connector-into-freshness) + - [Switch Notification Driver in a Tenant Overlay](#switch-notification-driver-in-a-tenant-overlay) + - [Override SLA Threshold for One Connector](#override-sla-threshold-for-one-connector) +- [3. Processes / Business Logic (CDSL)](#3-processes--business-logic-cdsl) + - [Resolve dbt Selector to Existing Bronze Schemas](#resolve-dbt-selector-to-existing-bronze-schemas) + - [Classify Source Status (PASS / WARN / ERROR / RUNTIME ERROR)](#classify-source-status-pass--warn--error--runtime-error) + - [Render Notification Body per Driver](#render-notification-body-per-driver) + - [Detect Re-Emit Trap](#detect-re-emit-trap) + - [Choose SLA Tier per Source](#choose-sla-tier-per-source) +- [4. States (CDSL)](#4-states-cdsl) + - [Source Freshness Status](#source-freshness-status) + - [Trap Suspect Lifecycle](#trap-suspect-lifecycle) +- [5. Definitions of Done](#5-definitions-of-done) + - [Per-Source Thresholds](#per-source-thresholds) + - [Per-Table Opt-Out](#per-table-opt-out) + - [Workflow Execution](#workflow-execution) + - [Notification Delivery](#notification-delivery) + - [Trap Detection](#trap-detection) + - [Local Verification](#local-verification) +- [6. Acceptance Criteria](#6-acceptance-criteria) + + + +Daily check that every Airbyte-managed `bronze_*` source has received fresh +rows within the last warn-after window for its tier. The mechanism is a +single dbt project policy plus an Argo `CronWorkflow` — no per-connector +code changes are required for new sources to be covered. + +## 1. Feature Context + +### 1.1 Overview + +Today the ingestion pipeline has no signal for "data is missing". +Connectors that silently stop emitting rows (expired tokens, upstream API +outage, Argo CronWorkflow failures, sync ran but produced 0 rows) only +become visible when a downstream metric goes flat days later. + +This feature wires `dbt source freshness` against the +`_airbyte_extracted_at` column that every Airbyte ClickHouse destination +writes per row, or against a business-date column for report-style +connectors. A source is `pass` when `MAX()` is within the tier's +warn-after window, `warn` between warn-after and error-after, `error` past +error-after, and `runtime error` if the freshness query itself fails. + +### 1.2 Purpose + +Catch ingestion-layer breaches before they propagate to silver/gold +metrics and to the dashboard. Keep the cost of adding new connectors low +— a new connector is covered the moment it declares a `freshness:` block on +its bronze source; the CronWorkflow picks it up via `source:*` with no +per-connector plumbing. + +**Requirements**: `cpt-insightspec-fr-mon-daily-cronworkflow`, +`cpt-insightspec-fr-mon-thresholds-sot`, +`cpt-insightspec-fr-mon-four-tiers`, +`cpt-insightspec-fr-mon-driver-selection` + +**Principles**: `cpt-insightspec-principle-mon-trap-advisory`, +`cpt-insightspec-principle-mon-vendor-thresholds` + +### 1.3 Actors + +| Actor | Role in Feature | +|---|---| +| Ingestion on-call | Reads Argo workflow status; triages `error` runs | +| Connector owner | Receives a follow-up issue from on-call when their connector breaches | +| `dbt-source-freshness-check` CronWorkflow | Runs daily at 13:00 UTC, parses `target/sources.json`, fans the result out to the configured notification driver | +| Notification consumer | Receives the webhook / Zulip / Slack / Teams / SMTP payload | + +### 1.4 References + +- Operational runbook: [`src/ingestion/MONITORING.md`](../../../../../src/ingestion/MONITORING.md) — verification steps, on-call matrix, parser exit codes, payload shape +- Workflow template: [`charts/insight/templates/ingestion/dbt-source-freshness.yaml`](../../../../../charts/insight/templates/ingestion/dbt-source-freshness.yaml) +- Threshold config: literal `warn_after`/`error_after` per source in each connector's `dbt/schema.yml` (declaration half tracked under EPIC #1321 / #1322 — PR #1346 + the per-connector business-date anchoring in PR SharedQA/insight#1) +- PRD: [../PRD.md](../PRD.md) +- DESIGN: [../DESIGN.md](../DESIGN.md) +- Per-source declarations: every connector's `dbt/schema.yml` carries + `loaded_at_field` at source level (dbt does not propagate this + property from project config). Streaming connectors anchor on + `_airbyte_extracted_at`; report-style connectors (M365 Graph reports, + Slack admin.analytics) anchor on the report's own business-day column + wrapped in `parseDateTimeBestEffortOrNull(...)`. + +## 2. Actor Flows (CDSL) + +### Trigger Freshness Check on Schedule + +- [ ] `p1` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-trigger` + +**Actors**: +- `dbt-source-freshness-check` CronWorkflow (Argo) +- `dbt-source-freshness` WorkflowTemplate (Argo) +- ClickHouse + +**Success Scenarios**: +- Cron fires; selector narrowing finds at least one deployed `bronze_*` + database; `dbt source freshness` writes `target/sources.json`; parser + exits 0 (no breach) or 1 (breach) and dispatches when a driver is + configured. + +**Error Scenarios**: +- `target/sources.json` missing because dbt crashed → parser exits 2. +- ClickHouse unreachable → dbt fails; parser exits 2 because no report + was produced. + +**Steps**: +1. [ ] - `p1` - Argo evaluates `ingestion.freshness.schedule` and spawns a Workflow from the template - `inst-cron-spawn` +2. [ ] - `p1` - Workflow removes any stale `target/sources.json` (workflow template line 252) - `inst-clean-stale` +3. [ ] - `p1` - Workflow queries `system.databases` for existing `bronze_*` schemas (lines 262–297) and builds the effective selector - `inst-narrow-selector` +4. [ ] - `p1` - Workflow runs `dbt source freshness --select ` (lines 305–307) - `inst-run-dbt` +5. [ ] - `p1` - Workflow invokes the inline Python parser - `inst-call-parser` +6. [ ] - `p1` - Workflow invokes the trap detector after the parser; its exit code is captured but not used to override the parser's verdict - `inst-call-trap` +7. [ ] - `p1` - Workflow exits with the parser's exit code - `inst-final-exit` + +### Triage a Page-Worthy Breach + +- [ ] `p1` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-triage` + +**Actors**: +- Ingestion on-call +- Connector owner + +**Success Scenarios**: +- On-call reads the payload, identifies the source, hands off a tracked + issue to the connector owner; next run is green. + +**Error Scenarios**: +- Payload missing `cluster` / `tenant` labels → on-call cannot route + multi-tenant fan-out; configuration bug to fix in Helm overlay. +- Receiver outage hides the page → fall back to Argo + `failedJobsHistoryLimit` retention to find the breach in the UI. + +**Steps**: +1. [ ] - `p1` - On-call receives the notification carrying `[cluster=…, tenant=…] N bronze source(s) breaching freshness SLA` - `inst-receive-page` +2. [ ] - `p1` - On-call identifies the breaching `source` and `age_hours` from `breaches[]` - `inst-read-payload` +3. [ ] - `p1` - On-call cross-references the source's tier in DESIGN to confirm page-worthiness - `inst-check-tier` +4. [ ] - `p1` - On-call inspects last sync in Argo / Airbyte UI to distinguish "sync stopped" from "sync ran but produced no rows" - `inst-inspect-sync` +5. [ ] - `p1` - On-call hands off to the connector owner with the payload as evidence - `inst-handoff` +6. [ ] - `p1` - Connector owner fixes the connector or revisits the SLA tier in `schema.yml` - `inst-fix-connector` + +### Wire a New Connector into Freshness + +- [ ] `p1` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-wire` + +**Actors**: +- Connector author +- CI lint + +**Success Scenarios**: +- New connector declares a `freshness:` block + `loaded_at_field` at source or + table level; the `dbt_coverage.py` gate counts it; next daily run includes it. + +**Error Scenarios**: +- Author leaves a windowed connector on `_airbyte_extracted_at` → false-green + (no breach raised while data is stale). Caught by confirming `ext_age` vs + `biz_age` on live data, not by a lint. +- Author anchors on a non-timestamp / wrong column → dbt emits `runtime error` + for that source (visible, paged), not a silent miss. + +**Steps**: +1. [ ] - `p1` - Author adds a `sources:` entry with a literal `freshness:` block in the connector's `dbt/schema.yml` - `inst-add-source-entry` +2. [ ] - `p1` - Author selects the anchor: `_airbyte_extracted_at` for incremental, a `parseDateTimeBestEffortOrNull()` expression for windowed - `inst-pick-anchor` +3. [ ] - `p1` - Author sets the literal tier (`default`/`report`/`report_extended`/`event`) `warn_after`/`error_after` per source - `inst-pick-tier` +4. [ ] - `p1` - Author opens a PR; the `dbt_coverage.py` gate (EPIC #1321) confirms the source is declared - `inst-coverage-gate` +5. [ ] - `p1` - PR merges; the next daily freshness run includes the new source with no further plumbing - `inst-included-next-run` + +### Switch Notification Driver in a Tenant Overlay + +- [ ] `p2` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-switch-driver` + +**Actors**: +- Platform engineer + +**Success Scenarios**: +- Engineer flips the driver in a Helm overlay; `helm upgrade` re-renders + the WorkflowTemplate; next run dispatches to the new channel. + +**Error Scenarios**: +- New driver's `urlSecret.name` references a missing Secret → workflow + pod fails to start; Argo controller logs surface the error; engineer + creates the Secret and retries. + +**Steps**: +1. [ ] - `p2` - Engineer creates / updates the Secret carrying the new driver's URL or SMTP password - `inst-create-secret` +2. [ ] - `p2` - Engineer sets `ingestion.freshness.notification.driver: ` and the driver's `urlSecret.{name,key}` (or `email.smtp.*`) in the overlay - `inst-edit-overlay` +3. [ ] - `p2` - `helm upgrade` re-renders the WorkflowTemplate; the `secretKeyRef` binding switches to the new driver's branch - `inst-helm-upgrade` +4. [ ] - `p2` - Engineer triggers an ad-hoc workflow run to confirm dispatch to the new channel - `inst-adhoc-run` + +### Override SLA Threshold for One Connector + +- [ ] `p2` - **ID**: `cpt-insightspec-flow-bronze-freshness-sla-override` + +**Actors**: +- Connector author + +**Success Scenarios**: +- Author re-tiers the connector by editing its `schema.yml`; next run + uses the new tier's warn / error pair. + +**Error Scenarios**: +- Author tries to override thresholds at project level via + `+loaded_at_field` → silently ignored by dbt; PRD-level rationale + + CI lint redirect them to per-source declarations. + +**Steps**: +1. [ ] - `p2` - Author updates the per-source `freshness:` block in `schema.yml` to reference a different tier env var (e.g. switch from `FRESHNESS_*_DEFAULT_H` to `FRESHNESS_*_REPORT_H`) - `inst-edit-tier` +2. [ ] - `p2` - Author documents rationale in the commit message (vendor publish-lag evidence, observation window) - `inst-document-rationale` +3. [ ] - `p2` - PR merges; the next run uses the new tier - `inst-take-effect` + +## 3. Processes / Business Logic (CDSL) + +### Resolve dbt Selector to Existing Bronze Schemas + +- [ ] `p1` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-resolve-selector` + +**Input**: configured `dbt_select` Helm value (default `source:*`), +ClickHouse `system.databases` rows. + +**Output**: effective dbt selector string used in +`dbt source freshness --select `. + +**Steps**: +1. [ ] - `p1` - Read `dbt_select` parameter (default `source:*`) - `inst-read-select` +2. [ ] - `p1` - Query ClickHouse: `SELECT name FROM system.databases WHERE name LIKE 'bronze_%'` - `inst-query-databases` +3. [ ] - `p1` - **IF** caller supplied a more specific selector than `source:*`, pass it through verbatim - `inst-passthrough-specific` +4. [ ] - `p1` - **ELSE** narrow `source:*` to `source: source: ...` for each existing bronze database - `inst-narrow-list` +5. [ ] - `p1` - **RETURN** the joined selector - `inst-return-selector` + +### Classify Source Status (PASS / WARN / ERROR / RUNTIME ERROR) + +- [ ] `p1` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-classify` + +**Input**: `target/sources.json` results array. + +**Output**: list of `breaches` records `{source, status, max_loaded_at, +age_hours, empty}`; workflow exit code (0 / 1 / 2). + +**Steps**: +1. [ ] - `p1` - **IF** `target/sources.json` is missing → exit 2 (workflow template line 323) - `inst-missing-report` +2. [ ] - `p1` - **FOR EACH** result in `results[]` - `inst-iter-results` + 1. [ ] - `p1` - Read `unique_id`, `status`, `max_loaded_at`, `max_loaded_at_time_ago_in_s` - `inst-read-fields` + 2. [ ] - `p1` - **IF** `max_loaded_at.startswith("1970-01-01")` → mark `empty=true`, set `max_loaded_at="(table is empty)"`, `age_hours=null` (lines 348–360) - `inst-empty-sentinel` + 3. [ ] - `p1` - **ELSE** convert `max_loaded_at_time_ago_in_s` to hours - `inst-compute-age` + 4. [ ] - `p1` - Append `{source, status, max_loaded_at, age_hours, empty}` to `breaches[]` if status != `pass` - `inst-append-breach` +3. [ ] - `p1` - **IF** any `breaches[i].status ∈ {error, runtime error}` → exit 1 (page-worthy) - `inst-decide-page` +4. [ ] - `p1` - **ELSE** exit 0 (line 559) - `inst-decide-clean` + +### Render Notification Body per Driver + +- [ ] `p1` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-render` + +**Input**: canonical breach payload `{topic, cluster, tenant, summary, +breaches[]}`, active `driver`. + +**Output**: driver-shaped HTTP body or SMTP message. + +**Steps**: +1. [ ] - `p1` - Build summary prefix: include `cluster=…` and `tenant=…` only when non-empty (workflow template lines 393–397) - `inst-build-prefix` +2. [ ] - `p1` - **MATCH** driver - `inst-match-driver` + 1. [ ] - `p1` - `webhook` → JSON body verbatim, `Content-Type: application/json` - `inst-render-webhook` + 2. [ ] - `p1` - `zulip` → rewrite path `/external/json` → `/external/slack`; `application/x-www-form-urlencoded` with Slack-compat fields; Slack mrkdwn (lines 445–478) - `inst-render-zulip` + 3. [ ] - `p1` - `slack` → JSON `{text, channel?}`; GitHub-style mrkdwn (lines 480–487) - `inst-render-slack` + 4. [ ] - `p1` - `teams` → MessageCard JSON; `themeColor` red if any page-worthy, orange otherwise (lines 489–500) - `inst-render-teams` + 5. [ ] - `p1` - `email` → text/plain body; `Subject: ` (lines 502–531) - `inst-render-email` +3. [ ] - `p1` - **RETURN** rendered body / message - `inst-return-body` + +### Detect Re-Emit Trap + +- [ ] `p2` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-trap` + +**Input**: bronze table rows (sampled), optional +`meta.bronze_business_date_col` SQL expression. + +**Output**: list of trap suspects `{source, table, kind, evidence}`. + +**Steps**: +1. [ ] - `p2` - **IF** `meta.bronze_freshness_trap_check == "skip"` → return [] for this source/table - `inst-honour-skip` +2. [ ] - `p2` - **MODE 1** (heuristic, no config) - `inst-mode-fullreemit` + 1. [ ] - `p2` - Compute pct of rows with `_airbyte_extracted_at` in last 30 h - `inst-pct-recent` + 2. [ ] - `p2` - Compute distinct count of `toDate(_airbyte_extracted_at)` - `inst-distinct-days` + 3. [ ] - `p2` - **IF** pct ≥ 95 % AND distinct days ≤ 2 AND row count ≥ 100 → flag `kind=full-reemit` (script lines 188–200) - `inst-flag-fullreemit` +3. [ ] - `p2` - **MODE 2** (opt-in) - `inst-mode-incremental-topup` + 1. [ ] - `p2` - **IF** source declares `meta.bronze_business_date_col` → compute `MAX()` - `inst-compute-bdate` + 2. [ ] - `p2` - Compute `MAX(_airbyte_extracted_at)` - `inst-compute-extracted` + 3. [ ] - `p2` - **IF** gap ≥ 24 h → flag `kind=incremental-topup` (lines 202–227) - `inst-flag-incremental` +4. [ ] - `p2` - **RETURN** suspects (advisory only — does not change parser exit code) - `inst-return-suspects` + +### Choose SLA Tier per Source + +- [ ] `p2` - **ID**: `cpt-insightspec-algo-bronze-freshness-sla-tier` + +**Input**: connector cadence (streaming / event / report / +report-extended), vendor-documented publish-lag evidence. + +**Output**: per-source `freshness:` block in `schema.yml` referencing +the chosen tier's env vars. + +**Steps**: +1. [ ] - `p2` - **IF** connector is streaming with daily cron and no natural quiet days → `default` (30/48 h) - `inst-tier-default` +2. [ ] - `p2` - **ELSE IF** connector has natural quiet days (Confluence edits, Zoom meetings) → `event` (72/96 h) - `inst-tier-event` +3. [ ] - `p2` - **ELSE IF** vendor documents 24–48 h publish lag (Microsoft Graph reports baseline) → `report` (48/96 h) - `inst-tier-report` +4. [ ] - `p2` - **ELSE IF** vendor publishes with ~3-day baseline lag (Slack admin.analytics typical) → `report_extended` (72/120 h) - `inst-tier-report-extended` +5. [ ] - `p2` - Encode the choice as a per-source `freshness:` block referencing the tier's env vars (`FRESHNESS_WARN_*_H` / `FRESHNESS_ERROR_*_H`) - `inst-encode-tier` + +## 4. States (CDSL) + +### Source Freshness Status + +- [ ] `p1` - **ID**: `cpt-insightspec-state-bronze-freshness-sla-source-status` + +A bronze source transitions between four classification states on each +daily run. The state is recomputed from the anchor every time — there is +no persistence between runs. + +| From | Event | To | +|---|---|---| +| (any) | `MAX()` within warn-after | `pass` | +| `pass` | anchor ages past warn-after, before error-after | `warn` | +| `warn` | anchor ages past error-after | `error` | +| (any) | freshness query itself fails | `runtime error` | +| `error` / `warn` | new rows land within warn-after | `pass` | +| `pass` (zero rows ever) | empty-table sentinel detected | `pass` (with `empty=true` flag in payload) | + +`pass` does not page; `warn` shows up in the payload but is not +page-worthy; `error` and `runtime error` page on the daily run. + +### Trap Suspect Lifecycle + +- [ ] `p2` - **ID**: `cpt-insightspec-state-bronze-freshness-sla-trap-suspect` + +A trap suspect has a single advisory-only state per run — it is logged +but never persisted. + +| From | Event | To | +|---|---|---| +| (none) | mode-1 heuristic threshold crossed | `flagged-fullreemit` (advisory) | +| (none) | mode-2 business-date divergence detected | `flagged-incremental-topup` (advisory) | +| `flagged-*` | next run no longer matches | (none) | +| (any) | `meta.bronze_freshness_trap_check: skip` | `suppressed` (no state machine entry) | + +## 5. Definitions of Done + +### Per-Source Thresholds + +- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-thresholds` + +The system **MUST** ensure every `bronze_*` source declared anywhere +under `src/ingestion/connectors/` is included in `dbt source freshness` +without per-connector wiring. Each source **MUST** declare its own +`freshness:` block + `loaded_at_field` literally in its `schema.yml` +(the `dbt_coverage.py` gate counts per-source declarations); the +CronWorkflow runs `--select source:*` and needs no project default. + +**Implements**: +- `cpt-insightspec-flow-bronze-freshness-sla-wire` +- `cpt-insightspec-algo-bronze-freshness-sla-tier` + +**Covers (PRD)**: +- `cpt-insightspec-fr-mon-thresholds-sot` +- `cpt-insightspec-fr-mon-four-tiers` + +**Touches**: +- `src/ingestion/connectors/*/*/dbt/schema.yml` + +### Per-Table Opt-Out + +- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-optout` + +`freshness: null` on a table **MUST** exclude it from the breach count +without removing the source itself from the report. Every opt-out +**MUST** carry a one-line rationale comment next to it, and is reserved +for *incremental* streams that legitimately go quiet for days (a +full-refresh roster/lookup keeps `_airbyte_extracted_at` as a +sync-liveness signal instead). + +**Implements**: +- `cpt-insightspec-flow-bronze-freshness-sla-wire` + +**Covers (PRD)**: +- `cpt-insightspec-fr-mon-optout-pertable` +- `cpt-insightspec-fr-mon-optout-rationale` + +**Touches**: +- `src/ingestion/connectors/*/*/dbt/schema.yml` + +### Workflow Execution + +- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-workflow` + +The daily run **MUST** complete within `activeDeadlineSeconds: 1200` for +the current connector set (~20 sources, ~80 tables; observed end-to-end +~2 min on a warm CH). A stale-data fixture **MUST** produce parser +exit code 1 with a log entry naming the source, max anchor, and lag in +hours. A clean run **MUST** produce parser exit code 0 and a one-line +"all sources within SLA" log. + +**Implements**: +- `cpt-insightspec-flow-bronze-freshness-sla-trigger` +- `cpt-insightspec-algo-bronze-freshness-sla-resolve-selector` +- `cpt-insightspec-algo-bronze-freshness-sla-classify` + +**Covers (PRD)**: +- `cpt-insightspec-fr-mon-daily-cronworkflow` +- `cpt-insightspec-fr-mon-selector-narrowing` +- `cpt-insightspec-fr-mon-stale-report-cleanup` +- `cpt-insightspec-fr-mon-exit-rederivation` +- `cpt-insightspec-nfr-mon-activation-deadline` +- `cpt-insightspec-nfr-mon-exit-codes` + +**Touches**: +- `charts/insight/templates/ingestion/dbt-source-freshness.yaml` + +### Notification Delivery + +- [ ] `p1` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-notification` + +When `notification.driver` is `""`, the workflow **MUST** succeed (or +fail on `error`) without attempting a fan-out. When set, the rendered +payload **MUST** be POSTed (or sent via SMTP). Delivery failures +**MUST** be logged but **MUST NOT** change the workflow's primary exit +code. + +**Implements**: +- `cpt-insightspec-flow-bronze-freshness-sla-switch-driver` +- `cpt-insightspec-algo-bronze-freshness-sla-render` + +**Covers (PRD)**: +- `cpt-insightspec-fr-mon-driver-selection` +- `cpt-insightspec-fr-mon-credential-isolation` +- `cpt-insightspec-fr-mon-dispatch-impl` +- `cpt-insightspec-fr-mon-delivery-failure-isolation` +- `cpt-insightspec-fr-mon-identity-helm` +- `cpt-insightspec-fr-mon-identity-summary` +- `cpt-insightspec-fr-mon-identity-payload` + +**Touches**: +- `charts/insight/templates/ingestion/dbt-source-freshness.yaml` +- `charts/insight/values.yaml` +- `charts/insight/values.schema.json` + +### Trap Detection + +- [ ] `p2` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-trap` + +The trap detector **MUST** run after the parser, support both heuristic +full-reemit detection and opt-in business-date divergence, honour +`meta.bronze_freshness_trap_check: skip`, and never override the +parser's exit code. + +**Implements**: +- `cpt-insightspec-algo-bronze-freshness-sla-trap` + +**Covers (PRD)**: +- `cpt-insightspec-fr-mon-trap-post-parser` +- `cpt-insightspec-fr-mon-trap-two-modes` +- `cpt-insightspec-fr-mon-trap-skip-annotation` +- `cpt-insightspec-fr-mon-trap-advisory` +- `cpt-insightspec-nfr-mon-trap-advisory-mode` + +**Touches**: +- `src/ingestion/scripts/freshness-trap-detect.py` +- `charts/insight/templates/ingestion/dbt-source-freshness.yaml` + +### Local Verification + +- [ ] `p2` - **ID**: `cpt-insightspec-dod-bronze-freshness-sla-local-verify` + +`dbt source freshness --select 'source:bronze_*'` **MUST** run from the +`insight-toolbox` image against `clickhouse-local` (HTTP `:8123`, +`host.docker.internal`) using the workspace dbt profile, with no +per-source environment-specific overrides. + +**Touches**: +- `src/ingestion/dbt/profiles.yml` +- `src/ingestion/MONITORING.md` + +## 6. Acceptance Criteria + +- [ ] Every bronze source under `src/ingestion/connectors/*/*/dbt/schema.yml` declares a reachable `loaded_at_field` (source / table / opt-out). +- [ ] `dbt source freshness --select 'source:bronze_*'` from a clean checkout produces a non-empty `target/sources.json` with one result per declared (source, table) pair, no `runtime error` due to missing `loaded_at_field`. +- [ ] CronWorkflow renders cleanly when `templates.enabled=true` and `toolboxImage` is pinned (no `:latest` defaults). +- [ ] Stale-data fixture (truncate or backdate `_airbyte_extracted_at` on a single source) produces exit code 1 and the source appears in the log + payload. +- [ ] On-call matrix and webhook payload contract documented in [`MONITORING.md`](../../../../../src/ingestion/MONITORING.md). +- [ ] Synthetic re-emit fixture is flagged by the trap detector with `kind=full-reemit`; opt-in business-date divergence fixture is flagged with `kind=incremental-topup`. +- [ ] Switching `notification.driver` in a Helm overlay re-points dispatch to the new channel after `helm upgrade`, with no Secret value visible in rendered manifests. diff --git a/src/ingestion/MONITORING.md b/src/ingestion/MONITORING.md new file mode 100644 index 000000000..4467774e2 --- /dev/null +++ b/src/ingestion/MONITORING.md @@ -0,0 +1,315 @@ +# Ingestion Monitoring — operator runbook + +Goal: detect "data is missing for several days" before users notice. Today's +coverage is **freshness only** — newly arrived rows in bronze tables. Volume +and Airbyte-job-level signals are listed under [Open work](#open-work). + +This file is the operator-facing runbook (verification steps, on-call +matrix, parser exit codes, payload shape). The feature design itself — +purpose, threshold inheritance, acceptance criteria — lives in +[`docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md`](../../docs/domain/ingestion-monitoring/specs/feature-bronze-freshness-sla/FEATURE.md). +The companion PRD and DESIGN for the broader monitoring domain live next +to it: [`PRD.md`](../../docs/domain/ingestion-monitoring/specs/PRD.md), +[`DESIGN.md`](../../docs/domain/ingestion-monitoring/specs/DESIGN.md). + +## What's wired + +### Bronze freshness (live) + +Every bronze source declares its own `freshness:` block + `loaded_at_field` +**per source in the connector's `schema.yml`** (not inherited from a project +default — `dbt_coverage.py` counts per-source declarations, so each source +declares explicitly). The declaration half is tracked under EPIC #1321 / #1322 +(PR #1346 + the per-connector anchoring in PR SharedQA/insight#1); this +CronWorkflow only *runs* the check and acts on the verdict. + +The critical choice is **which column** `loaded_at_field` points at, and it +depends on whether the connector is **incremental** or **windowed**: + +- **Incremental** (git, jira, youtrack — rows land as events happen) → + `loaded_at_field: _airbyte_extracted_at`. The technical extract timestamp + tracks reality because the cursor only advances when new data arrives. +- **Windowed / vendor-analytics** (M365 Graph, ChatGPT/Claude Team, Slack + analytics, Cursor daily, OpenAI usage — the API re-emits a fixed reporting + window every sync) → anchor on the **business-date column** + (`parseDateTimeBestEffortOrNull()`). Here `_airbyte_extracted_at` is + re-stamped every run and would be **false-green**: verified on live data, + M365 business data was 4.5 days stale while `_airbyte_extracted_at` was 10.6h + fresh. See the "Trap to avoid" item in the new-connector checklist below. + +Examples of the two forms: + +```yaml +# Streaming connector — Airbyte cursor follows business time. Rows land in +# bronze approximately when they happen, so the technical extracted-at +# timestamp tracks reality. +sources: + - name: bronze_ + schema: bronze_ + loaded_at_field: _airbyte_extracted_at + tables: ... +``` + +```yaml +# Report-style connector — Airbyte re-fetches a fixed window every run +# (e.g. Microsoft Graph reports, Slack admin.analytics.getFile). Even when +# the upstream has not advanced, the sync writes "fresh" rows for older +# business days, so `_airbyte_extracted_at` stays green forever. Anchor on +# the report's own business-day column instead. ISO-8601 strings sort +# lexically, so wrapping in `parseDateTimeBestEffortOrNull(...)` works +# directly inside `loaded_at_field`. +sources: + - name: bronze_ + schema: bronze_ + loaded_at_field: parseDateTimeBestEffortOrNull(reportRefreshDate) + tables: ... +``` + +Active per-source assignments. The `warn_after`/`error_after` thresholds are +**literal values in each source's `schema.yml`** (not Helm/env-var driven), so +there is a single source of truth and the local `dbt source freshness` reads +exactly what ships. Tiers used: **default** 36h/72h · **report** 72h/120h · +**report_extended** 120h/168h · **event** 96h/168h. + +| Source / table | `loaded_at_field` | Tier | Confidence | +|---|---|---|---| +| bronze_m365.*_activity | `parseDateTimeBestEffortOrNull(reportRefreshDate)` | report | **verified** (live) | +| bronze_chatgpt_team.{chat_activity,codex_user_daily} | `parseDateTimeBestEffortOrNull(date)` | report | **verified** | +| bronze_chatgpt_team.{subscription_usage,subscription_balance} | `parseDateTimeBestEffortOrNull(snapshot_date)` | report | inferred | +| bronze_claude_team.code_metrics | `parseDateTimeBestEffortOrNull(metric_date)` | report | **verified** | +| bronze_cursor.cursor_daily_usage | `parseDateTimeBestEffortOrNull(day)` | report | inferred (daily resync re-fetch) | +| bronze_slack.users_details | `parseDateTimeBestEffortOrNull(date)` | report_extended | inferred (3–5d Slack lag) | +| bronze_zoom.{meetings,participants} | `parseDateTimeBestEffortOrNull(end_time/join_time)` | event | inferred (30d window; quiet weekends real) | +| bronze_openai.usage_*/costs | `parseDateTimeBestEffortOrNull(bucket_start_time)` | report | inferred | +| bronze_claude_admin.{messages_usage,cost_report,code_usage} | `parseDateTimeBestEffortOrNull(date)` | report | inferred | +| bronze_claude_enterprise.summaries | `parseDateTimeBestEffortOrNull(date)` | report | inferred | +| bronze_github_copilot.{user,org}_metrics | `parseDateTimeBestEffortOrNull(day)` | report | inferred | +| bronze_confluence.wiki_pages / bronze_outline.wiki_pages | `parseDateTimeBestEffortOrNull(updated_at)` | event | inferred | +| git / jira / youtrack / figma / salesforce / hubspot / zulip_proxy | `_airbyte_extracted_at` | default | incremental — extracted_at tracks reality | +| rosters & lookups (bamboohr, ms_entra, workday, *_members, *_seats, *_users, *_statuses…) | `_airbyte_extracted_at` | default | full-refresh → sync-liveness signal | + +"verified" rows are backed by measured `ext_age` vs `biz_age` on live data; +"inferred" rows are anchored on the connector's cursor column (verified to +exist) but their windowed behavior is not yet confirmed against data — they are +marked inline in each `schema.yml` with `confirm once ingested`. + +Re-categorizing a connector across tiers is an engineering change (it usually +comes with a `loaded_at_field` revisit), not an ops dial — that's why the +mapping lives in connector `schema.yml`, literal per source. + +`loaded_at_field` is a dbt **property**, not a config — `+loaded_at_field` +at project level is silently ignored. The dbt-clickhouse adapter does not +support metadata-based freshness, so a source missing `loaded_at_field` +fails with `runtime error` instead of falling back to a default. + +A single CronWorkflow `dbt-source-freshness-check` runs `dbt source freshness` +daily at 13:00 UTC (after every connector's sync window of 02:00–11:00 UTC) +and parses `target/sources.json`. Any source in `warn` or `error` is logged +with name, max-loaded-at and lag; if a notification driver is configured +under `ingestion.freshness.notification.driver` (one of `webhook`, `zulip`, +`slack`, `teams`, `email`), the breach list is dispatched through the +matching driver. Driver shapes and credential bindings are documented in +[`docs/domain/ingestion-monitoring/specs/DESIGN.md` §3.3](../../docs/domain/ingestion-monitoring/specs/DESIGN.md#33-api-contracts). + +| Status | Meaning | Workflow exit | What to do | +|--------|---------|---------------|------------| +| `pass` | MAX(``) within the source's tier `warn_after` window | 0 | Nothing | +| `warn` | between `warn_after` and `error_after` (one missed run for daily-cadence sources) | 0 (visible in log + payload) | Investigate during business hours | +| `error` | past `error_after` for the source's tier | 1 (Argo Failed) | Page | +| `runtime error` | dbt couldn't even check the source (CH down, schema drift, query failure) | 1 (Argo Failed) | Page — investigate before trusting other sources | + +Concrete tier values are the literal `warn_after`/`error_after` in the source's +`schema.yml`. Tiers: `default` 36h/72h, `report` 72h/120h, `report_extended` +120h/168h, `event` 96h/168h. + +`error` and `runtime error` flip the workflow to Failed so Argo retains the +run in `failedJobsHistoryLimit`. Warn-only runs stay Successful — the breach +is still printed to the workflow log and POSTed in the notification payload, +but on-call doesn't get paged on a single missed sync. + +### How it stays generic + +The CronWorkflow is connector-agnostic: it runs `dbt source freshness --select +source:*` and acts on whatever every `bronze_*` source declares. A new +connector is covered the moment its `schema.yml` carries a `freshness:` block — +no per-connector pipeline plumbing. The declaration (which column, which tier) +is the connector author's call, captured per source in `schema.yml`. + +## Adding a new connector — freshness checklist + +1. In the new `connectors///dbt/schema.yml`, declare the + `bronze_*` source with a `freshness:` block + `loaded_at_field`. +2. Pick the `loaded_at_field` by sync shape: + - **Incremental / event-cursor** (commits, issues, edits) — + `loaded_at_field: _airbyte_extracted_at`, `default` tier (36h/72h). The + extract timestamp tracks reality because the cursor only advances on new data. + - **Windowed / vendor-analytics** (re-fetches a fixed reporting window every + run — Graph reports, Slack/Cursor/OpenAI daily) — + `loaded_at_field: parseDateTimeBestEffortOrNull()` with the + `report` (or `report_extended` for 3–5d lag) tier. **Do not** leave it on + `_airbyte_extracted_at` — it re-stamps every sync and goes false-green. + - **Event-style with quiet weekends** (Confluence/Zoom — a zero-row Saturday is + normal) — anchor on the event timestamp with the `event` tier (96h/168h): + + ```yaml + freshness: + warn_after: { count: 96, period: hour } + error_after: { count: 168, period: hour } + loaded_at_field: parseDateTimeBestEffortOrNull() + ``` + +3. Confirm the windowed-vs-incremental call against live data when the connector + has rows: compare `now() - max(_airbyte_extracted_at)` against + `now() - max()`. A large gap = windowed (anchor on the business + date); roughly equal = incremental (`_airbyte_extracted_at` is fine). +4. Roster/lookup tables (full-refresh, rarely-changing) keep `_airbyte_extracted_at` + on the `default` tier — the re-stamped extract time is a useful "sync alive" + signal. Use a per-table `freshness: null` opt-out only for streams that are + *incremental* and legitimately go quiet for days (where `_airbyte_extracted_at` + would false-alarm). +5. Coverage is measured by the QA-owned `dbt_coverage.py` gate (EPIC #1321), + which counts per-source declarations — which is why each source declares + explicitly rather than inheriting a project default. + +6. **Trap to avoid**: if your connector re-emits a fixed window every run + (`SELECT count(), max(_airbyte_extracted_at) - min(_airbyte_extracted_at) + FROM bronze_.
` shows all rows extracted within the last 24h + even though the table covers many days of history), `_airbyte_extracted_at` + will look fresh forever (the false-green failure mode). Anchor on the + business-date column instead. This is a judgment call about the source + shape — confirm it against live data and record it in the assignment table + above. + +## Who consumes the signal + +Per-environment ownership matrix until the delivery channel is wired (see +[Open work](#open-work)). + +Ownership for "ingestion on-call" and "connector owner" is not assigned yet +(no rotation document, no `CODEOWNERS` for `src/ingestion/connectors/*` as +of this commit). The matrix below describes the *roles* the freshness +signal expects to land on; see [Open work](#open-work) for the rotation gap. + +| Role | What they read | When | Action | +|---|---|---|---| +| Ingestion on-call (TBD) | Argo UI / `kubectl get workflows -n argo --sort-by=.metadata.creationTimestamp` for the `dbt-source-freshness-check` runs | Daily, after the 13:00 UTC run | Triage `error` / `runtime error` runs | +| `constructorfabric/insight` repo Issues | One issue per persistent breach (>2 consecutive runs) opened by the on-call | Within 1 business day of the breach | Hand off to the connector owner | +| Connector owner (TBD per connector) | The issue body — includes the failing source, max-loaded-at, lag in hours | On issue assignment | Fix the connector or update the SLA | +| Tenant on-call (post-MVP) | Webhook payload (Zulip / email / generic POST) routed by `cluster` field | Real-time | Same triage as above, scoped to one deployment | + +Until the webhook channel lands, the **only** push mechanism is Argo's +failed-runs list — on-call must check `kubectl get workflows -n argo +--sort-by=.metadata.creationTimestamp` at least once per business day. The +`failedJobsHistoryLimit: 5` ensures the latest five breaching runs are +retained for inspection. + +`pass`-only runs leave no trace beyond Argo's success history (kept by +`successfulJobsHistoryLimit: 3`) — silent green is the desired steady state. + +## Open work + +### Rotation / ownership — not assigned + +The matrix above describes roles, not people. There is no documented +ingestion on-call rotation as of this commit, and `src/ingestion/connectors/` +has no `CODEOWNERS` entries assigning per-connector owners. Until that +lands, the freshness signal lives in Argo's failed-runs list with no +named consumer. + +Action: agree on an on-call rotation (or a single owner during MVP) and +add a `CODEOWNERS` block listing the per-connector owner so the breach +hand-off has a real target. + +### Delivery channel — driver-based + +`charts/insight/values.yaml` exposes notification routing under +`ingestion.freshness.notification.*` (driver selector + per-driver +subblock). When `driver: ""` (the default), breaches surface only via: + +- Argo UI / `kubectl get workflows -n argo -l app.kubernetes.io/component=ingestion-monitoring` +- Workflow exit status (failed runs accumulate in + `failedJobsHistoryLimit: 5`) + +Setting `driver` to one of `webhook`, `zulip`, `slack`, `teams`, `email` +activates the matching driver's dispatcher; the URL or SMTP password is +read from a Kubernetes Secret on a `secretKeyRef` env binding so the +credential never lands in rendered manifests or Argo UI. Driver shapes, +secret bindings, and observed quirks (Zulip endpoint rewrite, Cloudflare +UA, Slack mrkdwn vs Zulip native) are documented in +[`docs/domain/ingestion-monitoring/specs/DESIGN.md` §3.3](../../docs/domain/ingestion-monitoring/specs/DESIGN.md#33-api-contracts). + +The webhook driver's body remains provider-agnostic so a custom relay +can sit in front of any of the above: + +```json +{ + "topic": "ingestion-freshness", + "cluster": "prod", + "tenant": "acme-co", + "summary": "[cluster=prod, tenant=acme-co] 3 bronze source(s) breaching freshness SLA", + "breaches": [ + { + "source": "source.ingestion.bronze_jira.jira_issue", + "status": "error", + "max_loaded_at": "2026-04-28T03:14:21Z", + "age_hours": 51.2, + "empty": false + } + ] +} +``` + +`cluster` and `tenant` come from `ingestion.freshness.{cluster,tenant}` +in values overrides — both empty by default; the `summary` drops empty +labels from its prefix. + +### Volume baseline (next iteration) + +Freshness catches "no rows arrived" but misses "API returned 50 rows instead +of 5000". Plan: a singular SQL test (or dbt operation) that compares today's +row count per stream against a 14-day median, alerts on <30%. Reuses the same +`dbt-source-freshness` workflow shell — just a different selector. + +### Source vs bronze attribution + +Today the freshness check flags "no rows in bronze" but cannot tell whether +the upstream Airbyte sync ran. To distinguish: + +- *source/credential issue*: Airbyte sync ✅, bronze pulled 0 rows +- *pipeline issue*: Airbyte sync ❌ (didn't run / errored) + +We'd need a sidecar that polls the Airbyte Jobs API after each sync and +appends a row to `staging.airbyte_runs (connection_id, status, +records_emitted, started_at, ended_at)`. Then the freshness step can JOIN +against that table and label each breach with a root cause. Future PR. + +## How to verify locally + +```bash +# After dev-up.sh + at least one successful sync +kubectl create -n argo -f - <<'EOF' +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: freshness-adhoc- + namespace: argo +spec: + workflowTemplateRef: + name: dbt-source-freshness + arguments: + parameters: + - name: dbt_select + value: "source:bronze_jira" + - name: toolbox_image + value: "insight-toolbox:local" + - name: clickhouse_host + value: "insight-clickhouse.insight.svc.cluster.local" + - name: clickhouse_port + value: "8123" + - name: clickhouse_user + value: "default" +EOF + +# Watch the run +kubectl logs -n argo -l workflows.argoproj.io/workflow= -f +``` diff --git a/src/ingestion/scripts/freshness-trap-detect.py b/src/ingestion/scripts/freshness-trap-detect.py new file mode 100755 index 000000000..ce3fab882 --- /dev/null +++ b/src/ingestion/scripts/freshness-trap-detect.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""Detect bronze sources whose `_airbyte_extracted_at` anchor is masking a +trap that the dbt source freshness check cannot see — every nightly sync +makes the anchor look fresh even when the upstream has stopped publishing. + +Two detection modes: + +1. Full re-emit (heuristic, no per-source config required). + Fires when ≥ 95% of rows have `_airbyte_extracted_at` within the last + 30h *and* `_airbyte_extracted_at` covers ≤ 2 distinct calendar days + *and* the table has ≥ 100 rows. Catches connectors that overwrite the + entire bronze table every run (Confluence-style). + +2. Business-date divergence (explicit opt-in via `meta`). + When a source declares + meta: + bronze_business_date_col: + the script compares `MAX()` against `MAX(_airbyte_extracted_at)` + and warns when the business date lags by ≥ 24h while extracted-at is + fresh. Catches connectors that incrementally top up the latest day's + row (M365 / Slack / Cursor-style) — the heuristic above can't see these + because most rows really are old, just `MAX(extract)` happens to be + today's because of the new top-up row. + +A source can opt out of mode 1 (e.g. genuine append-only full-refresh +rosters where neither the heuristic nor a business-date anchor applies) +via `meta: { bronze_freshness_trap_check: skip }` at source or table +level. + +Connection params: `CLICKHOUSE_{HOST,PORT,USER,PASSWORD}` env vars (same +as the freshness CronWorkflow). Defaults match the local toolbox setup. + +Exit codes: + 0 — no trap suspects + 1 — at least one suspect found (warn, not page) + 2 — script failure (schema parse, etc.) +""" +from __future__ import annotations + +import os +import sys +import urllib.parse +import urllib.request +from pathlib import Path + +try: + import yaml +except ImportError: + print("PyYAML required: pip install pyyaml", file=sys.stderr) + sys.exit(2) + +EXTRACTED_AT_LITERAL = "_airbyte_extracted_at" +RECENT_WINDOW_HOURS = 30 +SUSPECT_PCT_RECENT = 95.0 +SUSPECT_MAX_DISTINCT_DAYS = 2 +MIN_ROWS = 100 + + +class CHQueryError(Exception): + """Raised when ClickHouse rejects a query (4xx / 5xx). Distinguished from + transport errors so the caller can decide whether to silently skip + (database missing in fixture) or escalate.""" + + +CH_BASE_URL: str | None = None + + +def _resolve_ch_base_url() -> str: + """Resolve `http://host:port/?user=…&password=…` once at startup. + + Defaults match the local toolbox so the script is runnable from a + developer shell without exporting anything; in CI / Argo every var is + set explicitly. Missing vars fall back to defaults — the script logs + which params came from defaults so a half-configured prod env is + obvious in workflow logs.""" + host_default, port_default = "host.docker.internal", "8123" + user_default, password_default = "default", "clickhouse" + host = os.environ.get("CLICKHOUSE_HOST") or host_default + port = os.environ.get("CLICKHOUSE_PORT") or port_default + user = os.environ.get("CLICKHOUSE_USER") or user_default + password = os.environ.get("CLICKHOUSE_PASSWORD") or password_default + using_defaults = [ + name for name, used_default in [ + ("CLICKHOUSE_HOST", host == host_default and not os.environ.get("CLICKHOUSE_HOST")), + ("CLICKHOUSE_PORT", port == port_default and not os.environ.get("CLICKHOUSE_PORT")), + ("CLICKHOUSE_USER", user == user_default and not os.environ.get("CLICKHOUSE_USER")), + ("CLICKHOUSE_PASSWORD", password == password_default and not os.environ.get("CLICKHOUSE_PASSWORD")), + ] if used_default + ] + if using_defaults: + print( + f"[freshness-trap-detect] using local defaults for: {', '.join(using_defaults)}", + file=sys.stderr, + ) + creds = urllib.parse.urlencode({"user": user, "password": password}) + return f"http://{host}:{port}/?{creds}" + + +def ch_query(sql: str) -> str: + global CH_BASE_URL + if CH_BASE_URL is None: + CH_BASE_URL = _resolve_ch_base_url() + url = f"{CH_BASE_URL}&{urllib.parse.urlencode({'query': sql})}" + try: + with urllib.request.urlopen(url, timeout=30) as resp: + return resp.read().decode("utf-8").strip() + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + raise CHQueryError(f"HTTP {e.code}: {body[:200]}") from None + except urllib.error.URLError as e: + # DNS / refused / TLS / socket-level failures land here. Treat as + # transport errors so the caller skips the table without crashing + # the whole script. + raise CHQueryError(f"URLError: {e.reason}") from None + except (TimeoutError, OSError) as e: + # `socket.timeout` was aliased to `TimeoutError` in Py 3.10+; + # `OSError` covers connection-reset / broken-pipe at the socket + # layer that bypass the URLError translation. + raise CHQueryError(f"transport: {e}") from None + + +def candidate_tables(connectors_root: Path) -> list[dict]: + """Return one record per bronze table that should be checked. + + Each record has: database, table, source_meta_business_date (str|None), + table_meta_business_date (str|None), skip_heuristic (bool). + + A table is included if its effective `loaded_at_field` is + `_airbyte_extracted_at` (no business-date anchor was deliberately + chosen). Tables opted out via `meta.bronze_freshness_trap_check: skip` + are excluded entirely. Tables that declare + `meta.bronze_business_date_col` get the divergence check even when + they would otherwise be skipped by the heuristic. + """ + out: list[dict] = [] + for schema_file in sorted(connectors_root.glob("**/dbt/schema.yml")): + try: + doc = yaml.safe_load(schema_file.read_text()) + except yaml.YAMLError as e: + print(f"[freshness-trap-detect] skipping malformed yaml {schema_file}: {e}", file=sys.stderr) + continue + if not isinstance(doc, dict): + print(f"[freshness-trap-detect] skipping non-mapping yaml root {schema_file}", file=sys.stderr) + continue + for source in doc.get("sources") or []: + name = source.get("name", "") + # dbt convention: if a source omits `schema:`, the source `name` + # is used as the ClickHouse database. Mirror that here so we + # query the same DB dbt does. + schema = source.get("schema", name) + # Match bronze sources by either dbt name or CH schema (bamboohr + # uses `name: bamboohr, schema: bronze_bamboohr`). + is_bronze = ( + isinstance(name, str) and name.startswith("bronze_") + ) or ( + isinstance(schema, str) and schema.startswith("bronze_") + ) + if not is_bronze: + continue + source_anchor = source.get("loaded_at_field", "").strip() + source_meta = source.get("meta") or {} + for table in source.get("tables") or []: + tname = table.get("name") + if not tname: + continue + if "freshness" in table and table["freshness"] is None: + continue + anchor = table.get("loaded_at_field", "").strip() or source_anchor + if anchor != EXTRACTED_AT_LITERAL: + continue + meta = table.get("meta") or {} + skip = ( + (meta.get("bronze_freshness_trap_check") or "").lower() == "skip" + or (source_meta.get("bronze_freshness_trap_check") or "").lower() == "skip" + ) + business = ( + meta.get("bronze_business_date_col") + or source_meta.get("bronze_business_date_col") + ) + # If the table is opted-out and has no business-date hint, + # there is nothing to check. + if skip and not business: + continue + out.append({ + "database": schema, + "table": tname, + "skip_heuristic": skip, + "business_date_col": business, + }) + return out + + +def check_table(candidate: dict) -> dict | None: + """Return a finding dict if the table is a trap suspect, else None.""" + database = candidate["database"] + table = candidate["table"] + business = candidate["business_date_col"] + skip_heuristic = candidate["skip_heuristic"] + + select_parts = [ + "count() AS rows", + f"countIf(_airbyte_extracted_at >= now() - INTERVAL {RECENT_WINDOW_HOURS} HOUR) AS recent", + "uniqExact(toDate(_airbyte_extracted_at)) AS distinct_days", + "max(_airbyte_extracted_at) AS last_extract", + ] + if business: + select_parts.append(f"max({business}) AS last_business") + sql = ( + "SELECT " + ", ".join(select_parts) + + f" FROM `{database}`.`{table}` " + "FORMAT TabSeparated" + ) + try: + raw = ch_query(sql) + except CHQueryError as e: + # Common in fixtures: bronze_ database does not exist locally. + # In prod the same response means a connector was renamed / dropped + # without a schema.yml update — surface it on stderr so workflow + # logs catch the misconfig instead of swallowing it. + if "Code: 81" in str(e) or "UNKNOWN_DATABASE" in str(e) or "HTTP 404" in str(e): + print(f" [{database}.{table}] missing in ClickHouse (skipping): {e}", file=sys.stderr) + return None + print(f" [{database}.{table}] query failed: {e}", file=sys.stderr) + return None + if not raw: + return None + fields = raw.split("\t") + rows = int(fields[0]) + if rows < MIN_ROWS: + return None + recent = int(fields[1]) + distinct_days = int(fields[2]) + last_extract = fields[3] + pct_recent = 100.0 * recent / rows + last_business = fields[4] if business and len(fields) >= 5 else None + + finding: dict | None = None + + if not skip_heuristic and ( + pct_recent >= SUSPECT_PCT_RECENT + and distinct_days <= SUSPECT_MAX_DISTINCT_DAYS + ): + finding = { + "kind": "full-reemit", + "database": database, + "table": table, + "rows": rows, + "pct_recent": round(pct_recent, 1), + "distinct_days": distinct_days, + "last_extract": last_extract, + } + + if business and last_business and last_business not in ("\\N", "1970-01-01 00:00:00"): + # Compare gap. `_airbyte_extracted_at` arrives with milliseconds + # (e.g. '2026-05-04 02:01:58.805') which `toDateTime` cannot parse — + # use `parseDateTime64BestEffortOrNull` for both sides. + diff_sql = ( + "SELECT dateDiff('hour', " + f"parseDateTime64BestEffortOrNull('{last_business}'), " + f"parseDateTime64BestEffortOrNull('{last_extract}'))" + " FORMAT TabSeparated" + ) + try: + diff_h = int(ch_query(diff_sql)) + except (CHQueryError, ValueError): + diff_h = 0 + if diff_h >= 24: + finding = { + "kind": "incremental-topup", + "database": database, + "table": table, + "rows": rows, + "last_extract": last_extract, + "last_business": last_business, + "extract_minus_business_h": diff_h, + "business_date_col": business, + } + return finding + + +def main(argv: list[str]) -> int: + root = Path(argv[1]) if len(argv) >= 2 else Path("src/ingestion/connectors") + if not root.is_dir(): + print(f"connectors root not found: {root}", file=sys.stderr) + return 2 + + candidates = candidate_tables(root) + if not candidates: + print("no candidate sources (every bronze source already uses a non-extracted-at anchor or is opted out)") + return 0 + + print(f"checking {len(candidates)} bronze table(s) anchored on `_airbyte_extracted_at`...") + suspects: list[dict] = [] + for c in candidates: + result = check_table(c) + if result: + suspects.append(result) + + if not suspects: + print("no trap suspects") + return 0 + + full_reemit = [s for s in suspects if s["kind"] == "full-reemit"] + incremental = [s for s in suspects if s["kind"] == "incremental-topup"] + + if full_reemit: + print(f"\n{len(full_reemit)} full re-emit suspect(s):") + print( + f" ≥ {SUSPECT_PCT_RECENT:.0f}% of rows have `_airbyte_extracted_at` within the " + f"last {RECENT_WINDOW_HOURS}h, across ≤ {SUSPECT_MAX_DISTINCT_DAYS} distinct day(s). " + "The connector is rewriting the entire table on every run — " + "`_airbyte_extracted_at` will look fresh forever, even when the " + "upstream stops publishing. Switch `loaded_at_field` to a real " + "business-date column." + ) + for s in full_reemit: + print( + f" - {s['database']}.{s['table']}: " + f"rows={s['rows']}, pct_recent={s['pct_recent']}%, " + f"distinct_extract_days={s['distinct_days']}, " + f"last_extract={s['last_extract']}" + ) + + if incremental: + print(f"\n{len(incremental)} incremental top-up suspect(s):") + print( + " `MAX(_airbyte_extracted_at)` is fresh but the declared " + "business-date column is at least 24h behind. The connector is " + "writing today's row but the upstream's reported business date " + "has not advanced — the freshness check sees a green anchor that " + "has nothing to do with reality. Switch `loaded_at_field` to the " + "business-date expression you used in `meta`." + ) + for s in incremental: + print( + f" - {s['database']}.{s['table']}: " + f"last_extract={s['last_extract']}, " + f"last_business={s['last_business']}, " + f"gap={s['extract_minus_business_h']}h " + f"(business_date_col={s['business_date_col']})" + ) + + print( + "\nLegitimate full-refresh roster tables (no daily cadence, no " + "business-date column) can opt out per-table:\n" + " meta:\n" + " bronze_freshness_trap_check: skip" + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))