From f4a2bd2de72294da8d3b06546c747ec5d9fef973 Mon Sep 17 00:00:00 2001 From: Kenan Salim Date: Wed, 17 Jun 2026 11:54:09 +0300 Subject: [PATCH 1/3] test(data): spike check for collab activity vs per-user trailing baseline (#1321) Follow-up to the non-negative business-rule test. Adds a data_quality catalog check for the "too high" case the non-negative test cannot catch: a count that is plausible on its own but wildly out of line with the same user's own recent history (e.g. a feeder double-count turning a few hundred views into millions). Deliberately not a fixed upper limit. The baseline is derived per (tenant, user, product, metric) from each series' own trailing 28 observed days (excluding the current day). A day is flagged only when all hold: >=14 prior observations (cold-start guard), non-zero baseline spread, value > mean + 5*sd, and value > 10*mean. The z-score and the relative-jump condition together keep it to genuinely large spikes. severity=warn + store_failures (advisory, monitored, non-blocking) per the data_quality convention, so a noisy or mis-tuned threshold can never fail a build while we calibrate it. Reads FINAL. Note: thresholds are conservative starting points, not yet calibrated against real tenant data (no warehouse available locally to run dbt). They are meant to be tuned from the first scheduled-run findings. Signed-off-by: Kenan Salim --- ...assert_collab_document_counts_no_spike.sql | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql diff --git a/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql b/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql new file mode 100644 index 000000000..1feef6cdd --- /dev/null +++ b/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql @@ -0,0 +1,85 @@ +{{ config( + tags=['data_quality'], + severity='warn', + store_failures=true, + meta={ + 'title': 'Collab document activity spike vs per-user trailing baseline', + 'domain': 'collab', + 'category': 'anomaly', + 'tier': 'warn', + 'remediation': 'A user-day activity count is far above the recent history for that same user, product and metric (z-score > 5 AND > 10x the trailing mean over the last 28 observed days, with at least 14 days of history). Not necessarily wrong — it can be a genuine burst — but a sudden 100x-style jump usually means a feeder double-count, a unit change, or a backfill landing on one date. Inspect the stored rows, compare value against base_mean, and trace the m365 feeder. Thresholds are heuristic and advisory (warn only); tune them once observed against real tenant data.' + } +) }} +-- Statistical spike guard for collaboration document activity (#1321 follow-up). +-- Catches "too high" anomalies the non-negative check cannot: a count that is +-- plausible in isolation but wildly out of line with the same user's own recent +-- history. Deliberately NOT a fixed upper limit — the baseline is derived per +-- (tenant, user, product, metric) from each series' own trailing window, so it +-- adapts to heavy and light users alike instead of one global number. +-- +-- Method: unpivot the five activity counts, then for each series compute a +-- trailing baseline over the previous 28 observed days (excluding the current +-- day). A day is flagged only when ALL of these hold: +-- * at least 14 prior observations exist -- cold-start guard, no baseline yet +-- * the baseline has non-zero spread -- avoids divide-by-noise on flat series +-- * value > mean + 5 * stddev -- z-score outlier +-- * value > 10 * mean -- massive relative jump +-- The two outlier conditions together keep this to genuinely large spikes, so it +-- stays quiet on normal day-to-day variation. Advisory only (severity=warn): it +-- emits a finding and stores the rows, it never fails the pipeline. Read FINAL +-- so transient ReplacingMergeTree duplicates can't look like spikes. +WITH unpivoted AS ( + SELECT + tenant_id, + person_key, + product, + data_source, + date, + m.1 AS metric, + m.2 AS value + FROM {{ ref('class_collab_document_activity') }} FINAL + ARRAY JOIN + [ + ('viewed_or_edited_count', CAST(viewed_or_edited_count AS Nullable(Float64))), + ('synced_count', CAST(synced_count AS Nullable(Float64))), + ('shared_internally_count', CAST(shared_internally_count AS Nullable(Float64))), + ('shared_externally_count', CAST(shared_externally_count AS Nullable(Float64))), + ('visited_page_count', CAST(visited_page_count AS Nullable(Float64))) + ] AS m + WHERE m.2 IS NOT NULL +), +baselined AS ( + SELECT + tenant_id, + person_key, + product, + data_source, + date, + metric, + value, + count() OVER w AS prior_n, + avg(value) OVER w AS base_mean, + stddevSamp(value) OVER w AS base_sd + FROM unpivoted + WINDOW w AS ( + PARTITION BY tenant_id, person_key, product, data_source, metric + ORDER BY date + ROWS BETWEEN 28 PRECEDING AND 1 PRECEDING + ) +) +SELECT + tenant_id, + person_key, + product, + data_source, + date, + metric, + value, + prior_n, + base_mean, + base_sd +FROM baselined +WHERE prior_n >= 14 + AND base_sd > 0 + AND value > base_mean + 5 * base_sd + AND value > base_mean * 10 From f7886f56b6a8cfe641aae8c1eb95b58862ef0744 Mon Sep 17 00:00:00 2001 From: Kenan Salim Date: Wed, 17 Jun 2026 13:00:19 +0300 Subject: [PATCH 2/3] test(data): bound the spike check by recency (perf + stale-baseline) (#1321) Addresses two review points on the trailing-baseline spike check: - Performance: cap the input to the last 120 days (new `recent` CTE) and the output to the last 3 days. Filtering only the output would stop re-alerting on old spikes but the window function still sorts/scans all history; bounding the input is what actually keeps the cost flat as history grows. 120 days comfortably covers a 28-observed-day baseline for any active user; a user with <14 active days in the window isn't evaluated (the prior_n guard). - Stale baseline: the window is ROWS-based (observed days, not calendar days), so a user returning from a long leave would be compared against a months-old baseline. Add `date - prev_date <= 35` (prev_date = most recent prior observed day) so the baseline must itself be recent, removing that false-positive class. Still advisory (severity=warn) and uncalibrated; thresholds to be tuned from the first scheduled-run findings. Signed-off-by: Kenan Salim --- ...assert_collab_document_counts_no_spike.sql | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql b/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql index 1feef6cdd..d1ec3dfd7 100644 --- a/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql +++ b/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql @@ -7,7 +7,7 @@ 'domain': 'collab', 'category': 'anomaly', 'tier': 'warn', - 'remediation': 'A user-day activity count is far above the recent history for that same user, product and metric (z-score > 5 AND > 10x the trailing mean over the last 28 observed days, with at least 14 days of history). Not necessarily wrong — it can be a genuine burst — but a sudden 100x-style jump usually means a feeder double-count, a unit change, or a backfill landing on one date. Inspect the stored rows, compare value against base_mean, and trace the m365 feeder. Thresholds are heuristic and advisory (warn only); tune them once observed against real tenant data.' + 'remediation': 'A recent user-day activity count is far above the recent history for that same user, product and metric (z-score > 5 AND > 10x the trailing mean over the last 28 observed days, with at least 14 days of history and a baseline that is itself recent). Not necessarily wrong — it can be a genuine burst — but a sudden 100x-style jump usually means a feeder double-count, a unit change, or a backfill landing on one date. Inspect the stored rows, compare value against base_mean, and trace the m365 feeder. Thresholds are heuristic and advisory (warn only); tune them once observed against real tenant data.' } ) }} -- Statistical spike guard for collaboration document activity (#1321 follow-up). @@ -17,18 +17,45 @@ -- (tenant, user, product, metric) from each series' own trailing window, so it -- adapts to heavy and light users alike instead of one global number. -- +-- Bounded by design, so the cost stays flat as history grows: +-- * input is capped to the last 120 days (the `recent` CTE). A 28-observed-day +-- baseline for any reasonably active user fits well inside 120 calendar days, +-- so the window sort/scan never grows with total history. A user with fewer +-- than 14 active days in that window simply isn't evaluated (too little +-- signal), which the prior_n guard below enforces. +-- * output is capped to the last 3 days, so a scheduled daily run only reports +-- newly-appearing spikes, not the same old spike re-flagged every run. +-- -- Method: unpivot the five activity counts, then for each series compute a -- trailing baseline over the previous 28 observed days (excluding the current -- day). A day is flagged only when ALL of these hold: -- * at least 14 prior observations exist -- cold-start guard, no baseline yet -- * the baseline has non-zero spread -- avoids divide-by-noise on flat series +-- * the most recent prior observation is within 35 days -- baseline is not stale -- * value > mean + 5 * stddev -- z-score outlier -- * value > 10 * mean -- massive relative jump --- The two outlier conditions together keep this to genuinely large spikes, so it --- stays quiet on normal day-to-day variation. Advisory only (severity=warn): it --- emits a finding and stores the rows, it never fails the pipeline. Read FINAL --- so transient ReplacingMergeTree duplicates can't look like spikes. -WITH unpivoted AS ( +-- The window is ROWS-based (observed days, not calendar days), so without the +-- staleness guard a user returning from a long leave would be compared against a +-- months-old baseline and could false-fire; `date - prev_date <= 35` prevents +-- that. Advisory only (severity=warn): it emits a finding and stores the rows, it +-- never fails the pipeline. Read FINAL so transient ReplacingMergeTree duplicates +-- can't look like spikes. +WITH recent AS ( + SELECT + tenant_id, + person_key, + product, + data_source, + date, + viewed_or_edited_count, + synced_count, + shared_internally_count, + shared_externally_count, + visited_page_count + FROM {{ ref('class_collab_document_activity') }} FINAL + WHERE date >= today() - 120 +), +unpivoted AS ( SELECT tenant_id, person_key, @@ -37,7 +64,7 @@ WITH unpivoted AS ( date, m.1 AS metric, m.2 AS value - FROM {{ ref('class_collab_document_activity') }} FINAL + FROM recent ARRAY JOIN [ ('viewed_or_edited_count', CAST(viewed_or_edited_count AS Nullable(Float64))), @@ -59,7 +86,8 @@ baselined AS ( value, count() OVER w AS prior_n, avg(value) OVER w AS base_mean, - stddevSamp(value) OVER w AS base_sd + stddevSamp(value) OVER w AS base_sd, + max(date) OVER w AS prev_date FROM unpivoted WINDOW w AS ( PARTITION BY tenant_id, person_key, product, data_source, metric @@ -81,5 +109,7 @@ SELECT FROM baselined WHERE prior_n >= 14 AND base_sd > 0 + AND (date - prev_date) <= 35 AND value > base_mean + 5 * base_sd AND value > base_mean * 10 + AND date >= today() - 3 From d7e676a7b0d4fbbd9e51dd9e03c1601c7f81d21b Mon Sep 17 00:00:00 2001 From: Kenan Salim Date: Wed, 17 Jun 2026 14:29:48 +0300 Subject: [PATCH 3/3] test(data): isolate spike baseline per source + cap output by collection time (#1321) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address CodeRabbit review on PR #1355: - Add insight_source_id to the spike-detection grain (recent/unpivoted/ baselined CTEs, WINDOW PARTITION BY, and stored-failure output). Without it, multiple M365 source instances for one tenant share a contaminated baseline and failure rows can't identify which source to inspect. - Cap output by collected_at (row arrival) instead of activity date, so a late-arriving backfill landing today for an older activity date is still flagged — exactly the feeder/backfill anomaly class this check targets — rather than being silently dropped by the date filter. Both columns are present in class_collab_document_activity FINAL (insight_source_id is a documented not_null column; collected_at flows through from the m365 feeders via SELECT *). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Kenan Salim --- .../assert_collab_document_counts_no_spike.sql | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql b/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql index d1ec3dfd7..ca89845b9 100644 --- a/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql +++ b/src/ingestion/dbt/tests/collaboration/assert_collab_document_counts_no_spike.sql @@ -23,8 +23,10 @@ -- so the window sort/scan never grows with total history. A user with fewer -- than 14 active days in that window simply isn't evaluated (too little -- signal), which the prior_n guard below enforces. --- * output is capped to the last 3 days, so a scheduled daily run only reports --- newly-appearing spikes, not the same old spike re-flagged every run. +-- * output is capped to rows collected in the last 3 days (collected_at, the +-- row arrival time -- not activity date), so a scheduled daily run reports +-- newly-arrived spikes once, and a late backfill landing today for an older +-- activity date is still caught instead of being silently dropped. -- -- Method: unpivot the five activity counts, then for each series compute a -- trailing baseline over the previous 28 observed days (excluding the current @@ -43,10 +45,12 @@ WITH recent AS ( SELECT tenant_id, + insight_source_id, person_key, product, data_source, date, + collected_at, viewed_or_edited_count, synced_count, shared_internally_count, @@ -58,10 +62,12 @@ WITH recent AS ( unpivoted AS ( SELECT tenant_id, + insight_source_id, person_key, product, data_source, date, + collected_at, m.1 AS metric, m.2 AS value FROM recent @@ -78,10 +84,12 @@ unpivoted AS ( baselined AS ( SELECT tenant_id, + insight_source_id, person_key, product, data_source, date, + collected_at, metric, value, count() OVER w AS prior_n, @@ -90,13 +98,14 @@ baselined AS ( max(date) OVER w AS prev_date FROM unpivoted WINDOW w AS ( - PARTITION BY tenant_id, person_key, product, data_source, metric + PARTITION BY tenant_id, insight_source_id, person_key, product, data_source, metric ORDER BY date ROWS BETWEEN 28 PRECEDING AND 1 PRECEDING ) ) SELECT tenant_id, + insight_source_id, person_key, product, data_source, @@ -112,4 +121,4 @@ WHERE prior_n >= 14 AND (date - prev_date) <= 35 AND value > base_mean + 5 * base_sd AND value > base_mean * 10 - AND date >= today() - 3 + AND collected_at >= today() - 3