diff --git a/src/ingestion/scripts/create-bronze-placeholders.sh b/src/ingestion/scripts/create-bronze-placeholders.sh index 683eb4205..e39e0e33c 100644 --- a/src/ingestion/scripts/create-bronze-placeholders.sh +++ b/src/ingestion/scripts/create-bronze-placeholders.sh @@ -59,6 +59,7 @@ CREATE DATABASE IF NOT EXISTS bronze_slack; CREATE DATABASE IF NOT EXISTS bronze_bamboohr; CREATE DATABASE IF NOT EXISTS bronze_bitbucket_cloud; CREATE DATABASE IF NOT EXISTS bronze_zulip_proxy; +CREATE DATABASE IF NOT EXISTS bronze_hubspot; SQL # --------------------------------------------------------------------------- @@ -1152,4 +1153,213 @@ CREATE TABLE IF NOT EXISTS bronze_zulip_proxy.users ( SQL fi +# bronze_hubspot — needed by the CRM gold views (insight.crm_kpis, +# crm_chart_flow, crm_bullet_rows, crm_pipeline_now) via the HubSpot dbt +# staging models (hubspot__crm_{deals,users,activities}). Column sets mirror +# the live Airbyte HubSpot streams (verified against a dev sync) but carry +# only the columns the dbt staging models actually read plus the four CDK +# columns — the real sync overwrites the schema (hundreds of `properties_*`) +# in place on first run. HubSpot properties land as Nullable(String); dbt +# coerces with toFloat64OrNull / parseDateTime64BestEffortOrNull. ORDER BY +# unique_key matches the per-record natural key the envelope assigns, so RMT +# collapses re-synced duplicates the same way prod does. +run_ch <<'SQL' +CREATE DATABASE IF NOT EXISTS bronze_hubspot; +SQL +if ! ch_table_exists bronze_hubspot deals; then + echo " Creating placeholder: bronze_hubspot.deals" + run_ch <<'SQL' +CREATE TABLE IF NOT EXISTS bronze_hubspot.deals ( + tenant_id Nullable(String), + source_id Nullable(String), + unique_key Nullable(String), + id Nullable(String), + archived Nullable(Bool), + createdAt Nullable(DateTime64(3)), + updatedAt Nullable(DateTime64(3)), + archivedAt Nullable(DateTime64(3)), + collected_at Nullable(DateTime64(3)), + data_source Nullable(String), + associations_companies Nullable(String), + properties_dealname Nullable(String), + properties_dealstage Nullable(String), + properties_dealtype Nullable(String), + properties_pipeline Nullable(String), + properties_closedate Nullable(String), + properties_amount Nullable(String), + properties_amount_in_home_currency Nullable(String), + properties_hs_acv Nullable(String), + properties_hs_tcv Nullable(String), + properties_hs_arr Nullable(String), + properties_hs_is_closed Nullable(String), + properties_hs_is_closed_won Nullable(String), + properties_hubspot_owner_id Nullable(String), + properties_hs_created_by_user_id Nullable(String), + properties_hs_analytics_source Nullable(String), + properties_hs_deal_stage_probability Nullable(String), + properties_hs_manual_forecast_category Nullable(String), + properties_closed_lost_reason Nullable(String), + _airbyte_raw_id String DEFAULT toString(generateUUIDv4()), + _airbyte_extracted_at DateTime64(3) DEFAULT now64(3), + _airbyte_meta String DEFAULT '{}', + _airbyte_generation_id UInt32 DEFAULT 0 +) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key + SETTINGS allow_nullable_key = 1; +SQL +fi +if ! ch_table_exists bronze_hubspot owners; then + echo " Creating placeholder: bronze_hubspot.owners" + run_ch <<'SQL' +CREATE TABLE IF NOT EXISTS bronze_hubspot.owners ( + tenant_id Nullable(String), + source_id Nullable(String), + unique_key Nullable(String), + id Nullable(String), + email Nullable(String), + userId Nullable(Int64), + firstName Nullable(String), + lastName Nullable(String), + archived Nullable(Bool), + createdAt Nullable(DateTime64(3)), + updatedAt Nullable(DateTime64(3)), + archivedAt Nullable(DateTime64(3)), + collected_at Nullable(DateTime64(3)), + data_source Nullable(String), + _airbyte_raw_id String DEFAULT toString(generateUUIDv4()), + _airbyte_extracted_at DateTime64(3) DEFAULT now64(3), + _airbyte_meta String DEFAULT '{}', + _airbyte_generation_id UInt32 DEFAULT 0 +) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key + SETTINGS allow_nullable_key = 1; +SQL +fi +if ! ch_table_exists bronze_hubspot engagements_calls; then + echo " Creating placeholder: bronze_hubspot.engagements_calls" + run_ch <<'SQL' +CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_calls ( + tenant_id Nullable(String), + source_id Nullable(String), + unique_key Nullable(String), + id Nullable(String), + archived Nullable(Bool), + createdAt Nullable(DateTime64(3)), + updatedAt Nullable(DateTime64(3)), + archivedAt Nullable(DateTime64(3)), + collected_at Nullable(DateTime64(3)), + data_source Nullable(String), + associations_contacts Nullable(String), + associations_deals Nullable(String), + associations_companies Nullable(String), + properties_hubspot_owner_id Nullable(String), + properties_hs_created_by_user_id Nullable(String), + properties_hs_timestamp Nullable(String), + properties_hs_call_duration Nullable(String), + properties_hs_call_disposition Nullable(String), + properties_hs_call_title Nullable(String), + properties_hs_call_direction Nullable(String), + _airbyte_raw_id String DEFAULT toString(generateUUIDv4()), + _airbyte_extracted_at DateTime64(3) DEFAULT now64(3), + _airbyte_meta String DEFAULT '{}', + _airbyte_generation_id UInt32 DEFAULT 0 +) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key + SETTINGS allow_nullable_key = 1; +SQL +fi +if ! ch_table_exists bronze_hubspot engagements_emails; then + echo " Creating placeholder: bronze_hubspot.engagements_emails" + run_ch <<'SQL' +CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_emails ( + tenant_id Nullable(String), + source_id Nullable(String), + unique_key Nullable(String), + id Nullable(String), + archived Nullable(Bool), + createdAt Nullable(DateTime64(3)), + updatedAt Nullable(DateTime64(3)), + archivedAt Nullable(DateTime64(3)), + collected_at Nullable(DateTime64(3)), + data_source Nullable(String), + associations_contacts Nullable(String), + associations_deals Nullable(String), + associations_companies Nullable(String), + properties_hubspot_owner_id Nullable(String), + properties_hs_created_by_user_id Nullable(String), + properties_hs_timestamp Nullable(String), + properties_hs_email_status Nullable(String), + properties_hs_email_subject Nullable(String), + properties_hs_email_direction Nullable(String), + _airbyte_raw_id String DEFAULT toString(generateUUIDv4()), + _airbyte_extracted_at DateTime64(3) DEFAULT now64(3), + _airbyte_meta String DEFAULT '{}', + _airbyte_generation_id UInt32 DEFAULT 0 +) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key + SETTINGS allow_nullable_key = 1; +SQL +fi +if ! ch_table_exists bronze_hubspot engagements_meetings; then + echo " Creating placeholder: bronze_hubspot.engagements_meetings" + run_ch <<'SQL' +CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_meetings ( + tenant_id Nullable(String), + source_id Nullable(String), + unique_key Nullable(String), + id Nullable(String), + archived Nullable(Bool), + createdAt Nullable(DateTime64(3)), + updatedAt Nullable(DateTime64(3)), + archivedAt Nullable(DateTime64(3)), + collected_at Nullable(DateTime64(3)), + data_source Nullable(String), + associations_contacts Nullable(String), + associations_deals Nullable(String), + associations_companies Nullable(String), + properties_hubspot_owner_id Nullable(String), + properties_hs_created_by_user_id Nullable(String), + properties_hs_timestamp Nullable(String), + properties_hs_meeting_start_time Nullable(String), + properties_hs_meeting_end_time Nullable(String), + properties_hs_meeting_outcome Nullable(String), + properties_hs_meeting_title Nullable(String), + properties_hs_meeting_location Nullable(String), + _airbyte_raw_id String DEFAULT toString(generateUUIDv4()), + _airbyte_extracted_at DateTime64(3) DEFAULT now64(3), + _airbyte_meta String DEFAULT '{}', + _airbyte_generation_id UInt32 DEFAULT 0 +) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key + SETTINGS allow_nullable_key = 1; +SQL +fi +if ! ch_table_exists bronze_hubspot engagements_tasks; then + echo " Creating placeholder: bronze_hubspot.engagements_tasks" + run_ch <<'SQL' +CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_tasks ( + tenant_id Nullable(String), + source_id Nullable(String), + unique_key Nullable(String), + id Nullable(String), + archived Nullable(Bool), + createdAt Nullable(DateTime64(3)), + updatedAt Nullable(DateTime64(3)), + archivedAt Nullable(DateTime64(3)), + collected_at Nullable(DateTime64(3)), + data_source Nullable(String), + associations_contacts Nullable(String), + associations_deals Nullable(String), + associations_companies Nullable(String), + properties_hubspot_owner_id Nullable(String), + properties_hs_created_by_user_id Nullable(String), + properties_hs_timestamp Nullable(String), + properties_hs_task_status Nullable(String), + properties_hs_task_subject Nullable(String), + properties_hs_task_priority Nullable(String), + properties_hs_task_type Nullable(String), + _airbyte_raw_id String DEFAULT toString(generateUUIDv4()), + _airbyte_extracted_at DateTime64(3) DEFAULT now64(3), + _airbyte_meta String DEFAULT '{}', + _airbyte_generation_id UInt32 DEFAULT 0 +) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key + SETTINGS allow_nullable_key = 1; +SQL +fi + echo "=== Placeholders: done ===" diff --git a/src/ingestion/tests/e2e/conftest.py b/src/ingestion/tests/e2e/conftest.py index 88fe0460b..4d46815ff 100644 --- a/src/ingestion/tests/e2e/conftest.py +++ b/src/ingestion/tests/e2e/conftest.py @@ -137,6 +137,18 @@ def compose_stack(session_cfg: SessionConfig): ("staging", "jira_issue_field_snapshot"), ("staging", "jira_changelog_items"), ("staging", "jira__task_field_metadata"), + # CRM (sales-rep): the crm_* gold views + bullet query_refs read + # class_crm_{deals,activities,users} even when a fixture seeds only a subset + # (e.g. the quality bullet seeds deals but not activities), and the HubSpot + # staging models are incremental/append. Reset once at session start so warm + # re-runs are deterministic (CI is fresh anyway); per-test TRUNCATE handles + # cross-test isolation within a session. + ("silver", "class_crm_deals"), + ("silver", "class_crm_activities"), + ("silver", "class_crm_users"), + ("staging", "hubspot__crm_deals"), + ("staging", "hubspot__crm_activities"), + ("staging", "hubspot__crm_users"), ] diff --git a/src/ingestion/tests/e2e/metrics/crm_bullet_activity.test.yaml b/src/ingestion/tests/e2e/metrics/crm_bullet_activity.test.yaml new file mode 100644 index 000000000..43e46b320 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/crm_bullet_activity.test.yaml @@ -0,0 +1,228 @@ +spec_version: 1 +description: > + Metric: crm_bullet_activity — outreach-activity bullets (…0023), sales-rep dashboard. + Emits four keys: calls, emails, meetings, comms_per_won. + How it's computed (bronze → silver → gold): + • bronze: HubSpot engagements (calls/emails/meetings/tasks) + won deals + • silver: activities/deals deduped; owner email → person_id; department → org_unit_id + • gold: insight.crm_bullet_rows emits one row per engagement (calls via OWNER, + emails/meetings/tasks via CREATOR) + one deals_won row per won deal. + query_ref re-aggregates per person → calls/emails/meetings = counts, + comms_per_won = (calls+emails+meetings+tasks)/deals_won. value = the + requested rep; median/min/max = the rep's department team. + + Team = department "Sales" {alice, bob, carol} (odd team → unambiguous median): + calls alice 2 · bob 1 · carol 3 → median 2, range [1, 3] + emails alice 2 · bob 1 · carol 3 → median 2, range [1, 3] + meetings alice 2 · bob 1 · carol 3 → median 2, range [1, 3] + comms_per_won alice 4 ((2+2+2+2)/2 won) · bob 3 ((1+1+1+0)/1) · carol 9 ((3+3+3+0)/1) + → median 4, range [3, 9] + Tasks feed comms_per_won only (not a bullet key). Cases: alice's value + the + department median/range for each of the four keys. + +bronze: + bronze_bamboohr.employees: + - $ref: templates/people.yaml#/templates/alice + department: Sales + - $ref: templates/people.yaml#/templates/bob + department: Sales + - $ref: templates/people.yaml#/templates/carol + department: Sales + + bronze_hubspot.owners: + - $ref: templates/hubspot.yaml#/templates/alice_owner + - $ref: templates/hubspot.yaml#/templates/bob_owner + - $ref: templates/hubspot.yaml#/templates/carol_owner + + # Won deals → deals_won denominator for comms_per_won (alice 2, bob 1, carol 1). + bronze_hubspot.deals: + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-aw1 + unique_key: hubspot-deal-aw1 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-05T09:00:00" + properties_closedate: "2026-01-10T00:00:00Z" + properties_amount_in_home_currency: "10000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-aw2 + unique_key: hubspot-deal-aw2 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-06T09:00:00" + properties_closedate: "2026-01-20T00:00:00Z" + properties_amount_in_home_currency: "30000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-bw1 + unique_key: hubspot-deal-bw1 + properties_hubspot_owner_id: own-bob + createdAt: "2026-01-05T09:00:00" + properties_closedate: "2026-01-09T00:00:00Z" + properties_amount_in_home_currency: "8000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-cw1 + unique_key: hubspot-deal-cw1 + properties_hubspot_owner_id: own-carol + createdAt: "2026-01-02T09:00:00" + properties_closedate: "2026-01-22T00:00:00Z" + properties_amount_in_home_currency: "40000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + + # calls — attributed via OWNER (properties_hubspot_owner_id). alice 2, bob 1, carol 3. + bronze_hubspot.engagements_calls: + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-aa1 + unique_key: hubspot-call-aa1 + properties_hubspot_owner_id: own-alice + properties_hs_timestamp: "2026-01-12T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-aa2 + unique_key: hubspot-call-aa2 + properties_hubspot_owner_id: own-alice + properties_hs_timestamp: "2026-01-12T10:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-ba1 + unique_key: hubspot-call-ba1 + properties_hubspot_owner_id: own-bob + properties_hs_timestamp: "2026-01-12T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-ca1 + unique_key: hubspot-call-ca1 + properties_hubspot_owner_id: own-carol + properties_hs_timestamp: "2026-01-12T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-ca2 + unique_key: hubspot-call-ca2 + properties_hubspot_owner_id: own-carol + properties_hs_timestamp: "2026-01-12T10:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-ca3 + unique_key: hubspot-call-ca3 + properties_hubspot_owner_id: own-carol + properties_hs_timestamp: "2026-01-12T11:00:00Z" + + # emails — attributed via CREATOR (properties_hs_created_by_user_id). alice 2, bob 1, carol 3. + # owner_id is also set (non-null silver constraint) to the same rep; unused for attribution. + bronze_hubspot.engagements_emails: + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-aa1 + unique_key: hubspot-email-aa1 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-13T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-aa2 + unique_key: hubspot-email-aa2 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-13T10:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-ba1 + unique_key: hubspot-email-ba1 + properties_hubspot_owner_id: own-bob + properties_hs_created_by_user_id: "102" + properties_hs_timestamp: "2026-01-13T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-ca1 + unique_key: hubspot-email-ca1 + properties_hubspot_owner_id: own-carol + properties_hs_created_by_user_id: "103" + properties_hs_timestamp: "2026-01-13T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-ca2 + unique_key: hubspot-email-ca2 + properties_hubspot_owner_id: own-carol + properties_hs_created_by_user_id: "103" + properties_hs_timestamp: "2026-01-13T10:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-ca3 + unique_key: hubspot-email-ca3 + properties_hubspot_owner_id: own-carol + properties_hs_created_by_user_id: "103" + properties_hs_timestamp: "2026-01-13T11:00:00Z" + + # meetings — attributed via CREATOR. alice 2, bob 1, carol 3. + bronze_hubspot.engagements_meetings: + - $ref: templates/hubspot.yaml#/templates/hubspot_meeting + id: meeting-aa1 + unique_key: hubspot-meeting-aa1 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-14T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_meeting + id: meeting-aa2 + unique_key: hubspot-meeting-aa2 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-14T10:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_meeting + id: meeting-ba1 + unique_key: hubspot-meeting-ba1 + properties_hubspot_owner_id: own-bob + properties_hs_created_by_user_id: "102" + properties_hs_timestamp: "2026-01-14T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_meeting + id: meeting-ca1 + unique_key: hubspot-meeting-ca1 + properties_hubspot_owner_id: own-carol + properties_hs_created_by_user_id: "103" + properties_hs_timestamp: "2026-01-14T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_meeting + id: meeting-ca2 + unique_key: hubspot-meeting-ca2 + properties_hubspot_owner_id: own-carol + properties_hs_created_by_user_id: "103" + properties_hs_timestamp: "2026-01-14T10:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_meeting + id: meeting-ca3 + unique_key: hubspot-meeting-ca3 + properties_hubspot_owner_id: own-carol + properties_hs_created_by_user_id: "103" + properties_hs_timestamp: "2026-01-14T11:00:00Z" + + # tasks — attributed via CREATOR; feed comms_per_won only. alice 2, bob/carol 0. + bronze_hubspot.engagements_tasks: + - $ref: templates/hubspot.yaml#/templates/hubspot_task + id: task-aa1 + unique_key: hubspot-task-aa1 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-15T09:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_task + id: task-aa2 + unique_key: hubspot-task-aa2 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-15T10:00:00Z" + +cases: + - name: "crm_bullet_activity — alice value + Sales team median/range (4 keys)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000023 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + expect: + - assert: "status == 200" + - in: crm + assert: "result.status == 'ok'" + - in: crm + find: { metric_key: calls } + equal: { value: 2, median: 2, range_min: 1, range_max: 3 } + - in: crm + find: { metric_key: emails } + equal: { value: 2, median: 2, range_min: 1, range_max: 3 } + - in: crm + find: { metric_key: meetings } + equal: { value: 2, median: 2, range_min: 1, range_max: 3 } + - in: crm + find: { metric_key: comms_per_won } + equal: { value: 4, median: 4, range_min: 3, range_max: 9 } diff --git a/src/ingestion/tests/e2e/metrics/crm_bullet_quality.test.yaml b/src/ingestion/tests/e2e/metrics/crm_bullet_quality.test.yaml new file mode 100644 index 000000000..5dcf9ad30 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/crm_bullet_quality.test.yaml @@ -0,0 +1,151 @@ +spec_version: 1 +description: > + Metric: crm_bullet_velocity_quality — V&Q bullets (…0022), sales-rep dashboard. + Emits four keys: deals_opened, win_rate, avg_deal_size, cycle_days. + How it's computed (bronze → silver → gold): + • bronze: HubSpot deals (owner_id, createdAt, closedate, amount_home, is_closed/is_won) + • silver: deals deduped; owner email → person_id; department → org_unit_id + • gold: insight.crm_bullet_rows emits one long-format row per deal event + (deals_opened / deals_closed / deals_won / cycle_days / deal_size). + query_ref re-aggregates per person → win_rate = won·100/closed, + avg_deal_size = avg(amount_home of won), cycle_days = avg(close−created, + floored at 0). value = the requested rep; median/min/max = the rep's + department team (quantileExact / min / max over org_unit_id). + + Team = department "Sales" {alice, bob, carol} (odd team → unambiguous median): + deals_opened alice 5 · bob 2 · carol 3 → median 3, range [2, 5] + win_rate alice 66.67 (2/3) · bob 50 (1/2) · carol 100 (1/1) → median 66.67, range [50, 100] + avg_deal_size alice 20000 (10k,30k) · bob 8000 · carol 40000 → median 20000, range [8000, 40000] + cycle_days alice 9.5 (5,14) · bob 4 · carol 20 → median 9.5, range [4, 20] + Cases: alice's value + the department median/range for each of the four keys. + +bronze: + bronze_bamboohr.employees: + - $ref: templates/people.yaml#/templates/alice + department: Sales + - $ref: templates/people.yaml#/templates/bob + department: Sales + - $ref: templates/people.yaml#/templates/carol + department: Sales + + bronze_hubspot.owners: + - $ref: templates/hubspot.yaml#/templates/alice_owner + - $ref: templates/hubspot.yaml#/templates/bob_owner + - $ref: templates/hubspot.yaml#/templates/carol_owner + + bronze_hubspot.deals: + # ── alice: 5 opened, 3 closed, 2 won (10k @5d, 30k @14d), 1 lost, 2 open ── + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-aq1 + unique_key: hubspot-deal-aq1 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-05T09:00:00" + properties_closedate: "2026-01-10T00:00:00Z" + properties_amount_in_home_currency: "10000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-aq2 + unique_key: hubspot-deal-aq2 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-06T09:00:00" + properties_closedate: "2026-01-20T00:00:00Z" + properties_amount_in_home_currency: "30000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-aq3 + unique_key: hubspot-deal-aq3 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-07T09:00:00" + properties_closedate: "2026-01-15T00:00:00Z" + properties_amount_in_home_currency: "5000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "false" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-aq4 + unique_key: hubspot-deal-aq4 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-08T09:00:00" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-aq5 + unique_key: hubspot-deal-aq5 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-09T09:00:00" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + # ── bob: 2 opened, 2 closed, 1 won (8k @4d), 1 lost ── + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-bq1 + unique_key: hubspot-deal-bq1 + properties_hubspot_owner_id: own-bob + createdAt: "2026-01-05T09:00:00" + properties_closedate: "2026-01-09T00:00:00Z" + properties_amount_in_home_currency: "8000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-bq2 + unique_key: hubspot-deal-bq2 + properties_hubspot_owner_id: own-bob + createdAt: "2026-01-06T09:00:00" + properties_closedate: "2026-01-12T00:00:00Z" + properties_amount_in_home_currency: "4000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "false" + # ── carol: 3 opened, 1 closed, 1 won (40k @20d), 2 open ── + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-cq1 + unique_key: hubspot-deal-cq1 + properties_hubspot_owner_id: own-carol + createdAt: "2026-01-02T09:00:00" + properties_closedate: "2026-01-22T00:00:00Z" + properties_amount_in_home_currency: "40000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-cq2 + unique_key: hubspot-deal-cq2 + properties_hubspot_owner_id: own-carol + createdAt: "2026-01-03T09:00:00" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-cq3 + unique_key: hubspot-deal-cq3 + properties_hubspot_owner_id: own-carol + createdAt: "2026-01-04T09:00:00" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + +cases: + - name: "crm_bullet_quality — alice value + Sales team median/range (4 keys)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000022 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + expect: + - assert: "status == 200" + - in: crm + assert: "result.status == 'ok'" + - in: crm + find: { metric_key: deals_opened } + equal: { value: 5, median: 3, range_min: 2, range_max: 5 } + - in: crm + find: { metric_key: avg_deal_size } + equal: { value: 20000, median: 20000, range_min: 8000, range_max: 40000 } + - in: crm + find: { metric_key: cycle_days } + equal: { value: 9.5, median: 9.5, range_min: 4, range_max: 20 } + - in: crm + find: { metric_key: win_rate } + assert: > + double(it.value) > 66.6 && double(it.value) < 66.7 && + double(it.median) > 66.6 && double(it.median) < 66.7 && + double(it.range_min) == 50.0 && double(it.range_max) == 100.0 diff --git a/src/ingestion/tests/e2e/metrics/crm_chart_flow.test.yaml b/src/ingestion/tests/e2e/metrics/crm_chart_flow.test.yaml new file mode 100644 index 000000000..477ef9b2e --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/crm_chart_flow.test.yaml @@ -0,0 +1,108 @@ +spec_version: 1 +description: > + Metric: crm_chart_flow — weekly deal-flow chart (…0021), sales-rep dashboard. + How it's computed (bronze → silver → gold): + • bronze: HubSpot deals (owner_id, createdAt, closedate, is_closed/is_won) + • silver: deals deduped to one row per record; owner email → person_id + • gold: per ISO week (toMonday) per rep — opened = deals created that week; + closed = deals with close_date that week; won = closed deals with is_won. + metric_date = the Monday (string 'YYYY-MM-DD'); date_bucket = '%b %d'. + + Single rep (alice). 4 deals spread over 3 weeks (Mondays Jan 5 / 12 / 19): + wk Jan 5 — C1 created+won that week + wk Jan 12 — C2 created+won, C3 created (open) + wk Jan 19 — C4 created, closed-LOST (won 0) + Cases: one per week — proves the weekly bucketing and that a lost deal counts + as closed but not won. + +bronze: + bronze_bamboohr.employees: + - $ref: templates/people.yaml#/templates/alice + department: Sales + + bronze_hubspot.owners: + - $ref: templates/hubspot.yaml#/templates/alice_owner + + bronze_hubspot.deals: + # C1 — wk Jan 5: created 01-06, closed-won 01-08 + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-c1 + unique_key: hubspot-deal-c1 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-06T09:00:00" + properties_closedate: "2026-01-08T00:00:00Z" + properties_amount_in_home_currency: "10000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + # C2 — wk Jan 12: created 01-13, closed-won 01-14 + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-c2 + unique_key: hubspot-deal-c2 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-13T09:00:00" + properties_closedate: "2026-01-14T00:00:00Z" + properties_amount_in_home_currency: "20000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + # C3 — wk Jan 12: created 01-13, still open (opened only) + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-c3 + unique_key: hubspot-deal-c3 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-13T15:00:00" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + # C4 — wk Jan 19: created 01-20, closed-LOST 01-22 (closed 1, won 0) + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-c4 + unique_key: hubspot-deal-c4 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-20T09:00:00" + properties_closedate: "2026-01-22T00:00:00Z" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "false" + +cases: + - name: "crm_chart_flow — week of Jan 5 (opened 1, closed 1, won 1)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000021 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + expect: + - assert: "status == 200" + - in: crm + assert: "result.status == 'ok'" + # opened/closed/won are UInt64 → JSONEachRow quotes them as strings; cast with int(). + - in: crm + find: { metric_date: "2026-01-05" } + assert: "it.date_bucket == 'Jan 05' && int(it.opened) == 1 && int(it.closed) == 1 && int(it.won) == 1" + - name: "crm_chart_flow — week of Jan 12 (opened 2, closed 1, won 1)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000021 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + expect: + - in: crm + find: { metric_date: "2026-01-12" } + assert: "it.date_bucket == 'Jan 12' && int(it.opened) == 2 && int(it.closed) == 1 && int(it.won) == 1" + - name: "crm_chart_flow — week of Jan 19 (opened 1, closed 1, won 0 — lost deal)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000021 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + expect: + - in: crm + find: { metric_date: "2026-01-19" } + assert: "it.date_bucket == 'Jan 19' && int(it.opened) == 1 && int(it.closed) == 1 && int(it.won) == 0" diff --git a/src/ingestion/tests/e2e/metrics/crm_kpis.test.yaml b/src/ingestion/tests/e2e/metrics/crm_kpis.test.yaml new file mode 100644 index 000000000..cab5d65a1 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/crm_kpis.test.yaml @@ -0,0 +1,155 @@ +spec_version: 1 +description: > + Metric: crm_kpis — CRM KPIs hero strip (…0020), sales-rep dashboard. + How it's computed (bronze → silver → gold): + • bronze: HubSpot deals (owner_id, createdAt, closedate, amount_home, is_closed/is_won) + + engagements (calls/emails/meetings/tasks) + owners (id/userId/email) + • silver: deals/activities/users deduped (ReplacingMergeTree) to one row per record; + owner email → person_id = lower(email) + • gold: per (person, day) — deals_opened = deals created; deals_closed/deals_won = + deals with close_date in period (won = is_won); deals_value_closed = + round(Σ amount_home of WON deals); comms_count = all engagements attributed + to the rep (calls via owner, emails/meetings/tasks via creator). + query_ref sums each over the period and groups by person_id. + + Single rep (alice). 5 deals: 2 won (10k + 30k), 1 lost-but-closed, 2 still open. + 6 engagements (2 calls, 2 emails, 1 meeting, 1 task) → comms_count 6. + Cases: alice period sums — opened 5, closed 3, won 2, value 40000, comms 6; + a duplicate deal re-sync must dedup (counts unchanged). + +bronze: + # people → silver.class_people (org_unit_id from department; not asserted here, + # but the owner email must match workEmail for identity to resolve). + bronze_bamboohr.employees: + - $ref: templates/people.yaml#/templates/alice + department: Sales + + bronze_hubspot.owners: + - $ref: templates/hubspot.yaml#/templates/alice_owner + + bronze_hubspot.deals: + # A1 won — created 01-05, closed-won 01-10, 10000 + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-a1 + unique_key: hubspot-deal-a1 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-05T09:00:00" + properties_closedate: "2026-01-10T00:00:00Z" + properties_amount_in_home_currency: "10000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + # A1 duplicate (Airbyte re-sync) — same unique_key + updatedAt → must dedup + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-a1 + unique_key: hubspot-deal-a1 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-05T09:00:00" + properties_closedate: "2026-01-10T00:00:00Z" + properties_amount_in_home_currency: "10000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + # A2 won — created 01-06, closed-won 01-20, 30000 + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-a2 + unique_key: hubspot-deal-a2 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-06T09:00:00" + properties_closedate: "2026-01-20T00:00:00Z" + properties_amount_in_home_currency: "30000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + # A3 lost-but-closed — created 01-07, closed 01-15 (counts as closed, not won) + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-a3 + unique_key: hubspot-deal-a3 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-07T09:00:00" + properties_closedate: "2026-01-15T00:00:00Z" + properties_amount_in_home_currency: "5000" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "false" + # A4 open — created 01-08 (counts as opened only) + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-a4 + unique_key: hubspot-deal-a4 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-08T09:00:00" + properties_amount_in_home_currency: "20000" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + # A5 open — created 01-09 + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-a5 + unique_key: hubspot-deal-a5 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-09T09:00:00" + properties_amount_in_home_currency: "50000" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + + # 2 calls (attributed via OWNER) + 2 emails + 1 meeting + 1 task (via CREATOR) = comms 6 + bronze_hubspot.engagements_calls: + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-a1 + unique_key: hubspot-call-a1 + properties_hubspot_owner_id: own-alice + properties_hs_timestamp: "2026-01-12T10:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_call + id: call-a2 + unique_key: hubspot-call-a2 + properties_hubspot_owner_id: own-alice + properties_hs_timestamp: "2026-01-13T10:00:00Z" + # emails/meetings/tasks attribute via CREATOR, but silver requires a non-null + # record owner (owner_id) too — set both to alice (a rep owns + logs her own work). + bronze_hubspot.engagements_emails: + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-a1 + unique_key: hubspot-email-a1 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-12T11:00:00Z" + - $ref: templates/hubspot.yaml#/templates/hubspot_email + id: email-a2 + unique_key: hubspot-email-a2 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-13T11:00:00Z" + bronze_hubspot.engagements_meetings: + - $ref: templates/hubspot.yaml#/templates/hubspot_meeting + id: meeting-a1 + unique_key: hubspot-meeting-a1 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-14T11:00:00Z" + bronze_hubspot.engagements_tasks: + - $ref: templates/hubspot.yaml#/templates/hubspot_task + id: task-a1 + unique_key: hubspot-task-a1 + properties_hubspot_owner_id: own-alice + properties_hs_created_by_user_id: "101" + properties_hs_timestamp: "2026-01-15T11:00:00Z" + +cases: + - name: "crm_kpis — alice period rollup (opened 5, closed 3, won 2, value 40000, comms 6)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000020 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + expect: + - assert: "status == 200" + - in: crm + assert: "result.status == 'ok'" + # deals_*/comms_count are UInt64 → JSONEachRow quotes them as strings; cast with int(). + # deals_value_closed is round(Float64) → a JSON number. + - in: crm + find: { person_id: alice@example.com } + assert: > + int(it.deals_opened) == 5 && int(it.deals_closed) == 3 && + int(it.deals_won) == 2 && int(it.comms_count) == 6 && + double(it.deals_value_closed) == 40000.0 + - in: crm + assert: "size(items) == 1" diff --git a/src/ingestion/tests/e2e/metrics/crm_pipeline_now.test.yaml b/src/ingestion/tests/e2e/metrics/crm_pipeline_now.test.yaml new file mode 100644 index 000000000..9ec980d33 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/crm_pipeline_now.test.yaml @@ -0,0 +1,148 @@ +spec_version: 1 +description: > + Metric: crm_pipeline_now — open-deal snapshot (…0028), sales-rep dashboard. + + ⚠️ TARGET-BEHAVIOR SPEC — CURRENTLY EXPECTED TO FAIL (RED). ⚠️ + This test asserts the INTENDED "as of end_date" semantics. The shipped backend + does NOT implement them yet (the gold view `insight.crm_pipeline_now` is + date-less — openness is the static `is_closed` flag — and analytics-api errors + the per-query when a metric_date bound is supplied because the view has no such + column). So these cases fail today ON PURPOSE; they go green only once the + metric is made end-date-aware. Making it so is a backend change across: + • gold view src/ingestion/scripts/migrations/20260512000000_crm-gold-views.sql + • query_ref src/backend/services/analytics-api/src/migration/m20260507_000001_seed_crm_metrics.rs + • (likely) the date-filter injection in analytics-api api/handlers.rs + + Intended semantics (bronze → silver → gold): + • bronze: HubSpot deals (owner_id, is_closed, close_date, amount_home) + • silver: deals deduped to one row per record; owner email → person_id + • gold: Pipeline Now is a point-in-time snapshot AS OF the period END: + a deal is "open as of end_date" ⇔ is_closed = 0 OR close_date > end_date. + pipeline_count = #such deals; pipeline_value = round(Σ amount_home). + The period START must NOT affect the result. + + Consequences asserted below: + • a still-open deal counts as pipeline; + • a deal closed ON/BEFORE end_date is excluded (it belongs to the closed + metric — see crm_kpis deals_closed); + • a deal closed AFTER end_date counts as pipeline (it was still open then); + • moving the END across a deal's close_date changes the snapshot; + • moving only the START does not change anything. + + Single rep (alice). 2 always-open deals (20k + 50k), 1 closed-before-end (10k, + excluded), 1 closed-AFTER the Jan end (88888, must be IN the Jan snapshot). + +bronze: + bronze_bamboohr.employees: + - $ref: templates/people.yaml#/templates/alice + department: Sales + + bronze_hubspot.owners: + - $ref: templates/hubspot.yaml#/templates/alice_owner + + bronze_hubspot.deals: + # D1 open — 20000 (open as of any end_date) + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-d1 + unique_key: hubspot-deal-d1 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-05T09:00:00" + properties_amount_in_home_currency: "20000" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + # D1 duplicate (re-sync) — same unique_key + updatedAt → must dedup + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-d1 + unique_key: hubspot-deal-d1 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-05T09:00:00" + properties_amount_in_home_currency: "20000" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + # D2 open — 50000 + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-d2 + unique_key: hubspot-deal-d2 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-06T09:00:00" + properties_amount_in_home_currency: "50000" + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + # D3 closed-won, close 2026-01-10 — closed BEFORE the Jan end → excluded + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-d3 + unique_key: hubspot-deal-d3 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-07T09:00:00" + properties_closedate: "2026-01-10T00:00:00Z" + properties_amount_in_home_currency: "99999" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "true" + # D4 closed, close 2026-12-31 — closed AFTER the Jan end → was still open as + # of 2026-01-31, so it MUST be in the January pipeline snapshot (88888). + - $ref: templates/hubspot.yaml#/templates/hubspot_deal + id: deal-d4 + unique_key: hubspot-deal-d4 + properties_hubspot_owner_id: own-alice + createdAt: "2026-01-08T09:00:00" + properties_closedate: "2026-12-31T00:00:00Z" + properties_amount_in_home_currency: "88888" + properties_hs_is_closed: "true" + properties_hs_is_closed_won: "false" + +cases: + # ── 1. Snapshot as of 2026-01-31: open = D1 + D2 + D4 (D4 closes later) ───── + # Expected (target): count 3, value 158888. D3 (closed 01-10) excluded. + - name: "crm_pipeline_now — as of end 2026-01-31 (open incl. closed-after-end: count 3, value 158888)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000028 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + expect: + - assert: "status == 200" + - in: crm + assert: "result.status == 'ok'" + - in: crm + find: { person_id: alice@example.com } + assert: "int(it.pipeline_count) == 3 && double(it.pipeline_value) == 158888.0" + + # ── 2. Same END, later START → identical result (START must not matter) ───── + - name: "crm_pipeline_now — period START does not affect the snapshot (still 3 / 158888)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000028 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-20' and metric_date le '2026-01-31'" + expect: + - assert: "status == 200" + - in: crm + assert: "result.status == 'ok'" + - in: crm + find: { person_id: alice@example.com } + assert: "int(it.pipeline_count) == 3 && double(it.pipeline_value) == 158888.0" + + # ── 3. Move END past D4's close (2027-01-31) → D4 now closed → drops out ──── + # Expected (target): count 2, value 70000 (only D1 + D2 still open). + - name: "crm_pipeline_now — moving END past a close_date drops that deal (count 2, value 70000)" + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: crm + metric_id: 00000000-0000-0000-0001-000000000028 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2027-01-31'" + expect: + - assert: "status == 200" + - in: crm + assert: "result.status == 'ok'" + - in: crm + find: { person_id: alice@example.com } + assert: "int(it.pipeline_count) == 2 && double(it.pipeline_value) == 70000.0" diff --git a/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.deals.yaml b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.deals.yaml new file mode 100644 index 000000000..68947b6b4 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.deals.yaml @@ -0,0 +1,43 @@ +# Reusable JSON schema for bronze_hubspot.deals (the columns hubspot__crm_deals +# reads). HubSpot lands every `properties_*` as Nullable(String); dbt coerces +# (toFloat64OrNull, parseDateTime64BestEffortOrNull, 'true'/'false' compares). +# createdAt/updatedAt/archivedAt/collected_at are DateTime64 — seed ISO strings. +schemas: + bronze_hubspot.deals: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: false + properties: + _airbyte_raw_id: { type: string } + _airbyte_extracted_at: { type: string } + _airbyte_meta: { type: string } + _airbyte_generation_id: { type: integer } + tenant_id: { type: [string, "null"] } + source_id: { type: [string, "null"] } + unique_key: { type: [string, "null"] } + id: { type: [string, "null"] } + archived: { type: [boolean, "null"] } + createdAt: { type: [string, "null"] } + updatedAt: { type: [string, "null"] } + archivedAt: { type: [string, "null"] } + collected_at: { type: [string, "null"] } + data_source: { type: [string, "null"] } + associations_companies: { type: [string, "null"] } + properties_dealname: { type: [string, "null"] } + properties_dealstage: { type: [string, "null"] } + properties_dealtype: { type: [string, "null"] } + properties_pipeline: { type: [string, "null"] } + properties_closedate: { type: [string, "null"] } + properties_amount: { type: [string, "null"] } + properties_amount_in_home_currency: { type: [string, "null"] } + properties_hs_acv: { type: [string, "null"] } + properties_hs_tcv: { type: [string, "null"] } + properties_hs_arr: { type: [string, "null"] } + properties_hs_is_closed: { type: [string, "null"] } + properties_hs_is_closed_won: { type: [string, "null"] } + properties_hubspot_owner_id: { type: [string, "null"] } + properties_hs_created_by_user_id: { type: [string, "null"] } + properties_hs_analytics_source: { type: [string, "null"] } + properties_hs_deal_stage_probability: { type: [string, "null"] } + properties_hs_manual_forecast_category: { type: [string, "null"] } + properties_closed_lost_reason: { type: [string, "null"] } diff --git a/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_calls.yaml b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_calls.yaml new file mode 100644 index 000000000..83aa19ba8 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_calls.yaml @@ -0,0 +1,34 @@ +# Reusable JSON schema for bronze_hubspot.engagements_calls (the columns the +# `calls` arm of hubspot__crm_activities reads). Calls are attributed to the +# record OWNER (properties_hubspot_owner_id) — HubSpot never sets a creator on +# call records. timestamp falls back hs_timestamp → createdAt → epoch. +schemas: + bronze_hubspot.engagements_calls: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: false + properties: + _airbyte_raw_id: { type: string } + _airbyte_extracted_at: { type: string } + _airbyte_meta: { type: string } + _airbyte_generation_id: { type: integer } + tenant_id: { type: [string, "null"] } + source_id: { type: [string, "null"] } + unique_key: { type: [string, "null"] } + id: { type: [string, "null"] } + archived: { type: [boolean, "null"] } + createdAt: { type: [string, "null"] } + updatedAt: { type: [string, "null"] } + archivedAt: { type: [string, "null"] } + collected_at: { type: [string, "null"] } + data_source: { type: [string, "null"] } + associations_contacts: { type: [string, "null"] } + associations_deals: { type: [string, "null"] } + associations_companies: { type: [string, "null"] } + properties_hubspot_owner_id: { type: [string, "null"] } + properties_hs_created_by_user_id: { type: [string, "null"] } + properties_hs_timestamp: { type: [string, "null"] } + properties_hs_call_duration: { type: [string, "null"] } + properties_hs_call_disposition: { type: [string, "null"] } + properties_hs_call_title: { type: [string, "null"] } + properties_hs_call_direction: { type: [string, "null"] } diff --git a/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_emails.yaml b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_emails.yaml new file mode 100644 index 000000000..3a137f120 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_emails.yaml @@ -0,0 +1,33 @@ +# Reusable JSON schema for bronze_hubspot.engagements_emails (the columns the +# `emails` arm of hubspot__crm_activities reads). Emails are attributed to the +# CREATOR (properties_hs_created_by_user_id → owners.hs_user_id), no owner +# fallback — matches HubSpot's "Activities by user" report. +schemas: + bronze_hubspot.engagements_emails: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: false + properties: + _airbyte_raw_id: { type: string } + _airbyte_extracted_at: { type: string } + _airbyte_meta: { type: string } + _airbyte_generation_id: { type: integer } + tenant_id: { type: [string, "null"] } + source_id: { type: [string, "null"] } + unique_key: { type: [string, "null"] } + id: { type: [string, "null"] } + archived: { type: [boolean, "null"] } + createdAt: { type: [string, "null"] } + updatedAt: { type: [string, "null"] } + archivedAt: { type: [string, "null"] } + collected_at: { type: [string, "null"] } + data_source: { type: [string, "null"] } + associations_contacts: { type: [string, "null"] } + associations_deals: { type: [string, "null"] } + associations_companies: { type: [string, "null"] } + properties_hubspot_owner_id: { type: [string, "null"] } + properties_hs_created_by_user_id: { type: [string, "null"] } + properties_hs_timestamp: { type: [string, "null"] } + properties_hs_email_status: { type: [string, "null"] } + properties_hs_email_subject: { type: [string, "null"] } + properties_hs_email_direction: { type: [string, "null"] } diff --git a/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_meetings.yaml b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_meetings.yaml new file mode 100644 index 000000000..40c57ac89 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_meetings.yaml @@ -0,0 +1,35 @@ +# Reusable JSON schema for bronze_hubspot.engagements_meetings (the columns the +# `meetings` arm of hubspot__crm_activities reads). Meetings are attributed to +# the CREATOR (properties_hs_created_by_user_id). timestamp falls back +# meeting_start_time → hs_timestamp → createdAt → epoch. +schemas: + bronze_hubspot.engagements_meetings: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: false + properties: + _airbyte_raw_id: { type: string } + _airbyte_extracted_at: { type: string } + _airbyte_meta: { type: string } + _airbyte_generation_id: { type: integer } + tenant_id: { type: [string, "null"] } + source_id: { type: [string, "null"] } + unique_key: { type: [string, "null"] } + id: { type: [string, "null"] } + archived: { type: [boolean, "null"] } + createdAt: { type: [string, "null"] } + updatedAt: { type: [string, "null"] } + archivedAt: { type: [string, "null"] } + collected_at: { type: [string, "null"] } + data_source: { type: [string, "null"] } + associations_contacts: { type: [string, "null"] } + associations_deals: { type: [string, "null"] } + associations_companies: { type: [string, "null"] } + properties_hubspot_owner_id: { type: [string, "null"] } + properties_hs_created_by_user_id: { type: [string, "null"] } + properties_hs_timestamp: { type: [string, "null"] } + properties_hs_meeting_start_time: { type: [string, "null"] } + properties_hs_meeting_end_time: { type: [string, "null"] } + properties_hs_meeting_outcome: { type: [string, "null"] } + properties_hs_meeting_title: { type: [string, "null"] } + properties_hs_meeting_location: { type: [string, "null"] } diff --git a/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_tasks.yaml b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_tasks.yaml new file mode 100644 index 000000000..ea8aef612 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.engagements_tasks.yaml @@ -0,0 +1,34 @@ +# Reusable JSON schema for bronze_hubspot.engagements_tasks (the columns the +# `tasks` arm of hubspot__crm_activities reads). Tasks are attributed to the +# CREATOR (properties_hs_created_by_user_id). Tasks are not surfaced as a bullet +# row but feed the comms_per_won ratio (calls+emails+meetings+tasks)/deals_won. +schemas: + bronze_hubspot.engagements_tasks: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: false + properties: + _airbyte_raw_id: { type: string } + _airbyte_extracted_at: { type: string } + _airbyte_meta: { type: string } + _airbyte_generation_id: { type: integer } + tenant_id: { type: [string, "null"] } + source_id: { type: [string, "null"] } + unique_key: { type: [string, "null"] } + id: { type: [string, "null"] } + archived: { type: [boolean, "null"] } + createdAt: { type: [string, "null"] } + updatedAt: { type: [string, "null"] } + archivedAt: { type: [string, "null"] } + collected_at: { type: [string, "null"] } + data_source: { type: [string, "null"] } + associations_contacts: { type: [string, "null"] } + associations_deals: { type: [string, "null"] } + associations_companies: { type: [string, "null"] } + properties_hubspot_owner_id: { type: [string, "null"] } + properties_hs_created_by_user_id: { type: [string, "null"] } + properties_hs_timestamp: { type: [string, "null"] } + properties_hs_task_status: { type: [string, "null"] } + properties_hs_task_subject: { type: [string, "null"] } + properties_hs_task_priority: { type: [string, "null"] } + properties_hs_task_type: { type: [string, "null"] } diff --git a/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.owners.yaml b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.owners.yaml new file mode 100644 index 000000000..4ab6fbc43 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/schemas/bronze_hubspot.owners.yaml @@ -0,0 +1,27 @@ +# Reusable JSON schema for bronze_hubspot.owners (the columns hubspot__crm_users +# reads). `userId` is Nullable(Int64) (HubSpot Owners `userId`) — dbt casts it to +# String for `hs_user_id`. `id` is the owner record id used as `user_id`. +schemas: + bronze_hubspot.owners: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: false + properties: + _airbyte_raw_id: { type: string } + _airbyte_extracted_at: { type: string } + _airbyte_meta: { type: string } + _airbyte_generation_id: { type: integer } + tenant_id: { type: [string, "null"] } + source_id: { type: [string, "null"] } + unique_key: { type: [string, "null"] } + id: { type: [string, "null"] } + email: { type: [string, "null"] } + userId: { type: [integer, "null"] } + firstName: { type: [string, "null"] } + lastName: { type: [string, "null"] } + archived: { type: [boolean, "null"] } + createdAt: { type: [string, "null"] } + updatedAt: { type: [string, "null"] } + archivedAt: { type: [string, "null"] } + collected_at: { type: [string, "null"] } + data_source: { type: [string, "null"] } diff --git a/src/ingestion/tests/e2e/metrics/templates/hubspot.yaml b/src/ingestion/tests/e2e/metrics/templates/hubspot.yaml new file mode 100644 index 000000000..fb22e8d08 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/templates/hubspot.yaml @@ -0,0 +1,212 @@ +# Reusable HubSpot CRM bronze records (owners + deals + the four engagement +# streams). A record is a plain field map; an optional `$ref` inherits from +# another record and sibling keys override it (closest wins). The nested +# `$ref: "#/..."` below stays local to this file. +# +# Each base record lists EVERY schema column so a merged bronze row is complete; +# unused fields default to null. The four `_airbyte_*` CDK columns are included +# (transforms read updatedAt → `_version`, createdAt → metric_date, etc.). +# +# Per the test-composition rule, templates carry ONLY invariant scaffolding: +# the envelope (source_id/tenant_id/_airbyte_*), and the owner identity records +# (id/userId/email — the join keys). Every scenario field that drives an +# assertion (amounts, dates, is_won, activity_type counts, owner attribution) +# is written INLINE in each *.test.yaml via `$ref` + sibling overrides. +# +# Identity wiring (see crm-gold-views.sql): +# * owner `id` → deals/activities `properties_hubspot_owner_id` (owner side; deals + calls) +# * owner `userId` → `hs_user_id` (toString) ↔ `properties_hs_created_by_user_id` (creator side; emails/meetings/tasks) +# * person_id = lower(owner.email); org_unit_id = department of the matching bamboohr employee +# → the owner email MUST equal the bamboohr workEmail case-insensitively. +templates: + # ---- owners (→ silver.class_crm_users) ----------------------------------- + hubspot_owner: + _airbyte_raw_id: "00000000-0000-0000-0000-000000000000" + _airbyte_extracted_at: "2026-01-05T00:00:00" + _airbyte_meta: "{}" + _airbyte_generation_id: 0 + tenant_id: "00000000-0000-0000-0000-000000000000" + source_id: hubspot-test + unique_key: null + id: null + email: null + userId: null + firstName: null + lastName: null + archived: false + createdAt: null + updatedAt: "2026-01-01T00:00:00" + archivedAt: null + collected_at: "2026-01-05T00:00:00" + data_source: hubspot + + alice_owner: + $ref: "#/templates/hubspot_owner" + id: own-alice + unique_key: hubspot-owner-alice + email: alice@example.com + userId: 101 + firstName: Alice + lastName: Alpha + + bob_owner: + $ref: "#/templates/hubspot_owner" + id: own-bob + unique_key: hubspot-owner-bob + email: bob@example.com + userId: 102 + firstName: Bob + lastName: Beta + + carol_owner: + $ref: "#/templates/hubspot_owner" + id: own-carol + unique_key: hubspot-owner-carol + email: carol@example.com + userId: 103 + firstName: Carol + lastName: Gamma + + # ---- deals (→ silver.class_crm_deals) ------------------------------------ + hubspot_deal: + _airbyte_raw_id: "00000000-0000-0000-0000-000000000000" + _airbyte_extracted_at: "2026-01-05T00:00:00" + _airbyte_meta: "{}" + _airbyte_generation_id: 0 + tenant_id: "00000000-0000-0000-0000-000000000000" + source_id: hubspot-test + unique_key: null + id: null + archived: false + createdAt: null + updatedAt: "2026-01-01T00:00:00" + archivedAt: null + collected_at: "2026-01-05T00:00:00" + data_source: hubspot + associations_companies: null + properties_dealname: null + properties_dealstage: null + properties_dealtype: null + properties_pipeline: null + properties_closedate: null + properties_amount: null + properties_amount_in_home_currency: null + properties_hs_acv: null + properties_hs_tcv: null + properties_hs_arr: null + properties_hs_is_closed: "false" + properties_hs_is_closed_won: "false" + properties_hubspot_owner_id: null + properties_hs_created_by_user_id: null + properties_hs_analytics_source: null + properties_hs_deal_stage_probability: null + properties_hs_manual_forecast_category: null + properties_closed_lost_reason: null + + # ---- engagement: calls (→ silver.class_crm_activities) ------------------- + hubspot_call: + _airbyte_raw_id: "00000000-0000-0000-0000-000000000000" + _airbyte_extracted_at: "2026-01-05T00:00:00" + _airbyte_meta: "{}" + _airbyte_generation_id: 0 + tenant_id: "00000000-0000-0000-0000-000000000000" + source_id: hubspot-test + unique_key: null + id: null + archived: false + createdAt: null + updatedAt: "2026-01-01T00:00:00" + archivedAt: null + collected_at: "2026-01-05T00:00:00" + data_source: hubspot + associations_contacts: null + associations_deals: null + associations_companies: null + properties_hubspot_owner_id: null + properties_hs_created_by_user_id: null + properties_hs_timestamp: null + properties_hs_call_duration: null + properties_hs_call_disposition: null + properties_hs_call_title: null + properties_hs_call_direction: null + + # ---- engagement: emails -------------------------------------------------- + hubspot_email: + _airbyte_raw_id: "00000000-0000-0000-0000-000000000000" + _airbyte_extracted_at: "2026-01-05T00:00:00" + _airbyte_meta: "{}" + _airbyte_generation_id: 0 + tenant_id: "00000000-0000-0000-0000-000000000000" + source_id: hubspot-test + unique_key: null + id: null + archived: false + createdAt: null + updatedAt: "2026-01-01T00:00:00" + archivedAt: null + collected_at: "2026-01-05T00:00:00" + data_source: hubspot + associations_contacts: null + associations_deals: null + associations_companies: null + properties_hubspot_owner_id: null + properties_hs_created_by_user_id: null + properties_hs_timestamp: null + properties_hs_email_status: null + properties_hs_email_subject: null + properties_hs_email_direction: null + + # ---- engagement: meetings ------------------------------------------------ + hubspot_meeting: + _airbyte_raw_id: "00000000-0000-0000-0000-000000000000" + _airbyte_extracted_at: "2026-01-05T00:00:00" + _airbyte_meta: "{}" + _airbyte_generation_id: 0 + tenant_id: "00000000-0000-0000-0000-000000000000" + source_id: hubspot-test + unique_key: null + id: null + archived: false + createdAt: null + updatedAt: "2026-01-01T00:00:00" + archivedAt: null + collected_at: "2026-01-05T00:00:00" + data_source: hubspot + associations_contacts: null + associations_deals: null + associations_companies: null + properties_hubspot_owner_id: null + properties_hs_created_by_user_id: null + properties_hs_timestamp: null + properties_hs_meeting_start_time: null + properties_hs_meeting_end_time: null + properties_hs_meeting_outcome: null + properties_hs_meeting_title: null + properties_hs_meeting_location: null + + # ---- engagement: tasks --------------------------------------------------- + hubspot_task: + _airbyte_raw_id: "00000000-0000-0000-0000-000000000000" + _airbyte_extracted_at: "2026-01-05T00:00:00" + _airbyte_meta: "{}" + _airbyte_generation_id: 0 + tenant_id: "00000000-0000-0000-0000-000000000000" + source_id: hubspot-test + unique_key: null + id: null + archived: false + createdAt: null + updatedAt: "2026-01-01T00:00:00" + archivedAt: null + collected_at: "2026-01-05T00:00:00" + data_source: hubspot + associations_contacts: null + associations_deals: null + associations_companies: null + properties_hubspot_owner_id: null + properties_hs_created_by_user_id: null + properties_hs_timestamp: null + properties_hs_task_status: null + properties_hs_task_subject: null + properties_hs_task_priority: null + properties_hs_task_type: null