From 20a5afddb92b01663f82037df619b99b85c1c3e9 Mon Sep 17 00:00:00 2001 From: Roman Mitasov Date: Thu, 11 Jun 2026 22:24:20 +0300 Subject: [PATCH 1/2] feat(ingestion): add Workday HR connector (RaaS report contract, BambooHR parity) Add the Workday connector following the BambooHR pattern end to end: - Declarative manifest (CDK v7.0.4): two full-refresh streams (workers, leave_requests) fetched from customer-built RaaS custom reports with ISU Basic auth. Workday has no fixed bulk API, so the connector ships a report contract (exact column XML aliases + From_Date/To_Date prompts); extra report columns flow into raw_data. - dbt chain mirroring bamboohr model-for-model: RMT promotion, SCD2 snapshot (Last_Functionally_Updated deliberately untracked), field-level history, identity_inputs (DELETE on Worker_Status=Terminated), and class_people / class_hr_events / class_hr_working_hours staging views. - Specs: PRD + DESIGN (cpt validate PASS, registered in artifacts.toml); workday.md rewritten as the v1 overview. - Local test rig: RaaS fixtures + mock server enforcing Basic auth, format=json, and leave prompts. Verified: check SUCCEEDED, read returns 5 workers / 4 leave requests with 0 errors, raw_data passthrough of non-contract columns, deterministic full-refresh re-read. There is no public Workday developer tenant; real-tenant validation runs against the customer Sandbox during onboarding (check gates the report contract). Signed-off-by: Roman Mitasov Co-Authored-By: Claude Fable 5 --- cypilot/config/artifacts.toml | 9 +- .../hr-directory/workday/specs/DESIGN.md | 482 ++++++++++++++++++ .../hr-directory/workday/specs/PRD.md | 414 +++++++++++++++ .../hr-directory/workday/workday.md | 190 ++----- .../connectors/hr-directory/workday/README.md | 147 ++++++ .../hr-directory/workday/connector.yaml | 367 +++++++++++++ .../hr-directory/workday/dbt/schema.yml | 214 ++++++++ .../workday/dbt/workday__bronze_promoted.sql | 24 + .../workday/dbt/workday__hr_events.sql | 31 ++ .../workday/dbt/workday__identity_inputs.sql | 24 + .../workday/dbt/workday__to_class_people.sql | 58 +++ .../dbt/workday__workers_fields_history.sql | 21 + .../workday/dbt/workday__workers_snapshot.sql | 25 + .../workday/dbt/workday__working_hours.sql | 27 + .../hr-directory/workday/descriptor.yaml | 14 + .../workday/fixtures/leave_requests.json | 48 ++ .../workday/fixtures/mock_raas.py | 95 ++++ .../workday/fixtures/workers.json | 119 +++++ .../secrets/connectors/workday.yaml.example | 17 + 19 files changed, 2186 insertions(+), 140 deletions(-) create mode 100644 src/ingestion/connectors/hr-directory/workday/README.md create mode 100644 src/ingestion/connectors/hr-directory/workday/connector.yaml create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/schema.yml create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/workday__bronze_promoted.sql create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/workday__identity_inputs.sql create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/workday__to_class_people.sql create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_fields_history.sql create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_snapshot.sql create mode 100644 src/ingestion/connectors/hr-directory/workday/dbt/workday__working_hours.sql create mode 100644 src/ingestion/connectors/hr-directory/workday/descriptor.yaml create mode 100644 src/ingestion/connectors/hr-directory/workday/fixtures/leave_requests.json create mode 100644 src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py create mode 100644 src/ingestion/connectors/hr-directory/workday/fixtures/workers.json create mode 100644 src/ingestion/secrets/connectors/workday.yaml.example diff --git a/cypilot/config/artifacts.toml b/cypilot/config/artifacts.toml index 36144e872..b29fe4ed0 100644 --- a/cypilot/config/artifacts.toml +++ b/cypilot/config/artifacts.toml @@ -112,10 +112,15 @@ kind = "DESIGN" path = "docs/components/connectors/crm/salesforce/specs/DESIGN.md" name = "Salesforce Connector Design" +[[systems.artifacts]] +kind = "PRD" +path = "docs/components/connectors/hr-directory/workday/specs/PRD.md" +name = "Workday Connector PRD" + [[systems.artifacts]] kind = "DESIGN" -path = "docs/components/connectors/hr-directory/workday.md" -name = "Workday Connector Specification" +path = "docs/components/connectors/hr-directory/workday/specs/DESIGN.md" +name = "Workday Connector Design" [[systems.artifacts]] kind = "DESIGN" diff --git a/docs/components/connectors/hr-directory/workday/specs/DESIGN.md b/docs/components/connectors/hr-directory/workday/specs/DESIGN.md index e69de29bb..79dc0a8a3 100644 --- a/docs/components/connectors/hr-directory/workday/specs/DESIGN.md +++ b/docs/components/connectors/hr-directory/workday/specs/DESIGN.md @@ -0,0 +1,482 @@ +# DESIGN — Workday Connector + +- [ ] `p1` - **ID**: `cpt-insightspec-design-wd-connector` + +> Version 1.0 — June 2026 +> Based on: HR Directory domain (`docs/components/connectors/hr-directory/README.md`), [PRD.md](./PRD.md) + + + +- [1. Architecture Overview](#1-architecture-overview) + - [1.1 Architectural Vision](#11-architectural-vision) + - [1.2 Architecture Drivers](#12-architecture-drivers) + - [1.3 Architecture Layers](#13-architecture-layers) +- [2. Principles & Constraints](#2-principles--constraints) + - [2.1 Design Principles](#21-design-principles) + - [2.2 Constraints](#22-constraints) +- [3. Technical Architecture](#3-technical-architecture) + - [3.1 Domain Model](#31-domain-model) + - [3.2 Component Model](#32-component-model) + - [3.3 API Contracts](#33-api-contracts) + - [3.4 Internal Dependencies](#34-internal-dependencies) + - [3.5 External Dependencies](#35-external-dependencies) + - [3.6 Interactions & Sequences](#36-interactions--sequences) + - [3.7 Database schemas & tables](#37-database-schemas--tables) + - [3.8 Deployment Topology](#38-deployment-topology) +- [4. Additional context](#4-additional-context) + - [Identity Resolution Strategy](#identity-resolution-strategy) + - [Silver / Gold Mappings](#silver--gold-mappings) + - [Source-Specific Considerations](#source-specific-considerations) +- [5. Traceability](#5-traceability) +- [6. Non-Applicability Statements](#6-non-applicability-statements) + + + +--- + +## 1. Architecture Overview + +### 1.1 Architectural Vision + +The Workday connector is an Airbyte declarative manifest connector (YAML, no custom code) that extracts HR directory data from Workday RaaS (Reports-as-a-Service). It produces two Bronze streams: + +1. **`workers`** — worker directory via a customer-built RaaS report following the Insight report contract. +2. **`leave_requests`** — time-off requests via a second RaaS report with `From_Date`/`To_Date` prompts. + +**Authentication**: ISU (Integration System User) credentials via `BasicHttpAuthenticator`. + +**Pagination**: None. RaaS executes the report and returns the full result set per request. + +**Sync mode**: Full refresh on both streams. RaaS reports return current-state values — the BambooHR pattern. `Last_Functionally_Updated` is collected to enable future incremental extraction (PRD OQ-WD-1). + +**Field-set ownership**: Unlike BambooHR (where the connector requests an explicit field list), the field set lives in the customer's report definition. The connector ships a **report contract** (column aliases + prompts); the customer's Workday administrator implements it. Extra columns flow into `raw_data`. + +**Downstream**: Bronze data feeds the SCD2 chain (`workday__workers_snapshot` → `workday__workers_fields_history` → `workday__identity_inputs` → Identity Manager) and Silver staging (`class_people`, `class_hr_events`, `class_hr_working_hours`). + +### 1.2 Architecture Drivers + +#### Functional Drivers + +| Requirement | Design Response | +|-------------|-----------------| +| `cpt-insightspec-fr-wd-collect-workers` | Stream `workers` → `GET {base_url}/{workers_report_path}?format=json` | +| `cpt-insightspec-fr-wd-collect-leave-requests` | Stream `leave_requests` → `GET {base_url}/{leave_report_path}?format=json&From_Date={start}&To_Date={today}` | +| `cpt-insightspec-fr-wd-custom-columns` | `AddFields` puts the full record into `raw_data`; dbt var `workday_custom_fields` tracks selected keys | +| `cpt-insightspec-fr-wd-report-contract` | `CheckStream` on `workers`; explicit alias table in README; dbt `not_null` tests as the second gate | +| `cpt-insightspec-fr-wd-deduplication` | `unique_key` = `{tenant}-{source}-{Employee_ID \| Request_ID}`; RMT dedup at destination | +| `cpt-insightspec-fr-wd-identity-key` | `Work_Email` is a mandatory contract column; identity_inputs emits `email` observations | +| `cpt-insightspec-fr-wd-full-refresh-sync` | No `DatetimeBasedCursor`; both streams full refresh | +| `cpt-insightspec-fr-wd-fault-tolerance` | Default CDK retry behaviour for 5XX; auth errors fail the sync immediately | + +#### NFR Allocation + +| NFR ID | NFR Summary | Allocated To | Design Response | Verification Approach | +|--------|-------------|-------------|-----------------|----------------------| +| `cpt-insightspec-nfr-wd-auth-isu-basic` | ISU Basic auth; all connection params required, no defaults | `spec.connection_specification` | `workday_base_url`, `workday_isu_username`, `workday_isu_password`, both report paths in `required` | Verify config fields present and required in spec | +| `cpt-insightspec-nfr-wd-schema-compliance` | Contract aliases as Bronze field names | `InlineSchemaLoader` | Schemas use contract `Snake_Case` aliases; no renaming transformations | Compare schema fields to contract alias table | +| `cpt-insightspec-nfr-wd-idempotent-writes` | Deterministic dedup | `unique_key` + RMT | Same report output produces same `unique_key` set | Run twice; verify no duplicate rows after RMT merge | + +### 1.3 Architecture Layers + +| Layer | Responsibility | Technology | +|-------|---------------|------------| +| Source API | Workday RaaS custom reports | REST / JSON (`customreport2`) | +| Authentication | ISU credentials | `BasicHttpAuthenticator` | +| Connector | Stream definitions, record enrichment | Airbyte declarative manifest (YAML) | +| Execution | Container runtime | Airbyte Declarative Connector framework (CDK v7.0+) | +| Bronze | Raw data storage with contract-native schema | Destination connector (ClickHouse) | +| Silver (shipped with connector) | RMT promotion, SCD2 snapshot, field history, identity inputs, class staging | dbt models in `dbt/` | + +--- + +## 2. Principles & Constraints + +### 2.1 Design Principles + +#### Report Contract over API Contract + +- [ ] `p1` - **ID**: `cpt-insightspec-principle-wd-report-contract` + +Workday offers no fixed bulk endpoint, so the connector defines the contract instead: an exact column-alias table and prompt set the customer's reports must implement. The contract pins the response shape across tenants, making one manifest serve every customer. + +#### Contract-Native Schema + +- [ ] `p1` - **ID**: `cpt-insightspec-principle-wd-contract-native-schema` + +Bronze tables preserve the contract's column aliases (`Snake_Case`) exactly as they appear in the RaaS JSON. No renaming, no type coercion, no enum normalisation. Schema transformations are the Silver layer's responsibility. + +#### BambooHR Pipeline Parity + +- [ ] `p1` - **ID**: `cpt-insightspec-principle-wd-bamboohr-parity` + +The dbt chain mirrors BambooHR model-for-model (promotion, snapshot, fields_history, identity_inputs, class staging). Workday-specific logic is confined to column mapping and status/type normalisation. This keeps the HR-source pattern uniform and reuses proven macros unchanged. + +### 2.2 Constraints + +#### No Pagination Available + +- [ ] `p1` - **ID**: `cpt-insightspec-constraint-wd-no-pagination` + +RaaS executes the report and returns the complete result set. The connector uses `NoPagination` and relies on report execution staying within Workday's RaaS limits for the tenant's headcount. + +#### Current-State Extraction in v1 + +- [ ] `p1` - **ID**: `cpt-insightspec-constraint-wd-current-state` + +Although Workday data is natively effective-dated, a plain RaaS report returns as-of-today values. v1 extracts current state only; SCD2 history is constructed at the Silver layer by the snapshot chain. Native history extraction is a deliberate deferral (PRD OQ-WD-3). + +#### Customer-Owned Report Definitions + +- [ ] `p1` - **ID**: `cpt-insightspec-constraint-wd-customer-owned-reports` + +The reports live in the customer's tenant and are maintained by their Workday administrator. The connector cannot create or modify them. Contract conformance is enforced socially (onboarding docs, naming convention `Insight_*`) and technically (`check`, dbt tests) — not by API introspection. + +#### Silent Empty Fields on Missing Domains + +- [ ] `p2` - **ID**: `cpt-insightspec-constraint-wd-domain-security` + +If the ISU security group lacks a domain required by a report field, Workday returns the field empty rather than erroring. Completeness must be verified during onboarding (domain checklist + data review); the connector cannot distinguish "empty" from "forbidden". + +--- + +## 3. Technical Architecture + +### 3.1 Domain Model + +**Core Entities**: + +| Entity | API Source | Bronze Stream | Description | +|--------|-----------|--------------|-------------| +| Worker | workers RaaS report | `workers` | Current-state worker record with contract-defined HR attributes | +| Time-Off Request | leave RaaS report | `leave_requests` | Time-off request with dates, type, status, and quantity | + +**Relationships**: +- Worker `1:N` Time-Off Request (via `Employee_ID`) +- Worker `N:1` Supervisory Organization (inline name; hierarchy via manager chain) + +### 3.2 Component Model + +#### Workday Connector Manifest + +- [ ] `p1` - **ID**: `cpt-insightspec-component-wd-manifest` + +##### Why this component exists + +Defines the complete Workday connector as a YAML declarative manifest — the single artifact required to extract HR data from any contract-conformant Workday tenant into the Bronze layer. + +##### Responsibility scope + +Defines 2 streams with: ISU Basic auth, GET requests to config-driven report paths, `format=json` and date-prompt query parameters, `NoPagination`, full refresh sync, `AddFields` for `raw_data` (workers only), `tenant_id`, `source_id`, and `unique_key`, and inline JSON schemas matching the report contract. + +##### Responsibility boundaries + +Does not handle orchestration, scheduling, or state storage (managed by Airbyte/Orchestrator). Does not create or validate Workday report definitions beyond reachability (`check`). Does not perform Silver/Gold transformations (the dbt models shipped alongside do). Does not implement identity resolution. + +##### Related components (by ID) + +`cpt-insightspec-component-wd-dbt-chain` (downstream Bronze consumer). + +#### Workday dbt Chain + +- [ ] `p1` - **ID**: `cpt-insightspec-component-wd-dbt-chain` + +##### Why this component exists + +Turns current-state Bronze pulls into SCD2 history and identity observations — the part of the HR pipeline Workday's RaaS extraction (current-state only) cannot provide by itself. + +##### Responsibility scope + +`workday__bronze_promoted` (RMT promotion), `workday__workers_snapshot` (SCD2 via `snapshot` macro, tracked columns + `workday_custom_fields` from `raw_data`), `workday__workers_fields_history` (per-field change log via `fields_history` macro), `workday__identity_inputs` (UPSERT/DELETE observations via `identity_inputs_from_history` macro), plus staging views `workday__to_class_people`, `workday__hr_events`, `workday__working_hours`. + +##### Responsibility boundaries + +Does not deduplicate beyond RMT semantics; does not resolve `person_id` (Identity Manager's job); does not normalise customer-specific time-off types. + +##### Related components (by ID) + +`cpt-insightspec-component-wd-manifest` (upstream Bronze producer). + +### 3.3 API Contracts + +#### Workday RaaS (customreport2) + +- [ ] `p1` - **ID**: `cpt-insightspec-interface-wd-raas` + +**Base URL**: `{workday_base_url}` = `https://{host}/ccx/service/customreport2/{workday_tenant}` (config-provided, no trailing slash) + +**Authentication**: ISU credentials via `BasicHttpAuthenticator` (`workday_isu_username` / `workday_isu_password`) + +**Response wrapper**: every RaaS JSON response is `{"Report_Entry": [ {…}, … ]}` — the record selector extracts `Report_Entry`. + +**Field names**: the XML aliases set on report columns. The contract requires explicit aliases (auto-generated aliases differ per tenant). + +**Rate limits**: undocumented; report execution shares tenant compute. Transient 5XX retried with backoff. + +--- + +##### Endpoint: GET {workers_report_path} + +| Aspect | Detail | +|--------|--------| +| Method | `GET` | +| Path | `{workday_workers_report_path}` = `/` | +| Query params | `format=json` | +| Response | `{"Report_Entry": [{Employee_ID, Display_Name, ...}, ...]}` | +| Record path | `Report_Entry` | +| Auth | HTTP Basic (ISU) | + +**Contract columns** (exact aliases — see connector README for the Workday field mapping): `Employee_ID`, `Display_Name`, `First_Name`, `Last_Name`, `Work_Email`, `Business_Title`, `Job_Profile`, `Worker_Type`, `Worker_Status`, `Supervisory_Organization`, `Manager_Employee_ID`, `Manager_Work_Email`, `Location`, `Country`, `City`, `Hire_Date`, `Original_Hire_Date`, `Termination_Date`, `Last_Functionally_Updated`, `Scheduled_Weekly_Hours`. Extra columns allowed (land in `raw_data`). + +--- + +##### Endpoint: GET {leave_report_path} + +| Aspect | Detail | +|--------|--------| +| Method | `GET` | +| Path | `{workday_leave_report_path}` = `/` | +| Query params | `format=json`, `From_Date={workday_start_date}`, `To_Date={today UTC}` | +| Response | `{"Report_Entry": [{Request_ID, Employee_ID, ...}, ...]}` | +| Record path | `Report_Entry` | +| Auth | HTTP Basic (ISU) | + +**Contract columns**: `Request_ID`, `Employee_ID`, `Time_Off_Type`, `Start_Date`, `End_Date`, `Quantity`, `Unit`, `Status`, `Submitted_Moment`. + +**Prompts**: the report MUST define `From_Date` and `To_Date` prompts enabled as web service parameters; a report without them rejects the query parameters and the sync fails loudly. + +--- + +#### Source Config Schema + +```json +{ + "type": "object", + "required": [ + "insight_tenant_id", "insight_source_id", + "workday_base_url", "workday_isu_username", "workday_isu_password", + "workday_workers_report_path", "workday_leave_report_path" + ], + "properties": { + "insight_tenant_id": {"type": "string", "title": "Insight Tenant ID"}, + "insight_source_id": {"type": "string", "title": "Insight Source ID"}, + "workday_base_url": {"type": "string", "title": "Workday RaaS Base URL"}, + "workday_isu_username": {"type": "string", "title": "ISU Username"}, + "workday_isu_password": {"type": "string", "title": "ISU Password", "airbyte_secret": true}, + "workday_workers_report_path": {"type": "string", "title": "Workers Report Path"}, + "workday_leave_report_path": {"type": "string", "title": "Leave Report Path"}, + "workday_start_date": {"type": "string", "title": "Start Date", "default": "2020-01-01"} + } +} +``` + +### 3.4 Internal Dependencies + +| Dependency Module | Interface Used | Purpose | +|-------------------|---------------|---------| +| dbt macros (`snapshot`, `fields_history`, `identity_inputs_from_history`, `promote_bronze_to_rmt`) | dbt compilation | Silver SCD2 chain — reused unchanged from the shared macro library | +| Identity Manager | Downstream consumer | Reads `workday__identity_inputs` observations for person resolution | +| HR Silver layer | Downstream consumer | Reads `silver:class_*`-tagged staging views | + +### 3.5 External Dependencies + +| Dependency | Interface Used | Purpose | +|-----------|---------------|---------| +| Workday RaaS | HTTPS / JSON | Source system for worker and time-off extraction | +| Customer-built custom reports | Report contract | Extraction definition — owned by the customer's Workday administrator | +| Airbyte Declarative Connector framework (CDK v7.0+) | Container runtime | Executes the YAML manifest | +| ClickHouse destination connector | Airbyte protocol | Writes extracted records to Bronze tables | + +### 3.6 Interactions & Sequences + +#### Worker Collection — Full Refresh + +**ID**: `cpt-insightspec-seq-wd-worker-sync` + +```mermaid +sequenceDiagram + participant Orch as Orchestrator + participant Src as Workday Connector + participant WD as Workday RaaS + participant Dest as ClickHouse Destination + + Orch ->> Src: Trigger sync + Src ->> WD: GET {base}/{workers_report}?format=json + WD -->> Src: {"Report_Entry": [{Employee_ID, Work_Email, ...}, ...]} + loop For each worker + Src ->> Dest: Emit record (raw_data, tenant_id, source_id, unique_key) + end +``` + +#### Leave Collection — Full Refresh with Date Prompts + +**ID**: `cpt-insightspec-seq-wd-leave-sync` + +```mermaid +sequenceDiagram + participant Orch as Orchestrator + participant Src as Workday Connector + participant WD as Workday RaaS + participant Dest as ClickHouse Destination + + Orch ->> Src: Trigger sync + Src ->> WD: GET {base}/{leave_report}?format=json&From_Date=2020-01-01&To_Date={today} + WD -->> Src: {"Report_Entry": [{Request_ID, Employee_ID, ...}, ...]} + loop For each request + Src ->> Dest: Emit record (tenant_id, source_id, unique_key) + end +``` + +#### Silver Chain — Change Detection + +**ID**: `cpt-insightspec-seq-wd-silver-chain` + +```mermaid +sequenceDiagram + participant Bronze as bronze_workday.workers (RMT) + participant Snap as workday__workers_snapshot + participant Hist as workday__workers_fields_history + participant IdIn as workday__identity_inputs + participant IM as Identity Manager + + Bronze ->> Snap: FINAL read; hash tracked columns + Snap ->> Snap: append row when hash != latest version + Snap ->> Hist: version pairs → per-field old/new rows + Hist ->> IdIn: identity-relevant changes → UPSERT/DELETE observations + IdIn ->> IM: email / employee_id / names / department / manager / status +``` + +### 3.7 Database schemas & tables + +#### Table: `workers` + +- [ ] `p1` - **ID**: `cpt-insightspec-dbtable-wd-workers` + +| Column | Type | Description | +|--------|------|-------------| +| `tenant_id` | String | Tenant identifier — injected by connector | +| `source_id` | String | Source instance identifier (e.g. `workday-acme-prod`) — injected by connector | +| `unique_key` | String | PK: `{tenant}-{source}-{Employee_ID}` — injected by connector | +| `Employee_ID` | String | Workday Employee ID | +| `Display_Name` | String | Preferred name | +| `First_Name` | String | Legal first name | +| `Last_Name` | String | Legal last name | +| `Work_Email` | String | Primary work email — identity key; may be empty for contingent workers | +| `Business_Title` | String | Business title — role analytics | +| `Job_Profile` | String | Standardized job profile name | +| `Worker_Type` | String | `Employee` / `Contingent Worker` — workforce composition | +| `Worker_Status` | String | `Active` / `On Leave` / `Terminated` | +| `Supervisory_Organization` | String | Workday's standard org unit — org hierarchy dimension | +| `Manager_Employee_ID` | String | Manager's Employee ID (management chain) — org hierarchy | +| `Manager_Work_Email` | String | Manager's work email — org hierarchy + identity resolution | +| `Location` | String | Office location or remote designation | +| `Country` | String | Location address country | +| `City` | String | Location address city | +| `Hire_Date` | String | Employment start date — tenure analytics | +| `Original_Hire_Date` | String | Original hire date for rehires — tenure analytics | +| `Termination_Date` | String | Employment end date; empty if active — attrition analytics | +| `Last_Functionally_Updated` | String | Last modification entry moment — future incremental cursor | +| `Scheduled_Weekly_Hours` | String/Number | Scheduled weekly hours — FTE / working-hours calculation | +| `raw_data` | String (JSON) | Full report row incl. extra customer columns — custom-field passthrough | +| `_airbyte_extracted_at` | DateTime | Extraction timestamp (UTC) — auto-generated | + +--- + +#### Table: `leave_requests` + +- [ ] `p1` - **ID**: `cpt-insightspec-dbtable-wd-leave-requests` + +| Column | Type | Description | +|--------|------|-------------| +| `tenant_id` | String | Tenant identifier — injected by connector | +| `source_id` | String | Source instance identifier — injected by connector | +| `unique_key` | String | PK: `{tenant}-{source}-{Request_ID}` — injected by connector | +| `Request_ID` | String | Workday time-off request ID | +| `Employee_ID` | String | Workday Employee ID — joins to `workers.Employee_ID` | +| `Time_Off_Type` | String | Time-off type name (policy-defined, client-specific) | +| `Start_Date` | String | Leave start date | +| `End_Date` | String | Leave end date | +| `Quantity` | String/Number | Units requested | +| `Unit` | String | `hours` / `days` | +| `Status` | String | Request status (`Approved`, `Submitted`, `Canceled`, …) | +| `Submitted_Moment` | String | When the request was submitted | +| `_airbyte_extracted_at` | DateTime | Extraction timestamp (UTC) — auto-generated | + +--- + +#### Staging tables (dbt) + +`workday__workers_snapshot` (SCD2 append-only: bronze columns + `_row_hash`, `_tracked_at`), `workday__workers_fields_history` (`entity_id`, `tenant_id`, `source_id`, `field_name`, `old_value`, `new_value`, `updated_at`), `workday__identity_inputs` (shared `identity_inputs` schema). Column definitions follow the shared macros — see `src/ingestion/dbt/macros/`. + +### 3.8 Deployment Topology + +- [ ] `p1` - **ID**: `cpt-insightspec-topology-wd-deployment` + +``` +Connection: workday-{source-id} +├── Source image: airbyte/source-declarative-manifest +├── Manifest: src/ingestion/connectors/hr-directory/workday/connector.yaml +├── Descriptor: src/ingestion/connectors/hr-directory/workday/descriptor.yaml +├── Source config (from K8s Secret insight-workday-{suffix}): +│ workday_base_url, workday_isu_username, workday_isu_password, +│ workday_workers_report_path, workday_leave_report_path, workday_start_date +├── Configured catalog: 2 streams (all full refresh) +│ ├── workers +│ └── leave_requests +├── Destination image: airbyte/destination-clickhouse (shared) +├── Bronze namespace: bronze_workday +└── dbt: tag:workday+ (promotion → snapshot → history → identity_inputs → class staging) +``` + +--- + +## 4. Additional context + +### Identity Resolution Strategy + +`Work_Email` from `workers` is the primary identity signal, flowing to the Identity Manager through the `fields_history` → `identity_inputs` chain (UPSERT on change, DELETE on `Worker_Status` → `Terminated`). `Manager_Work_Email` enables org hierarchy construction without requiring `person_id` resolution of managers first. + +`Employee_ID` is the source-account binding (`value_type='id'` observation per ADR-0002) and the `employee_id` observation; it is Workday-internal and not used as a cross-system join key. + +### Silver / Gold Mappings + +| Bronze table | Silver target | Status | +|-------------|--------------|--------| +| `workers` | `identity_inputs` → Identity Manager | Via SCD2 chain (shipped) | +| `workers` | `class_people` | `workday__to_class_people` staging view (shipped) | +| `workers` | `class_hr_working_hours` | `workday__working_hours` staging view (shipped) | +| `leave_requests` | `class_hr_events` | `workday__hr_events` staging view (shipped) | + +**Gold**: leave analytics (burnout risk, availability), headcount metrics, org composition over time (from SCD2 snapshot history; point-in-time accuracy bounded by sync cadence until PRD OQ-WD-3 lands). + +### Source-Specific Considerations + +1. **Report contract is the schema**: there is no API-side guarantee of any field — only the contract. Onboarding must verify column aliases and the ISU domain checklist; dbt `not_null` tests are the second line of defence. + +2. **`Last_Functionally_Updated` excluded from snapshot tracking**: it changes on any worker update, including untracked fields, and would create spurious SCD2 versions. It is collected for future incremental extraction only. + +3. **No department/division standard fields**: `Supervisory_Organization` maps to `department` uniformly in v1. Customer-specific org modelling is PRD OQ-WD-4. + +4. **Flat report columns**: unlike BambooHR's nested JSON objects, RaaS report columns are flat strings — `workday__hr_events` needs no `JSONExtractString`. + +5. **Status vocabularies are contract-normative**: Silver mappings switch on `Worker_Status` ∈ {`Active`, `On Leave`, `Terminated`} and `Worker_Type` ∈ {`Employee`, `Contingent Worker`}. If a tenant's report emits different vocabulary (e.g. localized values), the report must normalise it via a calculated field — part of the contract. + +--- + +## 5. Traceability + +- **PRD**: [PRD.md](./PRD.md) +- **Domain README**: [../../README.md](../../README.md) — HR Silver Layer design +- **Connector Framework DESIGN**: [../../../../domain/connector/specs/DESIGN.md](../../../../domain/connector/specs/DESIGN.md) +- **BambooHR counterpart**: [../../bamboohr/specs/DESIGN.md](../../bamboohr/specs/DESIGN.md) — the pattern this connector mirrors +- **ADR directory**: [./ADR/](./ADR/) + +--- + +## 6. Non-Applicability Statements + +- **Custom Python components**: Not required. All RaaS extraction patterns are handled by declarative manifest components. +- **OAuth 2.0**: Not implemented. ISU Basic auth is sufficient for RaaS read-only extraction. +- **Pagination**: Not applicable. RaaS returns complete report results. +- **Workday SOAP (WWS) / WQL**: Not used in v1 — RaaS only. SOAP transaction-log extraction is a candidate for native history backfill (PRD OQ-WD-3). +- **Webhook / real-time streaming**: Not applicable. Workday change notifications are not used. +- **Write operations**: Not applicable. The connector is read-only. diff --git a/docs/components/connectors/hr-directory/workday/specs/PRD.md b/docs/components/connectors/hr-directory/workday/specs/PRD.md index e69de29bb..fb2822610 100644 --- a/docs/components/connectors/hr-directory/workday/specs/PRD.md +++ b/docs/components/connectors/hr-directory/workday/specs/PRD.md @@ -0,0 +1,414 @@ +# PRD — Workday Connector + +> Version 1.0 — June 2026 +> Based on: HR Directory domain (`docs/components/connectors/hr-directory/README.md`), BambooHR connector PRD (`../../bamboohr/specs/PRD.md`) + + + +- [1. Overview](#1-overview) + - [1.1 Purpose](#11-purpose) + - [1.2 Background / Problem Statement](#12-background--problem-statement) + - [1.3 Goals (Business Outcomes)](#13-goals-business-outcomes) + - [1.4 Glossary](#14-glossary) +- [2. Actors](#2-actors) + - [2.1 Human Actors](#21-human-actors) + - [2.2 System Actors](#22-system-actors) +- [3. Operational Concept & Environment](#3-operational-concept--environment) + - [3.1 Module-Specific Environment Constraints](#31-module-specific-environment-constraints) +- [4. Scope](#4-scope) + - [4.1 In Scope](#41-in-scope) + - [4.2 Out of Scope](#42-out-of-scope) +- [5. Functional Requirements](#5-functional-requirements) + - [5.1 Data Collection](#51-data-collection) + - [5.2 Data Integrity](#52-data-integrity) +- [6. Non-Functional Requirements](#6-non-functional-requirements) + - [6.1 NFR Inclusions](#61-nfr-inclusions) + - [6.2 NFR Exclusions](#62-nfr-exclusions) +- [7. Public Library Interfaces](#7-public-library-interfaces) + - [7.1 Public API Surface](#71-public-api-surface) + - [7.2 External Integration Contracts](#72-external-integration-contracts) +- [8. Use Cases](#8-use-cases) +- [9. Acceptance Criteria](#9-acceptance-criteria) +- [10. Dependencies](#10-dependencies) +- [11. Assumptions](#11-assumptions) +- [12. Risks](#12-risks) +- [13. Open Questions](#13-open-questions) + - [OQ-WD-1: Incremental extraction via Last Functionally Updated prompt](#oq-wd-1-incremental-extraction-via-last-functionally-updated-prompt) + - [OQ-WD-2: BambooHR + Workday coexistence](#oq-wd-2-bamboohr--workday-coexistence) + - [OQ-WD-3: Native effective-dated history (transaction log)](#oq-wd-3-native-effective-dated-history-transaction-log) + - [OQ-WD-4: Department / division mapping per instance](#oq-wd-4-department--division-mapping-per-instance) + + + +--- + +## 1. Overview + +### 1.1 Purpose + +The Workday connector extracts HR directory data — worker records and time-off requests — from Workday RaaS (Reports-as-a-Service) custom reports into the Insight platform's Bronze layer. This data feeds identity resolution (canonical `person_id` via work email), org hierarchy construction, and leave analytics. + +### 1.2 Background / Problem Statement + +Workday is the dominant enterprise HRIS (5000+ employees). The Insight platform requires HR directory data for: + +1. **Identity resolution** — mapping source-system user identifiers (GitHub login, Jira account ID, etc.) to real people via work email as the canonical identity anchor. +2. **Org hierarchy** — enabling team-level aggregation of engineering metrics by supervisory organization and manager chain. +3. **Leave analytics** — time-off patterns feed burnout risk signals and availability forecasting. + +Unlike BambooHR, Workday has **no fixed bulk-extraction endpoint** with a guaranteed field set. The extraction mechanism is RaaS: the customer builds custom reports in Workday Report Writer following an Insight-defined **report contract** (column set + XML aliases + prompts), enables them as web services, and shares them with an Integration System User (ISU). The connector fetches the reports as JSON. The field set is therefore controlled on the Workday side, and the connector must validate the configured reports against the contract. + +Although Workday records are natively effective-dated, the v1 connector extracts **current state only** (the report returns as-of-today values) — exactly like BambooHR. Historical change tracking is provided by the Silver-layer SCD2 snapshot chain (`snapshot` → `fields_history` → `identity_inputs`). Native history extraction is deferred (see OQ-WD-3). + +### 1.3 Goals (Business Outcomes) + +1. Enable identity resolution for all Insight workspaces using Workday as their HR system. +2. Provide org hierarchy data (supervisory organization, manager chain) for team-level metric scoping. +3. Collect time-off history for availability and burnout risk analytics. +4. Allow customer-specific report columns (custom/calculated fields) to flow into Bronze `raw_data` and be tracked by the Silver SCD2 chain without connector changes. + +### 1.4 Glossary + +| Term | Definition | +|------|-----------| +| **RaaS** | Reports-as-a-Service — Workday's mechanism for exposing a custom report as a web service endpoint (`/ccx/service/customreport2/{tenant}/{owner}/{report}`) | +| **ISU** | Integration System User — Workday service account used for API authentication; access is scoped via an Integration System Security Group and domain security policies | +| **Report contract** | The Insight-defined specification of report columns, XML aliases, and prompts that the customer's custom reports must follow | +| **XML alias** | The explicit column alias set in Report Writer; becomes the JSON field name in the RaaS response. Auto-generated aliases differ per tenant — the contract requires explicit aliases | +| **Workday-delivered field** | A field shipped as part of Workday's standard worker data model — present in every tenant (e.g., Employee ID, Hire Date, Supervisory Organization) | +| **Identity key** | The field used for cross-system person resolution — `Work_Email` for Workday | + +--- + +## 2. Actors + +### 2.1 Human Actors + +#### Platform Engineer + +**ID**: `cpt-insightspec-actor-wd-platform-engineer` + +Configures Workday connections (ISU credentials, base URL, report paths), monitors collection runs, and troubleshoots extraction failures. + +#### Customer Workday Administrator + +**ID**: `cpt-insightspec-actor-wd-customer-admin` + +Builds the two custom reports per the Insight report contract, creates the ISU and security group, grants domain access, and shares the reports with the ISU. Owns the Workday side of the integration. + +#### Data Analyst + +**ID**: `cpt-insightspec-actor-wd-data-analyst` + +Consumes Workday Bronze data through Silver/Gold layers for org hierarchy analysis, headcount reporting, and leave pattern analytics. + +### 2.2 System Actors + +#### Orchestrator + +**ID**: `cpt-insightspec-actor-wd-orchestrator` + +Triggers Workday connector runs on schedule and routes output to the destination. + +#### Identity Manager + +**ID**: `cpt-insightspec-actor-wd-identity-manager` + +Consumes identity observations derived from `workers` (via the `fields_history` → `identity_inputs` chain) to maintain the canonical `person_id` mapping used by all Silver streams. + +#### Destination (ClickHouse) + +**ID**: `cpt-insightspec-actor-wd-destination` + +Receives extracted records and writes them to Bronze tables. + +--- + +## 3. Operational Concept & Environment + +### 3.1 Module-Specific Environment Constraints + +- Workday API access requires an ISU whose security group has the domains needed by the report fields (at minimum: Worker Data: Public Worker Reports, Person Data: Work Email). **A missing domain does not fail the request — the field silently arrives empty.** +- RaaS runs the report on demand; response time and size scale with tenant headcount. Large tenants (50k+ workers) may approach RaaS execution timeouts on full pulls. +- There is **no public Workday developer tenant**. Development uses mock fixtures replaying the RaaS response shape; validation requires the customer's Sandbox tenant during onboarding. +- All API requests require HTTPS (mock endpoints for local development excepted). + +--- + +## 4. Scope + +### 4.1 In Scope + +- Extraction of worker directory data via a customer-built RaaS report following the Insight report contract (Workday-delivered fields only in the standard column set). +- Extraction of time-off requests via a second RaaS report with `From_Date`/`To_Date` prompts. +- Full refresh sync on both streams (current-state extraction, BambooHR-style). +- Customer-specific extra report columns flowing into Bronze `raw_data` (key-value tracking via dbt var `workday_custom_fields`). +- `tenant_id`, `source_id`, `unique_key` injection on all records (platform invariants). +- Silver chain: bronze→RMT promotion, SCD2 snapshot, fields_history, identity_inputs, class_people / class_hr_events / class_hr_working_hours staging. + +### 4.2 Out of Scope + +- Incremental extraction via `Last_Functionally_Updated` report prompt — deferred (OQ-WD-1). +- Native effective-dated history / transaction-log extraction — deferred (OQ-WD-3). +- Supervisory organization hierarchy as a separate stream (org names are inline in worker records; hierarchy construction from manager chain is a Silver concern). +- Workday SOAP (WWS) and WQL access paths — RaaS only in v1. +- OAuth 2.0 authentication — ISU Basic auth is sufficient for RaaS (OAuth may be revisited if a customer mandates it). +- Silver/Gold layer transformations beyond the staging models shipped with the connector. +- Write operations (worker updates, time-off approval). + +--- + +## 5. Functional Requirements + +### 5.1 Data Collection + +#### Worker Data Collection + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-wd-collect-workers` + +The system **MUST** extract worker records from the customer's workers RaaS report, collecting the contract's standard column set: worker identity (Employee ID, names, work email), org hierarchy (supervisory organization, manager ID/email), job (business title, job profile), classification (worker type, worker status), location (location, country, city), employment dates (hire, original hire, termination), last-modified timestamp (Last Functionally Updated), and scheduled weekly hours. + +**Rationale**: Worker data is the foundation for identity resolution, org hierarchy, and all person-level analytics in the Insight platform. + +**Actors**: `cpt-insightspec-actor-wd-orchestrator`, `cpt-insightspec-actor-wd-destination` + +#### Leave Request Collection + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-wd-collect-leave-requests` + +The system **MUST** extract time-off requests from the customer's leave RaaS report, collecting: request ID, employee ID, time-off type, start date, end date, quantity, unit, status, and submission moment. The report's date range **MUST** be controlled via `From_Date`/`To_Date` web service prompts. + +**Rationale**: Leave request data feeds burnout risk signals, availability forecasting, and team capacity analytics. + +**Actors**: `cpt-insightspec-actor-wd-orchestrator`, `cpt-insightspec-actor-wd-destination` + +#### Custom Column Passthrough + +- [ ] `p2` - **ID**: `cpt-insightspec-fr-wd-custom-columns` + +The system **MUST** preserve any extra columns present in the workers report (custom fields, calculated fields) in the Bronze `raw_data` column, without requiring connector changes. Tracking of specific custom columns at the Silver layer is configured via the dbt var `workday_custom_fields`. + +**Rationale**: Workday has no field-discovery endpoint analogous to BambooHR `meta/fields`; the report definition IS the field registry. Extra columns must flow through without schema changes. + +**Actors**: `cpt-insightspec-actor-wd-customer-admin`, `cpt-insightspec-actor-wd-destination` + +### 5.2 Data Integrity + +#### Report Contract Validation + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-wd-report-contract` + +The connector's `check` operation **MUST** fail when the configured workers report is unreachable or returns a payload without the `Report_Entry` wrapper. Documentation **MUST** specify the exact column aliases so that a mis-built report is detected at onboarding, not silently ingested with missing fields. + +**Rationale**: With RaaS, nothing guarantees the field set — the customer's report definition does. A report built off-contract must fail loudly (fail-fast platform principle), because missing identity columns silently break person resolution downstream. + +**Actors**: `cpt-insightspec-actor-wd-platform-engineer`, `cpt-insightspec-actor-wd-customer-admin` + +#### Deduplication + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-wd-deduplication` + +The system **MUST** define primary keys for each stream to enable deduplication at the destination: +- `workers`: `unique_key` = `{tenant_id}-{source_id}-{Employee_ID}` +- `leave_requests`: `unique_key` = `{tenant_id}-{source_id}-{Request_ID}` + +**Rationale**: Primary keys enable ReplacingMergeTree deduplication at the destination, preventing duplicate records across collection runs. + +**Actors**: `cpt-insightspec-actor-wd-destination` + +#### Identity Key + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-wd-identity-key` + +The system **MUST** collect `Work_Email` for every worker record. This field serves as the primary identity anchor for cross-system person resolution via the Identity Manager. The field may legitimately be empty for contingent workers; such records still carry `Employee_ID` as the source-account binding. + +**Rationale**: Work email is the most reliable cross-system identifier for HR-to-engineering-tool person matching. + +**Actors**: `cpt-insightspec-actor-wd-identity-manager` + +#### Sync Mode + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-wd-full-refresh-sync` + +Both streams use **full refresh** sync mode. The RaaS reports return current-state values on every call; there is no built-in delta mechanism. Leave requests use a date-range filter (`workday_start_date` to current date) via report prompts. The `Last_Functionally_Updated` column is collected to enable future incremental extraction (OQ-WD-1). + +**Rationale**: Full refresh is simple, reliable, and identical to the proven BambooHR pattern, including the downstream SCD2 snapshot chain. Incremental extraction requires additional customer-side report setup (a prompt-enabled filter) and is deferred until tenant scale demands it. + +**Actors**: `cpt-insightspec-actor-wd-orchestrator` + +#### Fault Tolerance + +- [ ] `p1` - **ID**: `cpt-insightspec-fr-wd-fault-tolerance` + +The system **MUST** handle transient API failures with retry and backoff, and fail clearly on authentication errors (401/403) without retry. + +**Rationale**: RaaS report execution can fail transiently under tenant load. Robust retry ensures collection completes; auth failures must surface immediately as configuration errors. + +**Actors**: `cpt-insightspec-actor-wd-orchestrator` + +--- + +## 6. Non-Functional Requirements + +### 6.1 NFR Inclusions + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-wd-auth-isu-basic` + +The connector **MUST** support ISU Basic authentication. ISU credentials, RaaS base URL, and both report paths **MUST** be configurable via the source connection specification (not hardcoded) and **MUST** be required fields with no defaults. + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-wd-schema-compliance` + +All Bronze records **MUST** use the report contract's column aliases as field names (Workday-style `Snake_Case`) with no renaming. Schema transformations occur at the Silver layer. + +- [ ] `p1` - **ID**: `cpt-insightspec-nfr-wd-idempotent-writes` + +Re-running the connector **MUST** produce records that deduplicate cleanly at the destination via `unique_key`. The connector performs no writes — idempotency is ensured by deterministic report output and primary-key deduplication. + +### 6.2 NFR Exclusions + +- **Performance SLAs**: Not applicable — RaaS report execution time depends on the customer's tenant size and Workday's infrastructure. +- **High availability**: The connector runs as a scheduled batch job; no real-time availability requirement. +- **Data encryption at rest**: Handled by the destination (ClickHouse) infrastructure, not the connector. + +--- + +## 7. Public Library Interfaces + +### 7.1 Public API Surface + +Not applicable. The Workday connector is a declarative manifest (YAML) executed by the Airbyte Declarative Connector framework. It does not expose a public API. + +### 7.2 External Integration Contracts + +- [ ] `p1` - **ID**: `cpt-insightspec-contract-wd-raas` + +**Workday RaaS** — the connector consumes two customer-built custom reports exposed as web services. The report contract (columns, aliases, prompts) and API details are specified in the [DESIGN](./DESIGN.md) §3.3 and the connector README. + +--- + +## 8. Use Cases + +- [ ] `p1` - **ID**: `cpt-insightspec-usecase-wd-onboarding` + +**UC-1: Customer Onboarding** + +**Trigger**: A customer wants Workday as their HR source. + +**Flow**: +1. Insight provides the report contract (column aliases, prompts, ISU domain checklist) to the customer's Workday administrator. +2. The administrator creates the ISU + security group, builds both reports, enables them as web services, and shares them with the ISU. +3. Platform engineer configures the K8s Secret (base URL, ISU credentials, report paths). +4. The connector `check` validates the workers report end-to-end against the customer's Sandbox, then Production tenant. + +**Postcondition**: Connection is validated; scheduled syncs can start. + +--- + +- [ ] `p1` - **ID**: `cpt-insightspec-usecase-wd-scheduled-sync` + +**UC-2: Scheduled Full Refresh** + +**Trigger**: Orchestrator triggers a scheduled run. + +**Flow**: +1. Connector fetches all workers via the workers report (`?format=json`). +2. Connector fetches time-off requests via the leave report (`From_Date`/`To_Date` prompts). +3. All records are written to Bronze tables. Destination deduplicates on `unique_key`. +4. dbt runs the Silver chain: snapshot detects changed workers, fields_history emits per-field transitions, identity_inputs feeds the Identity Manager. + +**Postcondition**: Bronze reflects the current state; SCD2 history is extended with detected changes. + +--- + +- [ ] `p2` - **ID**: `cpt-insightspec-usecase-wd-identity-feed` + +**UC-3: Identity Manager Feed** + +**Trigger**: Fresh worker records land in the `workers` Bronze table and dbt runs. + +**Flow**: +1. `workday__workers_snapshot` appends SCD2 versions for changed workers. +2. `workday__workers_fields_history` derives per-field change rows. +3. `workday__identity_inputs` emits UPSERT/DELETE identity observations (email, employee_id, names, department, manager, status). +4. Identity Manager resolves observations to canonical `person_id`. + +**Postcondition**: All Workday workers have a canonical `person_id` usable by all Silver streams. + +--- + +## 9. Acceptance Criteria + +1. The connector successfully extracts worker records from a contract-conformant RaaS report (mock fixture or customer Sandbox) and writes them to the destination. +2. The connector successfully extracts time-off requests within the configured date range via report prompts. +3. Full refresh sync correctly fetches all workers and all in-range leave requests on every run. +4. Extra report columns appear in Bronze `raw_data` without connector changes. +5. The connector fails gracefully on 401/403 with a clear error message; `check` fails on an unreachable or off-contract report. +6. All records include `tenant_id`, `source_id`, and `unique_key`. +7. The dbt chain (promotion → snapshot → fields_history → identity_inputs → class staging models) builds without errors and passes schema tests. + +--- + +## 10. Dependencies + +| Dependency | Type | Purpose | +|-----------|------|---------| +| Workday RaaS (customreport2) | External | Source system API | +| Customer-built custom reports | External | Extraction contract — owned by the customer's Workday administrator | +| Airbyte Declarative Connector framework (CDK v7.0+) | Runtime | Connector execution engine | +| ClickHouse destination connector | Runtime | Bronze table writes | +| dbt macros `snapshot`, `fields_history`, `identity_inputs_from_history`, `promote_bronze_to_rmt` | Internal | Silver SCD2 chain | +| Identity Manager | Downstream | Consumes identity observations for person resolution | + +--- + +## 11. Assumptions + +1. The customer can build and maintain the two custom reports per the Insight report contract (standard Workday Report Writer capability). +2. The ISU security group grants the domains required by the report fields; the onboarding checklist covers domain verification. +3. Workday-delivered fields in the contract (Employee ID, names, hire/termination dates, worker status/type, supervisory organization, manager chain, location, scheduled weekly hours, Last Functionally Updated) exist in every Workday tenant. +4. Tenant headcount is small enough for full-pull RaaS execution within timeouts; incremental extraction (OQ-WD-1) is the planned mitigation for larger tenants. +5. `Last Functionally Updated` is updated by Workday whenever any worker field changes. + +--- + +## 12. Risks + +| Risk | Impact | Mitigation | +|------|--------|-----------| +| Customer builds the report off-contract (wrong aliases, missing columns) | Missing/empty Bronze fields; broken identity resolution | Explicit alias requirements in the contract; `check` validation at onboarding; dbt `not_null` tests fail loudly | +| ISU lacks a security domain — fields silently arrive empty | Silent data gaps (worse than errors) | ISU domain checklist in onboarding docs; post-onboarding data completeness review | +| RaaS timeout on very large tenants | Failed or partial collection | Move to incremental extraction via `Last_Functionally_Updated` prompt (OQ-WD-1) | +| Customer edits the report (renames/removes columns) after onboarding | Schema drift, missing fields | Report named `Insight_*` and documented as Insight-owned; dbt tests detect missing data | +| No Workday tenant available during development | Cannot test against real API | Mock fixtures replaying `Report_Entry` shape; contract validated in customer Sandbox at onboarding | + +--- + +## 13. Open Questions + +### OQ-WD-1: Incremental extraction via Last Functionally Updated prompt + +RaaS has no built-in delta mechanism, but the workers report can define a prompt-enabled filter on `Last Functionally Updated`, turning the cursor into a URL parameter (`?Last_Functionally_Updated=...`). With incremental extraction, every fetched row is by definition a new version — the dbt `snapshot()` change-detection step becomes redundant and Bronze can store versions directly (cursor must be the entry moment, NOT effective date, to survive retroactive transactions). + +- At what tenant scale does full pull become impractical (RaaS timeout)? +- Should the prompt filter be part of the v1 report contract (unused until needed) to avoid a second customer-side change later? + +### OQ-WD-2: BambooHR + Workday coexistence + +Some clients may use both BambooHR and Workday (e.g. Workday for enterprise HR, BambooHR for a subsidiary). When both HR sources are active: + +- Does the Identity Manager merge them by email? +- Which source takes precedence for org hierarchy (manager chain)? +- Should `class_people` carry a `source` field indicating which HR system is authoritative per person? + +### OQ-WD-3: Native effective-dated history (transaction log) + +Workday records are natively effective-dated; full history — including changes made before the connector was installed — can be extracted via transaction-log data sources (SOAP `Get_Workers` with `Transaction_Log_Criteria` or a transaction-based report). This would backfill `fields_history` with true historical transitions instead of poll-time snapshots. + +- Is pre-onboarding org history valuable enough to justify the extra report/SOAP complexity? +- The current architecture supports it as a backfill of versioned rows — no redesign required. + +### OQ-WD-4: Department / division mapping per instance + +Workday has no standard `department`/`division` fields — supervisory organization is the only guaranteed org unit. Customers may model departments as custom organization types, cost centers, or levels in the supervisory hierarchy. + +- v1 maps `Supervisory_Organization` → `department` uniformly. Should the mapping be configurable per customer (e.g. extra contract columns `Department`/`Division` filled from customer-specific org types)? diff --git a/docs/components/connectors/hr-directory/workday/workday.md b/docs/components/connectors/hr-directory/workday/workday.md index 24b5ed8c1..8c7d28d0d 100644 --- a/docs/components/connectors/hr-directory/workday/workday.md +++ b/docs/components/connectors/hr-directory/workday/workday.md @@ -1,24 +1,19 @@ # Workday Connector Specification -> Version 1.0 — March 2026 -> Based on: `docs/CONNECTORS_REFERENCE.md` Source 11 (Workday) +> Version 2.0 — June 2026 +> Superseded in detail by [specs/PRD.md](./specs/PRD.md) and [specs/DESIGN.md](./specs/DESIGN.md); this page is the overview. -Standalone specification for the Workday (HR) connector. Expands Source 11 in the main Connector Reference with full table schemas, identity mapping, Silver/Gold pipeline notes, and open questions. +Standalone overview for the Workday (HR) connector. The v1 implementation follows the **BambooHR pattern**: current-state extraction via RaaS custom reports, with SCD2 history constructed at the Silver layer by the shared snapshot chain. - [Overview](#overview) +- [Extraction Model — Report Contract](#extraction-model--report-contract) - [Bronze Tables](#bronze-tables) - - [`workday_workers` — Worker records (point-in-time)](#workdayworkers-worker-records-point-in-time) - - [`workday_organizations` — Org units (departments, supervisory orgs, cost centers)](#workdayorganizations-org-units-departments-supervisory-orgs-cost-centers) - - [`workday_leave` — Leave of absence and time off](#workdayleave-leave-of-absence-and-time-off) - - [`workday_worker_ext` — Custom worker fields (key-value)](#workday_worker_ext--custom-worker-fields-key-value) - - [`workday_collection_runs` — Connector execution log](#workdaycollectionruns-connector-execution-log) +- [Silver Pipeline](#silver-pipeline) - [Identity Resolution](#identity-resolution) -- [Silver / Gold Mappings](#silver-gold-mappings) +- [Testing Without a Workday Tenant](#testing-without-a-workday-tenant) - [Open Questions](#open-questions) - - [OQ-WD-1: Effective dating strategy — full snapshot vs delta](#oq-wd-1-effective-dating-strategy-full-snapshot-vs-delta) - - [OQ-WD-2: BambooHR + Workday coexistence](#oq-wd-2-bamboohr-workday-coexistence) @@ -26,175 +21,94 @@ Standalone specification for the Workday (HR) connector. Expands Source 11 in th ## Overview -**API**: Workday REST API (Workday RaaS — Reports-as-a-Service) or Workday SOAP/WSDL API +**API**: Workday RaaS (Reports-as-a-Service) — two customer-built custom reports exposed as web services (`/ccx/service/customreport2/{tenant}/{owner}/{report}?format=json`) **Category**: HR / Directory -**Authentication**: ISU (Integration System User) credentials — Workday integration service account +**Authentication**: ISU (Integration System User) credentials via HTTP Basic — the ISU security group must hold the domains required by the report fields (missing domains yield silently empty fields, not errors) -**Identity**: `workday_workers.email` — resolved to canonical `person_id` via Identity Manager. HR connectors feed the Identity Manager directly alongside their Bronze tables. +**Identity**: `workers.Work_Email` — resolved to canonical `person_id` via Identity Manager through the `fields_history` → `identity_inputs` chain -**Field naming**: snake_case — Workday API uses mixed formats; normalised to snake_case at Bronze level. - -**Why multiple tables**: Workers, org units, and leave are distinct entities. The supervisory org structure is a separate entity tree (`workday_organizations`) that workers reference — storing org details inline in the worker record would denormalize all org changes. +**Field naming**: contract-defined `Snake_Case` XML aliases, preserved as-is at Bronze level **Key differences from BambooHR:** | Aspect | BambooHR | Workday | |--------|----------|---------| -| Record versioning | Current state only | Effective-dated point-in-time snapshots | -| Worker types | Employee + Contractor (freeform) | `Employee` / `Contingent_Worker` (explicit enum) | -| Org model | Department name (freeform string) | Supervisory Organization (separate entity with hierarchy) | -| Position model | Not tracked | `position_id` (job slot) is separate from the person | -| Scale | SMB (100–5000 employees) | Enterprise (5000+ employees) | +| Bulk endpoint | Fixed (`POST /reports/custom` with field list) | None — customer-built RaaS report per the Insight report contract | +| Field guarantees | API-defined field set | Only Workday-delivered fields exist everywhere; the report contract pins the actual set | +| Field discovery | `GET /meta/fields` stream | None — the report definition IS the field registry; extra columns land in `raw_data` | +| Record versioning | Current state only | Natively effective-dated, but v1 extracts current state only (parity with BambooHR) | +| Org model | Department name (freeform string) | Supervisory Organization (no standard department/division fields) | +| Worker types | Freeform | `Employee` / `Contingent Worker` (explicit enum) | +| Scale | SMB (100–5,000 employees) | Enterprise (5,000+ employees) | -**Primary use in Insight**: identity resolution, historical org structure for time-series team analytics, leave history with full categorisation. +**Primary use in Insight**: identity resolution, org structure for team analytics, leave history. --- -## Bronze Tables - -### `workday_workers` — Worker records (point-in-time) - -| Field | Type | Description | -|-------|------|-------------| -| `worker_id` | String | Workday internal worker ID | -| `email` | String | Work email — primary key for cross-system identity resolution | -| `full_name` | String | Display name | -| `first_name` | String | First name | -| `last_name` | String | Last name | -| `worker_type` | String | `Employee` / `Contingent_Worker` | -| `employment_status` | String | `Active` / `Terminated` / `Leave` | -| `job_title` | String | Business title | -| `job_profile` | String | Standardized job profile name | -| `position_id` | String | Position (job slot) identifier | -| `supervisory_org_id` | String | Supervisory organization ID — defines the reporting chain; joins to `workday_organizations.org_id` | -| `supervisory_org_name` | String | Supervisory organization name | -| `department` | String | Department name | -| `cost_center_id` | String | Cost center ID | -| `cost_center_name` | String | Cost center name | -| `manager_id` | String | Manager's Workday worker ID | -| `manager_email` | String | Manager's email | -| `location` | String | Office or `Remote` | -| `hire_date` | Date | Employment start date | -| `termination_date` | Date | Employment end date (NULL if active) | -| `effective_date` | Date | Date from which this record version is valid | - -`effective_date` makes this a point-in-time snapshot. Multiple rows per worker are expected — each org change produces a new versioned row. - ---- +## Extraction Model — Report Contract -### `workday_organizations` — Org units (departments, supervisory orgs, cost centers) - -| Field | Type | Description | -|-------|------|-------------| -| `org_id` | String | Workday org unit ID — primary key | -| `org_type` | String | `Supervisory` / `Department` / `CostCenter` / `Company` | -| `name` | String | Org unit name | -| `parent_org_id` | String | Parent org unit ID (NULL for root) | -| `head_worker_id` | String | Org head's Workday worker ID | -| `effective_date` | Date | Date from which this org version is valid | - -Org units are also effective-dated — a department rename or restructure produces a new versioned row. - ---- +Workday has no fixed bulk API, so the connector ships a **report contract** instead: the customer's Workday administrator builds two Advanced reports, sets the exact column XML aliases, enables them as web services, and shares them with the ISU. -### `workday_leave` — Leave of absence and time off +1. **Workers report** — data source All Workers; 20 contract columns (identity, org, job, classification, location, dates, `Last_Functionally_Updated`, `Scheduled_Weekly_Hours`). Extra customer columns are allowed and flow into Bronze `raw_data`. +2. **Leave report** — time-off requests; 9 contract columns plus mandatory `From_Date`/`To_Date` prompts enabled as web service parameters. -| Field | Type | Description | -|-------|------|-------------| -| `leave_id` | String | Workday leave request ID — primary key | -| `worker_id` | String | Worker's Workday ID — joins to `workday_workers.worker_id` | -| `worker_email` | String | Worker email | -| `leave_category` | String | `Leave_of_Absence` / `Time_Off` | -| `leave_type` | String | e.g. `Vacation`, `Sick`, `Parental`, `FMLA` (policy-defined) | -| `start_date` | Date | Leave start | -| `end_date` | Date | Leave end | -| `duration_days` | Float64 | Working days absent | -| `status` | String | `Approved` / `Pending` / `Cancelled` | -| `created_at` | DateTime64(3) | When the request was submitted | +The full alias tables live in the connector [README](../../../../src/ingestion/connectors/hr-directory/workday/README.md); the API contract is specified in [DESIGN §3.3](./specs/DESIGN.md). -`leave_category` is Workday-specific (high-level), while `leave_type` is policy-defined and client-specific. +Both streams are **full refresh** — RaaS returns the complete report result per call (no pagination, no delta). This is the same extraction model as BambooHR and reuses its entire downstream pipeline. --- -### `workday_worker_ext` — Custom worker fields (key-value) - -Workday workers support custom integration fields via custom reports (RaaS — Reports-as-a-Service) or integration system fields. Any field not in the core `workday_workers` schema — including custom integration attributes and calculated fields from custom reports — is written here. - -**Note**: Like BambooHR, Workday exposes custom fields inline in the worker response (or via a custom RaaS report that includes extra columns). This table captures those fields in the standard key-value pattern. `class_people.custom_str_attrs` and `class_people.custom_num_attrs` are populated from these values at Silver processing time. +## Bronze Tables -| Field | Type | Description | -|-------|------|-------------| -| `employee_id` | String | Parent worker ID — joins to `workday_workers.worker_id` | -| `field_id` | String | Workday field or report column ID, e.g. `Custom_Attribute_1` | -| `field_name` | String | Field display name (from RaaS report column header or field metadata) | -| `field_value` | String | Field value as string | -| `value_type` | String | Type hint: `string` / `number` / `date` / `json` | -| `collected_at` | DateTime64(3) | Collection timestamp | +| Table | Stream | Primary key | Contents | +|-------|--------|-------------|----------| +| `bronze_workday.workers` | `workers` | `unique_key` = `{tenant}-{source}-{Employee_ID}` | Current-state worker records, contract columns + `raw_data` | +| `bronze_workday.leave_requests` | `leave_requests` | `unique_key` = `{tenant}-{source}-{Request_ID}` | Time-off requests in the configured date range | -**Discovery**: custom fields are defined in the Workday RaaS report configuration or Integration System configuration. The connector administrator must declare which custom report columns to collect via the connector configuration. +Column-level definitions: [DESIGN §3.7](./specs/DESIGN.md). Both tables are promoted to ReplacingMergeTree by `workday__bronze_promoted` (dedup on `unique_key`). --- -### `workday_collection_runs` — Connector execution log +## Silver Pipeline -| Field | Type | Description | -|-------|------|-------------| -| `run_id` | String | Unique run identifier | -| `started_at` | DateTime64(3) | Run start time | -| `completed_at` | DateTime64(3) | Run end time | -| `status` | String | `running` / `completed` / `failed` | -| `workers_collected` | Float64 | Rows collected for `workday_workers` | -| `organizations_collected` | Float64 | Rows collected for `workday_organizations` | -| `leave_records_collected` | Float64 | Rows collected for `workday_leave` | -| `api_calls` | Float64 | API calls made | -| `errors` | Float64 | Errors encountered | -| `settings` | String | Collection configuration (tenant, RaaS report URLs, lookback) | +Model-for-model mirror of BambooHR: -Monitoring table — not an analytics source. +| Model | Role | +|-------|------| +| `workday__bronze_promoted` | Bronze → RMT promotion | +| `workday__workers_snapshot` | SCD2 append-only snapshot (`snapshot` macro); tracks contract columns + `workday_custom_fields` var from `raw_data`; `Last_Functionally_Updated` deliberately untracked | +| `workday__workers_fields_history` | Per-field change log (`fields_history` macro) | +| `workday__identity_inputs` | Identity observations (`identity_inputs_from_history` macro); DELETE on `Worker_Status` → `Terminated` | +| `workday__to_class_people` | `class_people` staging; `Supervisory_Organization` → `department_name`; `Worker_Type` → `employment_type` | +| `workday__hr_events` | `class_hr_events` staging (leave); flat columns, no JSON extraction | +| `workday__working_hours` | `class_hr_working_hours` staging; `Scheduled_Weekly_Hours` with 40h fallback | --- ## Identity Resolution -`workday_workers.email` is the primary identity key — mapped to canonical `person_id` via Identity Manager. HR connectors feed the Identity Manager directly as part of Bronze ingestion. +`workers.Work_Email` is the primary identity key — mapped to canonical `person_id` via Identity Manager. The field may legitimately be empty for contingent workers; those records still bind via the `value_type='id'` observation on `Employee_ID`. -`worker_id` (Workday internal worker ID) is Workday-internal — not used for cross-system resolution. - -`manager_email` enables org hierarchy construction from email addresses without requiring `worker_id` resolution. The `supervisory_org_id` provides an additional hierarchy path via `workday_organizations`. +`Manager_Work_Email` enables org hierarchy construction from email addresses without requiring `Employee_ID` resolution first. --- -## Silver / Gold Mappings +## Testing Without a Workday Tenant -| Bronze table | Silver target | Status | -|-------------|--------------|--------| -| `workday_workers` | Identity Manager (email → `person_id`) | ✓ Feeds identity resolution directly | -| `workday_workers` | `class_people` | Planned — HR unified stream not yet defined | -| `workday_organizations` | *(reference table)* | Available — no unified stream defined yet | -| `workday_leave` | `class_people` | Leave periods captured as `status = 'on_leave'` transitions in SCD2 | +There is no public Workday developer tenant; sandboxes exist only for customers and certified partners. Development therefore uses: -**Gold**: Historical org structure (team composition over time), leave analytics (burnout risk, availability), and headcount metrics. Effective dating in `workday_workers` enables point-in-time queries — "what was the team composition in Q3 2025?" — which is not possible with BambooHR. +1. **Mock fixtures** — a local HTTP server replaying the RaaS response shape (`{"Report_Entry": [...]}` with contract aliases). Reliable because the response contract is Insight-defined. +2. **Customer Sandbox** — every Workday customer has one; the connector `check` and a data completeness review run there during onboarding before Production. --- ## Open Questions -### OQ-WD-1: Effective dating strategy — full snapshot vs delta - -Workday records are effective-dated (multiple versions per worker). The connector can collect: - -- **Full snapshot**: all effective-dated records on every run (expensive, complete) -- **Delta**: only records with `effective_date >= last_run_date` (efficient, may miss retroactive changes) - -- Which collection strategy is used? -- Are retroactive effective date changes (back-dated org moves) expected in the client data? - -### OQ-WD-2: BambooHR + Workday coexistence - -Some clients may use both BambooHR and Workday (e.g. Workday for enterprise HR, BambooHR for a subsidiary). When both HR sources are active: +Tracked in [PRD §13](./specs/PRD.md): -- Does the Identity Manager merge them by email? -- Which source takes precedence for org hierarchy (manager chain)? -- Should `class_people` carry a `source` field indicating which HR system is authoritative per person? +- **OQ-WD-1** — incremental extraction via a prompt-enabled `Last Functionally Updated` filter (also removes the need for the `snapshot()` step; cursor must be entry moment, not effective date, to survive retroactive transactions) +- **OQ-WD-2** — BambooHR + Workday coexistence (identity merge, org precedence) +- **OQ-WD-3** — native effective-dated history backfill via transaction log (pre-onboarding history) +- **OQ-WD-4** — per-customer department/division mapping beyond `Supervisory_Organization` diff --git a/src/ingestion/connectors/hr-directory/workday/README.md b/src/ingestion/connectors/hr-directory/workday/README.md new file mode 100644 index 000000000..46acabe1d --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/README.md @@ -0,0 +1,147 @@ +# Workday Connector + +Worker directory and time-off requests from Workday via RaaS (Reports-as-a-Service) custom reports with ISU Basic authentication. + +Unlike BambooHR, Workday has no fixed bulk-extraction endpoint: the customer builds two custom reports in Workday Report Writer following the **report contract** below, and the connector fetches them as JSON. The field set is therefore controlled on the Workday side; the connector validates the configured reports at `check` time. + +## Prerequisites + +1. Create an **Integration System User (ISU)** in Workday and an Integration System Security Group containing it. +2. Grant the security group the domains needed by the report fields (at minimum: Worker Data: Public Worker Reports, Person Data: Work Email). +3. Build the two custom reports per the contract below (type **Advanced**, data source **All Workers** / time-off requests), enable them as a web service, and share them with the ISU. +4. Note the RaaS base URL from the report's web-service URL: `https:///ccx/service/customreport2/`. + +## Report Contract + +### Workers report (`workday_workers_report_path`) + +The report MUST expose the following columns with these exact XML aliases (set the alias explicitly on each column — auto-generated aliases differ per tenant): + +| Column alias | Workday field | Notes | +|--------------|---------------|-------| +| `Employee_ID` | Employee ID | Primary key | +| `Display_Name` | Preferred Name | | +| `First_Name` | Legal First Name | | +| `Last_Name` | Legal Last Name | | +| `Work_Email` | Primary Work Email | Identity key — may be empty for contingent workers | +| `Business_Title` | Business Title | | +| `Job_Profile` | Job Profile | | +| `Worker_Type` | Worker Type | `Employee` / `Contingent Worker` | +| `Worker_Status` | Active Status | `Active` / `On Leave` / `Terminated` | +| `Supervisory_Organization` | Supervisory Organization | Workday's standard org unit | +| `Manager_Employee_ID` | Manager Employee ID (management chain) | | +| `Manager_Work_Email` | Manager Work Email (management chain) | | +| `Location` | Location | | +| `Country` | Location Address — Country | | +| `City` | Location Address — City | | +| `Hire_Date` | Hire Date | | +| `Original_Hire_Date` | Original Hire Date | | +| `Termination_Date` | Termination Date | Empty if active | +| `Last_Functionally_Updated` | Last Functionally Updated | | +| `Scheduled_Weekly_Hours` | Scheduled Weekly Hours | | + +Extra columns (custom fields, calculated fields) are allowed: they land in the Bronze `raw_data` column and can be tracked by dbt via the `workday_custom_fields` var. + +### Leave report (`workday_leave_report_path`) + +| Column alias | Workday field | Notes | +|--------------|---------------|-------| +| `Request_ID` | Time Off Request ID | Primary key | +| `Employee_ID` | Employee ID | Joins to workers | +| `Time_Off_Type` | Time Off Type | Policy-defined, client-specific | +| `Start_Date` | First Day of Time Off | | +| `End_Date` | Last Day of Time Off | | +| `Quantity` | Units Requested | | +| `Unit` | Unit of Time | hours / days | +| `Status` | Request Status | | +| `Submitted_Moment` | Request Initiated (moment) | | + +The report MUST define two prompts enabled as web service parameters: `From_Date` and `To_Date` (date range filter on time-off dates). The connector passes `From_Date` from `workday_start_date` and `To_Date` as the current UTC date. + +## K8s Secret + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: insight-workday-main + labels: + app.kubernetes.io/part-of: insight + annotations: + insight.cyberfabric.com/connector: workday + insight.cyberfabric.com/source-id: workday-main +type: Opaque +stringData: + workday_base_url: "" # https:///ccx/service/customreport2/ + workday_isu_username: "" # Integration System User + workday_isu_password: "" # ISU password + workday_workers_report_path: "" # / + workday_leave_report_path: "" # / + workday_start_date: "2020-01-01" # Optional: time-off history start date +``` + +### Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `workday_base_url` | Yes | RaaS base URL incl. Workday tenant, no trailing slash: `https:///ccx/service/customreport2/` | +| `workday_isu_username` | Yes | Integration System User username | +| `workday_isu_password` | Yes | Integration System User password | +| `workday_workers_report_path` | Yes | Workers report path: `/` | +| `workday_leave_report_path` | Yes | Leave report path: `/` | +| `workday_start_date` | No | Time-off history start date, ISO format (default: `2020-01-01`) | + +> **Note on `username` / `password` spec fields.** +> The Airbyte Builder auto-generates `username` and `password` properties in +> `connection_specification` because the connector uses `BasicHttpAuthenticator`. +> These are managed automatically by the authenticator config: +> `username` = `workday_isu_username`, `password` = `workday_isu_password`. +> Do **not** set them in the K8s Secret or credentials file — they are not +> user-provided values. + +### Automatically injected + +| Field | Source | +|-------|--------| +| `insight_tenant_id` | `tenant_id` from tenant YAML | +| `insight_source_id` | `insight.cyberfabric.com/source-id` annotation | + +### Local development + +Create `src/ingestion/secrets/connectors/workday.yaml` (gitignored) from the example: + +```bash +cp src/ingestion/secrets/connectors/workday.yaml.example src/ingestion/secrets/connectors/workday.yaml +# Fill in real values, then apply: +kubectl apply -f src/ingestion/secrets/connectors/workday.yaml +``` + +There is no public Workday developer tenant. For local development without tenant access, use the bundled mock RaaS server ([fixtures/mock_raas.py](./fixtures/mock_raas.py)) — it replays the fixture responses (`{"Report_Entry": [...]}` wrapper) and enforces Basic auth, `format=json`, and the leave report's `From_Date`/`To_Date` prompts: + +```bash +# Terminal 1: start the mock +python3 src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py --port 8765 + +# Terminal 2: point the local secret at the mock and run the connector +# (workday_base_url: "http://host.docker.internal:8765/ccx/service/customreport2/acme", +# report paths: ISU_Insight/Insight_Employee_Sync, ISU_Insight/Insight_Leave_Sync) +cd src/ingestion +./tools/declarative-connector/source.sh check hr-directory/workday example-tenant +./tools/declarative-connector/source.sh read hr-directory/workday example-tenant +``` + +Validate against the customer's Sandbox tenant during onboarding. + +## Streams + +| Stream | Description | Sync Mode | +|--------|-------------|-----------| +| `workers` | Worker directory via RaaS custom report | Full refresh | +| `leave_requests` | Time-off requests via RaaS custom report (From_Date/To_Date prompts) | Full refresh | + +## Silver Targets + +- `class_people` — unified person registry +- `class_hr_events` — leave events +- `class_hr_working_hours` — scheduled working hours +- `identity_inputs` — identity observations via SCD2 snapshot → fields_history chain diff --git a/src/ingestion/connectors/hr-directory/workday/connector.yaml b/src/ingestion/connectors/hr-directory/workday/connector.yaml new file mode 100644 index 000000000..954d4bf06 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/connector.yaml @@ -0,0 +1,367 @@ +version: 7.0.4 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - workers + +definitions: + linked: + HttpRequester: + url_base: "{{ config['workday_base_url'] }}/" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['workday_isu_username'] }}" + password: "{{ config['workday_isu_password'] }}" + request_headers: + Accept: application/json + SimpleRetriever: + paginator: + type: NoPagination + +streams: + - type: DeclarativeStream + name: workers + retriever: + type: SimpleRetriever + paginator: + $ref: "#/definitions/linked/SimpleRetriever/paginator" + requester: + type: HttpRequester + path: "{{ config['workday_workers_report_path'] }}" + url_base: "{{ config['workday_base_url'] }}/" + http_method: GET + authenticator: + $ref: "#/definitions/linked/HttpRequester/authenticator" + request_headers: + $ref: "#/definitions/linked/HttpRequester/request_headers" + request_parameters: + format: json + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - Report_Entry + primary_key: unique_key + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + required: + - unique_key + properties: + Employee_ID: + type: + - string + - "null" + Display_Name: + type: + - string + - "null" + First_Name: + type: + - string + - "null" + Last_Name: + type: + - string + - "null" + Work_Email: + type: + - string + - "null" + Business_Title: + type: + - string + - "null" + Job_Profile: + type: + - string + - "null" + Worker_Type: + type: + - string + - "null" + Worker_Status: + type: + - string + - "null" + Supervisory_Organization: + type: + - string + - "null" + Manager_Employee_ID: + type: + - string + - "null" + Manager_Work_Email: + type: + - string + - "null" + Location: + type: + - string + - "null" + Country: + type: + - string + - "null" + City: + type: + - string + - "null" + Hire_Date: + type: + - string + - "null" + Original_Hire_Date: + type: + - string + - "null" + Termination_Date: + type: + - string + - "null" + Last_Functionally_Updated: + type: + - string + - "null" + Scheduled_Weekly_Hours: + type: + - number + - string + - "null" + raw_data: + type: + - object + - "null" + additionalProperties: true + source_id: + type: + - string + - "null" + tenant_id: + type: + - string + - "null" + unique_key: + type: string + additionalProperties: true + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - raw_data + value: "{{ record }}" + - type: AddedFieldDefinition + path: + - tenant_id + value: "{{ config['insight_tenant_id'] }}" + - type: AddedFieldDefinition + path: + - source_id + value: "{{ config['insight_source_id'] }}" + - type: AddedFieldDefinition + path: + - unique_key + value: >- + {{ config['insight_tenant_id'] }}-{{ config['insight_source_id'] + }}-{{ record['Employee_ID'] }} + - type: DeclarativeStream + name: leave_requests + retriever: + type: SimpleRetriever + paginator: + $ref: "#/definitions/linked/SimpleRetriever/paginator" + requester: + type: HttpRequester + path: "{{ config['workday_leave_report_path'] }}" + url_base: "{{ config['workday_base_url'] }}/" + http_method: GET + authenticator: + $ref: "#/definitions/linked/HttpRequester/authenticator" + request_headers: + $ref: "#/definitions/linked/HttpRequester/request_headers" + request_parameters: + format: json + From_Date: "{{ config.get('workday_start_date', '2020-01-01') }}" + To_Date: "{{ now_utc().strftime('%Y-%m-%d') }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - Report_Entry + primary_key: unique_key + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + required: + - unique_key + properties: + Request_ID: + type: + - string + - "null" + Employee_ID: + type: + - string + - "null" + Time_Off_Type: + type: + - string + - "null" + Start_Date: + type: + - string + - "null" + End_Date: + type: + - string + - "null" + Quantity: + type: + - number + - string + - "null" + Unit: + type: + - string + - "null" + Status: + type: + - string + - "null" + Submitted_Moment: + type: + - string + - "null" + source_id: + type: + - string + - "null" + tenant_id: + type: + - string + - "null" + unique_key: + type: string + additionalProperties: true + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - tenant_id + value: "{{ config['insight_tenant_id'] }}" + - type: AddedFieldDefinition + path: + - source_id + value: "{{ config['insight_source_id'] }}" + - type: AddedFieldDefinition + path: + - unique_key + value: >- + {{ config['insight_tenant_id'] }}-{{ config['insight_source_id'] + }}-{{ record['Request_ID'] }} + +concurrency_level: + type: ConcurrencyLevel + default_concurrency: 1 + +spec: + type: Spec + documentation_url: https://community.workday.com/sites/default/files/file-hosting/restapi/index.html + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - insight_tenant_id + - insight_source_id + - workday_base_url + - workday_isu_username + - workday_isu_password + - workday_workers_report_path + - workday_leave_report_path + properties: + insight_tenant_id: + type: string + description: Tenant isolation identifier (UUID). + order: 0 + title: Insight Tenant ID + insight_source_id: + type: string + description: >- + Unique identifier for this Workday connector instance (e.g., + 'workday-acme-prod'). + order: 1 + title: Insight Source ID + examples: + - workday-acme-prod + workday_base_url: + type: string + description: >- + RaaS (Reports-as-a-Service) base URL including the Workday tenant, + without a trailing slash: + https:///ccx/service/customreport2/. + order: 2 + title: Workday RaaS Base URL + examples: + - https://wd5-services1.workday.com/ccx/service/customreport2/acme + workday_isu_username: + type: string + description: >- + Integration System User (ISU) username. The ISU security group must + be shared on both custom reports. + order: 3 + title: ISU Username + workday_isu_password: + type: string + description: Integration System User (ISU) password. + order: 4 + title: ISU Password + airbyte_secret: true + workday_workers_report_path: + type: string + description: >- + Path to the workers custom report relative to the base URL: + /. The report must follow the Insight + report contract (see connector README). + order: 5 + title: Workers Report Path + examples: + - ISU_Insight/Insight_Employee_Sync + workday_leave_report_path: + type: string + description: >- + Path to the time-off custom report relative to the base URL: + /. The report must define From_Date and + To_Date prompts enabled as web service parameters. + order: 6 + title: Leave Report Path + examples: + - ISU_Insight/Insight_Leave_Sync + workday_start_date: + type: string + description: >- + Start date for time-off history (ISO format YYYY-MM-DD), passed to + the leave report From_Date prompt. Defaults to 2020-01-01 if not + set. + order: 7 + title: Start Date + default: "2020-01-01" + examples: + - "2024-01-01" + additionalProperties: true + +metadata: + autoImportSchema: + workers: true + leave_requests: true diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/schema.yml b/src/ingestion/connectors/hr-directory/workday/dbt/schema.yml new file mode 100644 index 000000000..4342f708e --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/schema.yml @@ -0,0 +1,214 @@ +version: 2 + +sources: + - name: workday + schema: bronze_workday + tables: + - name: workers + - name: leave_requests + +models: + - name: workday__to_class_people + description: > + Workday Workers -> class_people (Silver Step 1). + Full-refresh RaaS source. Identity key: email (Work_Email). + Maps Workday Worker_Status to normalised status (active/on_leave/terminated) + and Worker_Type to employment_type (full_time/contractor). + SCD Type 2 fields (valid_from/valid_to) are set as current-state snapshot + (valid_from = Last_Functionally_Updated, valid_to = NULL). + columns: + - name: tenant_id + description: "Tenant isolation identifier" + tests: + - not_null + - name: source_id + description: "Connector instance identifier" + tests: + - not_null + - name: unique_key + description: "Composite dedup key" + tests: + - not_null + - name: source_person_id + description: "Workday Employee ID" + tests: + - not_null + - name: email + description: "Work email — identity key for Silver Step 2" + - name: display_name + description: "Full display name" + - name: first_name + description: "First name" + - name: last_name + description: "Last name" + - name: status + description: "Normalised: active, on_leave, terminated" + tests: + - accepted_values: + arguments: + values: ['active', 'on_leave', 'terminated'] + - name: employment_type + description: "Normalised: full_time (Employee), contractor (Contingent Worker)" + tests: + - accepted_values: + arguments: + values: ['full_time', 'contractor'] + - name: source + description: "Always 'workday'" + tests: + - not_null + - accepted_values: + arguments: + values: ['workday'] + - name: workspace_id + description: "Tenant identifier" + tests: + - not_null + + - name: workday__hr_events + description: > + Workday leave_requests → class_hr_events (Silver Step 2 staging). + One row per time-off request. Joins to workers to resolve email. + Unlike BambooHR, RaaS report columns are flat — no JSON extraction needed. + columns: + - name: insight_tenant_id + tests: + - not_null + - name: source_id + tests: + - not_null + - name: unique_key + tests: + - not_null + - unique + - name: source_person_id + description: "Workday Employee ID" + tests: + - not_null + - name: email + description: "Resolved from workers.Work_Email via JOIN" + - name: event_type + description: "Always 'leave' for leave_requests" + tests: + - not_null + - accepted_values: + arguments: + values: ['leave'] + - name: event_subtype + description: "Time-off type name (policy-defined, client-specific) from Time_Off_Type" + - name: start_date + description: "Leave start date" + tests: + - not_null + - name: end_date + description: "Leave end date" + tests: + - not_null + - name: duration_amount + description: "Quantity (numeric)" + - name: duration_unit + description: "Unit: hours or days" + - name: request_status + description: "Request status (Approved, Submitted, Canceled, ...)" + - name: source + tests: + - not_null + - accepted_values: + arguments: + values: ['workday'] + - name: created_at + description: "When the time-off request was submitted in Workday" + - name: ingested_at + tests: + - not_null + + - name: workday__working_hours + description: > + Workday active workers → class_hr_working_hours (Silver Step 2 staging). + One row per active worker with their scheduled working hours. + Scheduled_Weekly_Hours is Workday-delivered; defaults to 40h/week when empty. + columns: + - name: insight_tenant_id + tests: + - not_null + - name: source_id + tests: + - not_null + - name: unique_key + tests: + - not_null + - unique + - name: source_person_id + description: "Workday Employee ID" + tests: + - not_null + - name: email + description: "Work email — identity key" + tests: + - not_null + - name: display_name + tests: + - not_null + - name: employment_type + description: "Workday Worker_Type (Employee, Contingent Worker)" + - name: source + tests: + - not_null + - accepted_values: + arguments: + values: ['workday'] + - name: working_hours_per_day + description: "Scheduled working hours per working day" + tests: + - not_null + - name: working_hours_per_week + description: "Scheduled working hours per week" + tests: + - not_null + - name: ingested_at + tests: + - not_null + + - name: workday__workers_snapshot + description: > + SCD2 append-only snapshot of Workday workers. + Tracks changes in standard report columns and custom report columns + from raw_data. Custom columns list is passed via dbt var + workday_custom_fields. + columns: + - name: unique_key + tests: + - not_null + - name: _row_hash + description: "cityHash64 of tracked columns for change detection" + - name: _tracked_at + description: "Timestamp when this version was captured" + + - name: workday__workers_fields_history + description: > + Field-level change log for Workday workers. + One row per changed field per version transition. + Includes both standard and custom report columns (from raw_data). + columns: + - name: entity_id + description: "Workday Employee ID" + tests: + - not_null + - name: tenant_id + tests: + - not_null + - name: source_id + tests: + - not_null + - name: field_name + description: "Name of the changed field" + tests: + - not_null + - name: old_value + description: "Previous value (as string)" + - name: new_value + description: "New value (as string)" + - name: updated_at + description: "Timestamp of the change" + tests: + - not_null diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__bronze_promoted.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__bronze_promoted.sql new file mode 100644 index 000000000..73a857688 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__bronze_promoted.sql @@ -0,0 +1,24 @@ +{# ------------------------------------------------------------------------- + Bootstrap model for Workday bronze → RMT promotion. + + Counterpart of `bamboohr__bronze_promoted` for Workday. See ADR-0002 for + the reasoning; the macro `promote_bronze_to_rmt` is idempotent — + already-RMT tables are detected and skipped on subsequent runs. + + All Workday bronze tables carry a `unique_key` column added by the + connector AddFields transformation (formula: + `{tenant}-{source}-{natural_id}`), so `order_by='unique_key'` is + equivalent to the natural-key composite. + ------------------------------------------------------------------------- #} + +-- @cpt-principle:cpt-dataflow-principle-promote-bronze:p1 +{{ config( + materialized='view', + schema='staging', + tags=['workday'] +) }} + +{% do promote_bronze_to_rmt(table='bronze_workday.workers', order_by='unique_key') %} +{% do promote_bronze_to_rmt(table='bronze_workday.leave_requests', order_by='unique_key') %} + +SELECT 1 AS promoted diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql new file mode 100644 index 000000000..54cde79f1 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('workday__bronze_promoted') }} +{{ config( + materialized='view', + schema='staging', + tags=['workday', 'silver:class_hr_events'] +) }} + +SELECT + lr.tenant_id AS insight_tenant_id, + lr.source_id, + lr.unique_key, + lr.Employee_ID AS source_person_id, + w.Work_Email AS email, + 'leave' AS event_type, + lr.Time_Off_Type AS event_subtype, + parseDateTimeBestEffortOrNull(lr.Start_Date) AS start_date, + parseDateTimeBestEffortOrNull(lr.End_Date) AS end_date, + toFloat64OrNull(toString(lr.Quantity)) AS duration_amount, + lr.Unit AS duration_unit, + lr.Status AS request_status, + 'workday' AS source, + parseDateTimeBestEffortOrNull(lr.Submitted_Moment) AS created_at, + lr._airbyte_extracted_at AS ingested_at, + toUnixTimestamp64Milli(lr._airbyte_extracted_at) AS _version +FROM {{ source('workday', 'leave_requests') }} lr +LEFT JOIN {{ source('workday', 'workers') }} w + ON lr.Employee_ID = w.Employee_ID + AND lr.tenant_id = w.tenant_id +WHERE lr.Employee_ID IS NOT NULL + AND parseDateTimeBestEffortOrNull(lr.Start_Date) IS NOT NULL + AND parseDateTimeBestEffortOrNull(lr.End_Date) IS NOT NULL diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__identity_inputs.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__identity_inputs.sql new file mode 100644 index 000000000..131844936 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__identity_inputs.sql @@ -0,0 +1,24 @@ +{{ config( + materialized='incremental', + incremental_strategy='append', + schema='staging', + tags=['workday', 'silver', 'silver:identity_inputs'] +) }} + +{{ identity_inputs_from_history( + fields_history_ref=ref('workday__workers_fields_history'), + source_type='workday', + identity_fields=[ + {'field': 'Work_Email', 'value_type': 'email', 'value_field_name': 'bronze_workday.workers.Work_Email'}, + {'field': 'Employee_ID', 'value_type': 'employee_id', 'value_field_name': 'bronze_workday.workers.Employee_ID'}, + {'field': 'Display_Name', 'value_type': 'display_name', 'value_field_name': 'bronze_workday.workers.Display_Name'}, + {'field': 'First_Name', 'value_type': 'first_name', 'value_field_name': 'bronze_workday.workers.First_Name'}, + {'field': 'Last_Name', 'value_type': 'last_name', 'value_field_name': 'bronze_workday.workers.Last_Name'}, + {'field': 'Supervisory_Organization', 'value_type': 'department', 'value_field_name': 'bronze_workday.workers.Supervisory_Organization'}, + {'field': 'Business_Title', 'value_type': 'job_title', 'value_field_name': 'bronze_workday.workers.Business_Title'}, + {'field': 'Worker_Status', 'value_type': 'status', 'value_field_name': 'bronze_workday.workers.Worker_Status'}, + {'field': 'Manager_Work_Email', 'value_type': 'parent_email', 'value_field_name': 'bronze_workday.workers.Manager_Work_Email'}, + {'field': 'Manager_Employee_ID', 'value_type': 'parent_id', 'value_field_name': 'bronze_workday.workers.Manager_Employee_ID'}, + ], + deactivation_condition="field_name = 'Worker_Status' AND new_value = 'Terminated'" +) }} diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__to_class_people.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__to_class_people.sql new file mode 100644 index 000000000..73d120436 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__to_class_people.sql @@ -0,0 +1,58 @@ +-- depends_on: {{ ref('workday__bronze_promoted') }} +-- Bronze → Silver step 1: Workday Workers → class_people +-- Full-refresh source (RaaS report returns current state only). +-- SCD Type 2: valid_from = Last_Functionally_Updated, valid_to = NULL +-- (current-state snapshot). Full SCD history tracking is handled downstream. +-- @cpt-constraint:cpt-dataflow-constraint-staging-class-column-types-match:p1 +{{ config( + materialized='view', + schema='staging', + tags=['workday', 'silver:class_people'] +) }} + +SELECT + tenant_id, + source_id, + -- SCD2 grain: per (entity, valid_from). Bronze `unique_key` is at entity + -- level (`{tenant}-{source}-{employee_id}`); we extend it with valid_from + -- so silver `class_people` can dedup by a single ORDER BY column. + CAST(concat(coalesce(unique_key, ''), '-', toString(Last_Functionally_Updated)) AS String) AS unique_key, + coalesce(tenant_id, '') AS workspace_id, + -- person_id resolved in Silver Step 2 via Identity Manager + CAST(NULL AS Nullable(UUID)) AS person_id, + parseDateTimeBestEffortOrNull(Last_Functionally_Updated) AS valid_from, + CAST(NULL AS Nullable(DateTime)) AS valid_to, + 'workday' AS source, + Employee_ID AS source_person_id, + Employee_ID AS employee_number, + Display_Name AS display_name, + First_Name AS first_name, + Last_Name AS last_name, + Work_Email AS email, + Business_Title AS job_title, + -- Workday has no freeform department; the supervisory organization is the + -- standard org unit every tenant is guaranteed to have. + Supervisory_Organization AS department_name, + CAST(NULL AS Nullable(UUID)) AS org_unit_id, + Manager_Employee_ID AS manager_person_id, + multiIf( + Worker_Status = 'Terminated', 'terminated', + Worker_Status = 'On Leave', 'on_leave', + 'active' + ) AS status, + multiIf( + Worker_Type = 'Contingent Worker', 'contractor', + 'full_time' + ) AS employment_type, + parseDateTimeBestEffortOrNull(Hire_Date) AS hire_date, + parseDateTimeBestEffortOrNull(Termination_Date) AS termination_date, + Location AS location, + Country AS country, + CAST(NULL AS Nullable(Float64)) AS fte, + CAST(map( + 'job_profile', coalesce(Job_Profile, ''), + 'worker_type', coalesce(Worker_Type, '') + ) AS Map(String, String)) AS custom_str_attrs, + CAST(map() AS Map(String, Float64)) AS custom_num_attrs, + _airbyte_extracted_at AS ingested_at +FROM {{ source('workday', 'workers') }} diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_fields_history.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_fields_history.sql new file mode 100644 index 000000000..46cde73cd --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_fields_history.sql @@ -0,0 +1,21 @@ +-- depends_on: {{ ref('workday__bronze_promoted') }} +{{ config( + materialized='table', + schema='staging', + tags=['workday', 'silver'] +) }} + +{{ fields_history( + snapshot_ref=ref('workday__workers_snapshot'), + entity_id_col='Employee_ID', + fields=[ + 'Display_Name', 'First_Name', 'Last_Name', 'Work_Email', + 'Business_Title', 'Job_Profile', 'Worker_Type', 'Worker_Status', + 'Supervisory_Organization', + 'Manager_Employee_ID', 'Manager_Work_Email', + 'Location', 'Country', 'City', + 'Hire_Date', 'Original_Hire_Date', 'Termination_Date', + 'Scheduled_Weekly_Hours' + ], + fields_raw_data=var('workday_custom_fields', []) +) }} diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_snapshot.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_snapshot.sql new file mode 100644 index 000000000..ca7d363e0 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__workers_snapshot.sql @@ -0,0 +1,25 @@ +-- depends_on: {{ ref('workday__bronze_promoted') }} +{{ config( + materialized='incremental', + incremental_strategy='append', + schema='staging', + tags=['workday'] +) }} + +{# Last_Functionally_Updated is deliberately NOT tracked: it changes on any + worker update, including fields we do not monitor, and would create + spurious SCD2 versions. #} +{{ snapshot( + source_ref=source('workday', 'workers'), + unique_key_col='unique_key', + check_cols=[ + 'Display_Name', 'First_Name', 'Last_Name', 'Work_Email', + 'Business_Title', 'Job_Profile', 'Worker_Type', 'Worker_Status', + 'Supervisory_Organization', + 'Manager_Employee_ID', 'Manager_Work_Email', + 'Location', 'Country', 'City', + 'Hire_Date', 'Original_Hire_Date', 'Termination_Date', + 'Scheduled_Weekly_Hours' + ], + check_raw_data_cols=var('workday_custom_fields', []) +) }} diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__working_hours.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__working_hours.sql new file mode 100644 index 000000000..21316c480 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__working_hours.sql @@ -0,0 +1,27 @@ +-- depends_on: {{ ref('workday__bronze_promoted') }} +{{ config( + materialized='view', + schema='staging', + tags=['workday', 'silver:class_hr_working_hours'] +) }} + +SELECT + tenant_id AS insight_tenant_id, + source_id, + unique_key, + Employee_ID AS source_person_id, + Work_Email AS email, + COALESCE(Display_Name, Work_Email) AS display_name, + Worker_Type AS employment_type, + 'workday' AS source, + -- Scheduled_Weekly_Hours is a Workday-delivered field; fall back to a + -- 40h week when the report column is empty for a worker. + coalesce(toFloat64OrNull(toString(Scheduled_Weekly_Hours)), 40.0) / 5.0 + AS working_hours_per_day, + coalesce(toFloat64OrNull(toString(Scheduled_Weekly_Hours)), 40.0) + AS working_hours_per_week, + _airbyte_extracted_at AS ingested_at +FROM {{ source('workday', 'workers') }} +WHERE Worker_Status = 'Active' + AND Employee_ID IS NOT NULL + AND Work_Email IS NOT NULL diff --git a/src/ingestion/connectors/hr-directory/workday/descriptor.yaml b/src/ingestion/connectors/hr-directory/workday/descriptor.yaml new file mode 100644 index 000000000..a528cb5c4 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/descriptor.yaml @@ -0,0 +1,14 @@ +name: workday +version: "1.0.0" +schedule: 0 4 * * * +dbt_select: tag:workday+ +workflow: sync +connection: + namespace: bronze_workday +secret: + required_fields: + - workday_base_url + - workday_isu_username + - workday_isu_password + - workday_workers_report_path + - workday_leave_report_path diff --git a/src/ingestion/connectors/hr-directory/workday/fixtures/leave_requests.json b/src/ingestion/connectors/hr-directory/workday/fixtures/leave_requests.json new file mode 100644 index 000000000..8ff41e975 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/fixtures/leave_requests.json @@ -0,0 +1,48 @@ +{ + "Report_Entry": [ + { + "Request_ID": "TOR-90001", + "Employee_ID": "21001", + "Time_Off_Type": "Vacation", + "Start_Date": "2026-07-13", + "End_Date": "2026-07-24", + "Quantity": "10", + "Unit": "days", + "Status": "Approved", + "Submitted_Moment": "2026-05-30T10:02:00.000Z" + }, + { + "Request_ID": "TOR-90002", + "Employee_ID": "21002", + "Time_Off_Type": "Parental Leave", + "Start_Date": "2026-06-01", + "End_Date": "2026-08-31", + "Quantity": "65", + "Unit": "days", + "Status": "Approved", + "Submitted_Moment": "2026-04-15T08:30:00.000Z" + }, + { + "Request_ID": "TOR-90003", + "Employee_ID": "21001", + "Time_Off_Type": "Sick Leave", + "Start_Date": "2026-02-09", + "End_Date": "2026-02-10", + "Quantity": "16", + "Unit": "hours", + "Status": "Approved", + "Submitted_Moment": "2026-02-09T07:45:00.000Z" + }, + { + "Request_ID": "TOR-90004", + "Employee_ID": "C-3007", + "Time_Off_Type": "Unpaid Time Off", + "Start_Date": "2026-06-22", + "End_Date": "2026-06-23", + "Quantity": "2", + "Unit": "days", + "Status": "Submitted", + "Submitted_Moment": "2026-06-08T14:12:00.000Z" + } + ] +} diff --git a/src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py b/src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py new file mode 100644 index 000000000..734d410ee --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Mock Workday RaaS server for local connector development. + +Replays fixture responses in the RaaS shape ({"Report_Entry": [...]}) and +enforces the parts of the RaaS protocol the connector relies on: + +- HTTP Basic auth header must be present (any credentials accepted) +- `format=json` query parameter is required +- the leave report requires `From_Date` and `To_Date` parameters, mirroring + a report whose prompts are enabled as web service parameters + +Routing is by report name (last path segment): +- ...//Insight_Employee_Sync -> workers.json +- ...//Insight_Leave_Sync -> leave_requests.json + +Usage: + python3 mock_raas.py [--port 8765] + +Point the connector at it via the K8s Secret file: + workday_base_url: "http://host.docker.internal:/ccx/service/customreport2/acme" +""" + +from __future__ import annotations + +import argparse +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +FIXTURES_DIR = Path(__file__).resolve().parent + +REPORT_FIXTURES = { + "Insight_Employee_Sync": "workers.json", + "Insight_Leave_Sync": "leave_requests.json", +} + +PROMPT_REQUIRED = { + "Insight_Leave_Sync": ("From_Date", "To_Date"), +} + + +class MockRaasHandler(BaseHTTPRequestHandler): + def _reply(self, status: int, payload: dict) -> None: + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 - http.server API + parsed = urlparse(self.path) + params = parse_qs(parsed.query) + report = parsed.path.rstrip("/").rsplit("/", 1)[-1] + + if "Authorization" not in self.headers: + self._reply(401, {"error": "Authentication required"}) + return + + if report not in REPORT_FIXTURES: + self._reply(404, {"error": f"Unknown report: {report}"}) + return + + if params.get("format") != ["json"]: + self._reply(400, {"error": "format=json query parameter is required"}) + return + + missing = [p for p in PROMPT_REQUIRED.get(report, ()) if p not in params] + if missing: + # Real RaaS rejects parameters that do not match report prompts and + # runs unfiltered when prompts are omitted; the mock is stricter so + # a manifest that stops sending the prompts fails loudly in tests. + self._reply(400, {"error": f"Missing prompt parameters: {missing}"}) + return + + fixture = FIXTURES_DIR / REPORT_FIXTURES[report] + self._reply(200, json.loads(fixture.read_text())) + + def log_message(self, fmt: str, *args) -> None: + print(f"[mock-raas] {self.address_string()} {fmt % args}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", type=int, default=8765) + args = parser.parse_args() + + server = HTTPServer(("0.0.0.0", args.port), MockRaasHandler) + print(f"[mock-raas] serving fixtures from {FIXTURES_DIR} on :{args.port}") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/src/ingestion/connectors/hr-directory/workday/fixtures/workers.json b/src/ingestion/connectors/hr-directory/workday/fixtures/workers.json new file mode 100644 index 000000000..378581149 --- /dev/null +++ b/src/ingestion/connectors/hr-directory/workday/fixtures/workers.json @@ -0,0 +1,119 @@ +{ + "Report_Entry": [ + { + "Employee_ID": "21001", + "Display_Name": "Alice Johnson", + "First_Name": "Alice", + "Last_Name": "Johnson", + "Work_Email": "alice.johnson@acme.com", + "Business_Title": "Senior Software Engineer", + "Job_Profile": "Software Engineer 3", + "Worker_Type": "Employee", + "Worker_Status": "Active", + "Supervisory_Organization": "Platform Engineering", + "Manager_Employee_ID": "21005", + "Manager_Work_Email": "edward.kim@acme.com", + "Location": "Berlin Office", + "Country": "Germany", + "City": "Berlin", + "Hire_Date": "2021-03-15", + "Original_Hire_Date": "2021-03-15", + "Termination_Date": "", + "Last_Functionally_Updated": "2026-05-28T09:14:02.000Z", + "Scheduled_Weekly_Hours": "40", + "Cost_Center": "CC-1200 Engineering" + }, + { + "Employee_ID": "21002", + "Display_Name": "Bob Smith", + "First_Name": "Robert", + "Last_Name": "Smith", + "Work_Email": "bob.smith@acme.com", + "Business_Title": "Engineering Manager", + "Job_Profile": "Engineering Manager 1", + "Worker_Type": "Employee", + "Worker_Status": "On Leave", + "Supervisory_Organization": "Platform Engineering", + "Manager_Employee_ID": "21005", + "Manager_Work_Email": "edward.kim@acme.com", + "Location": "Remote", + "Country": "Spain", + "City": "Valencia", + "Hire_Date": "2019-09-01", + "Original_Hire_Date": "2017-02-01", + "Termination_Date": "", + "Last_Functionally_Updated": "2026-06-02T16:40:11.000Z", + "Scheduled_Weekly_Hours": "40", + "Cost_Center": "CC-1200 Engineering" + }, + { + "Employee_ID": "21003", + "Display_Name": "Carol Diaz", + "First_Name": "Carol", + "Last_Name": "Diaz", + "Work_Email": "carol.diaz@acme.com", + "Business_Title": "Product Designer", + "Job_Profile": "UX Designer 2", + "Worker_Type": "Employee", + "Worker_Status": "Terminated", + "Supervisory_Organization": "Product Design", + "Manager_Employee_ID": "21005", + "Manager_Work_Email": "edward.kim@acme.com", + "Location": "Lisbon Office", + "Country": "Portugal", + "City": "Lisbon", + "Hire_Date": "2022-01-10", + "Original_Hire_Date": "2022-01-10", + "Termination_Date": "2026-04-30", + "Last_Functionally_Updated": "2026-04-30T18:00:00.000Z", + "Scheduled_Weekly_Hours": "32", + "Cost_Center": "CC-1400 Design" + }, + { + "Employee_ID": "C-3007", + "Display_Name": "Dmitri Volkov", + "First_Name": "Dmitri", + "Last_Name": "Volkov", + "Work_Email": "", + "Business_Title": "QA Contractor", + "Job_Profile": "Contingent QA", + "Worker_Type": "Contingent Worker", + "Worker_Status": "Active", + "Supervisory_Organization": "Quality Assurance", + "Manager_Employee_ID": "21005", + "Manager_Work_Email": "edward.kim@acme.com", + "Location": "Remote", + "Country": "Serbia", + "City": "Belgrade", + "Hire_Date": "2025-11-03", + "Original_Hire_Date": "2025-11-03", + "Termination_Date": "", + "Last_Functionally_Updated": "2026-05-20T07:55:43.000Z", + "Scheduled_Weekly_Hours": "20", + "Cost_Center": "CC-1300 QA" + }, + { + "Employee_ID": "21005", + "Display_Name": "Edward Kim", + "First_Name": "Edward", + "Last_Name": "Kim", + "Work_Email": "edward.kim@acme.com", + "Business_Title": "VP of Engineering", + "Job_Profile": "Engineering Executive", + "Worker_Type": "Employee", + "Worker_Status": "Active", + "Supervisory_Organization": "Engineering", + "Manager_Employee_ID": "", + "Manager_Work_Email": "", + "Location": "Berlin Office", + "Country": "Germany", + "City": "Berlin", + "Hire_Date": "2018-06-01", + "Original_Hire_Date": "2018-06-01", + "Termination_Date": "", + "Last_Functionally_Updated": "2026-03-12T11:22:30.000Z", + "Scheduled_Weekly_Hours": "40", + "Cost_Center": "CC-1000 Leadership" + } + ] +} diff --git a/src/ingestion/secrets/connectors/workday.yaml.example b/src/ingestion/secrets/connectors/workday.yaml.example new file mode 100644 index 000000000..708fffaa3 --- /dev/null +++ b/src/ingestion/secrets/connectors/workday.yaml.example @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: insight-workday-main + labels: + app.kubernetes.io/part-of: insight + annotations: + insight.cyberfabric.com/connector: workday + insight.cyberfabric.com/source-id: workday-main +type: Opaque +stringData: + workday_base_url: "CHANGE_ME" # https:///ccx/service/customreport2/ + workday_isu_username: "CHANGE_ME" # Integration System User + workday_isu_password: "CHANGE_ME" # ISU password + workday_workers_report_path: "CHANGE_ME" # / + workday_leave_report_path: "CHANGE_ME" # / + workday_start_date: "2020-01-01" # Optional: time-off history start date From 175c82aec908abd8b2706c2575feda83fe9e9c31 Mon Sep 17 00:00:00 2001 From: Roman Mitasov Date: Fri, 12 Jun 2026 18:47:35 +0300 Subject: [PATCH 2/2] fix(ingestion): address workday PR review findings - Forward raw_data on leave_requests too: the leave report is also customer-built, so extra columns must survive into Bronze (manifest transformation + schema, DESIGN table, README note). - Add source_id to the hr_events workers JOIN to prevent cross-instance identity mixing when one tenant runs multiple workday sources. - Mock RaaS now requires the Basic auth scheme, not just any Authorization header, so connector auth regressions fail the rig. - Label the deployment topology fence in DESIGN.md (MD040). Re-verified against the mock: read returns 5 workers / 4 leave_requests with 0 errors, leave records carry raw_data, Bearer auth is rejected. Signed-off-by: Roman Mitasov Co-Authored-By: Claude Fable 5 --- .../connectors/hr-directory/workday/specs/DESIGN.md | 5 +++-- src/ingestion/connectors/hr-directory/workday/README.md | 2 ++ .../connectors/hr-directory/workday/connector.yaml | 9 +++++++++ .../hr-directory/workday/dbt/workday__hr_events.sql | 1 + .../hr-directory/workday/fixtures/mock_raas.py | 5 +++-- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/components/connectors/hr-directory/workday/specs/DESIGN.md b/docs/components/connectors/hr-directory/workday/specs/DESIGN.md index 79dc0a8a3..fe420ff87 100644 --- a/docs/components/connectors/hr-directory/workday/specs/DESIGN.md +++ b/docs/components/connectors/hr-directory/workday/specs/DESIGN.md @@ -166,7 +166,7 @@ Defines the complete Workday connector as a YAML declarative manifest — the si ##### Responsibility scope -Defines 2 streams with: ISU Basic auth, GET requests to config-driven report paths, `format=json` and date-prompt query parameters, `NoPagination`, full refresh sync, `AddFields` for `raw_data` (workers only), `tenant_id`, `source_id`, and `unique_key`, and inline JSON schemas matching the report contract. +Defines 2 streams with: ISU Basic auth, GET requests to config-driven report paths, `format=json` and date-prompt query parameters, `NoPagination`, full refresh sync, `AddFields` for `raw_data`, `tenant_id`, `source_id`, and `unique_key`, and inline JSON schemas matching the report contract. ##### Responsibility boundaries @@ -399,6 +399,7 @@ sequenceDiagram | `Unit` | String | `hours` / `days` | | `Status` | String | Request status (`Approved`, `Submitted`, `Canceled`, …) | | `Submitted_Moment` | String | When the request was submitted | +| `raw_data` | String (JSON) | Full report row incl. extra customer columns — custom-field passthrough | | `_airbyte_extracted_at` | DateTime | Extraction timestamp (UTC) — auto-generated | --- @@ -411,7 +412,7 @@ sequenceDiagram - [ ] `p1` - **ID**: `cpt-insightspec-topology-wd-deployment` -``` +```text Connection: workday-{source-id} ├── Source image: airbyte/source-declarative-manifest ├── Manifest: src/ingestion/connectors/hr-directory/workday/connector.yaml diff --git a/src/ingestion/connectors/hr-directory/workday/README.md b/src/ingestion/connectors/hr-directory/workday/README.md index 46acabe1d..354d5cb3f 100644 --- a/src/ingestion/connectors/hr-directory/workday/README.md +++ b/src/ingestion/connectors/hr-directory/workday/README.md @@ -58,6 +58,8 @@ Extra columns (custom fields, calculated fields) are allowed: they land in the B The report MUST define two prompts enabled as web service parameters: `From_Date` and `To_Date` (date range filter on time-off dates). The connector passes `From_Date` from `workday_start_date` and `To_Date` as the current UTC date. +As with the workers report, extra columns are allowed and land in the Bronze `raw_data` column. + ## K8s Secret ```yaml diff --git a/src/ingestion/connectors/hr-directory/workday/connector.yaml b/src/ingestion/connectors/hr-directory/workday/connector.yaml index 954d4bf06..bce0db5e2 100644 --- a/src/ingestion/connectors/hr-directory/workday/connector.yaml +++ b/src/ingestion/connectors/hr-directory/workday/connector.yaml @@ -243,6 +243,11 @@ streams: type: - string - "null" + raw_data: + type: + - object + - "null" + additionalProperties: true source_id: type: - string @@ -257,6 +262,10 @@ streams: transformations: - type: AddFields fields: + - type: AddedFieldDefinition + path: + - raw_data + value: "{{ record }}" - type: AddedFieldDefinition path: - tenant_id diff --git a/src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql b/src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql index 54cde79f1..1e5814a66 100644 --- a/src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql +++ b/src/ingestion/connectors/hr-directory/workday/dbt/workday__hr_events.sql @@ -26,6 +26,7 @@ FROM {{ source('workday', 'leave_requests') }} lr LEFT JOIN {{ source('workday', 'workers') }} w ON lr.Employee_ID = w.Employee_ID AND lr.tenant_id = w.tenant_id + AND lr.source_id = w.source_id WHERE lr.Employee_ID IS NOT NULL AND parseDateTimeBestEffortOrNull(lr.Start_Date) IS NOT NULL AND parseDateTimeBestEffortOrNull(lr.End_Date) IS NOT NULL diff --git a/src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py b/src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py index 734d410ee..ac725b464 100644 --- a/src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py +++ b/src/ingestion/connectors/hr-directory/workday/fixtures/mock_raas.py @@ -54,8 +54,9 @@ def do_GET(self) -> None: # noqa: N802 - http.server API params = parse_qs(parsed.query) report = parsed.path.rstrip("/").rsplit("/", 1)[-1] - if "Authorization" not in self.headers: - self._reply(401, {"error": "Authentication required"}) + auth = self.headers.get("Authorization", "") + if not auth.lower().startswith("basic "): + self._reply(401, {"error": "Basic authentication required"}) return if report not in REPORT_FIXTURES: