diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index 3a90688c3d7..343874283cd 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -1,4 +1,11 @@ # newer versions go on top +- version: "3.10.4" + changes: + - description: >- + Fix `document_parsing_exception` on `o365.audit.Sender` by reverting it to `keyword` + and moving structured Sender objects to the new `o365.audit.SenderEntity` field. + type: bugfix + link: https://github.com/elastic/integrations/pull/20068 - version: "3.10.3" changes: - description: Fix `script_item_attachments` processor failing with a Painless regex character-limit error on long attachment filenames that contain parentheses. The regex is replaced with string operations that have no backtracking cost. diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-edge-cases.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-edge-cases.json-expected.json index 3d3e940bace..343652f10a8 100644 --- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-edge-cases.json-expected.json +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-edge-cases.json-expected.json @@ -206,7 +206,7 @@ "MailboxOwnerSid": "S-1-5-21-1111111111-2222222222-3333333333-44444444", "RecordType": "2", "ResultStatus": "Succeeded", - "Sender": { + "SenderEntity": { "DisplayName": "IT Support", "MRI": "8:orgid:a1b2c3d4-e5f6-7890-abcd-ef1234567890", "OrganizationId": "aaaabbbb-cccc-dddd-eeee-ffffffffffff", diff --git a/packages/o365/data_stream/audit/_dev/test/scripts/upgrade.txt b/packages/o365/data_stream/audit/_dev/test/scripts/upgrade.txt index db15d7fc5c8..3b68369083d 100644 --- a/packages/o365/data_stream/audit/_dev/test/scripts/upgrade.txt +++ b/packages/o365/data_stream/audit/_dev/test/scripts/upgrade.txt @@ -263,7 +263,7 @@ rules: body: |- {"contentType":"Audit.General","status":"enabled","webhook":null} - # List content for Audit.SharePoint (includes original + new content). + # List content for Audit.SharePoint (only new content post-upgrade). - path: /api/v1.0/test-cel-tenant-id/activity/feed/subscriptions/content methods: [GET] query_params: @@ -280,9 +280,9 @@ rules: Content-Type: - "application/json" body: |- - [{"contentType":"Audit.SharePoint","contentId":"sp-content-1","contentUri":"http://{{ hostname }}:{{ env "PORT" }}/api/v1.0/test-cel-tenant-id/activity/feed/audit/sp-content-1","contentCreated":"{{ .request.vars.endTime }}","contentExpiration":"2199-12-31T23:59:59.000Z"},{"contentType":"Audit.SharePoint","contentId":"sp-content-2","contentUri":"http://{{ hostname }}:{{ env "PORT" }}/api/v1.0/test-cel-tenant-id/activity/feed/audit/sp-content-2","contentCreated":"{{ .request.vars.endTime }}","contentExpiration":"2199-12-31T23:59:59.000Z"}] + [{"contentType":"Audit.SharePoint","contentId":"sp-content-2","contentUri":"http://{{ hostname }}:{{ env "PORT" }}/api/v1.0/test-cel-tenant-id/activity/feed/audit/sp-content-2","contentCreated":"{{ .request.vars.endTime }}","contentExpiration":"2199-12-31T23:59:59.000Z"}] - # List content for Audit.General (includes original + new content). + # List content for Audit.General (only new content post-upgrade). - path: /api/v1.0/test-cel-tenant-id/activity/feed/subscriptions/content methods: [GET] query_params: @@ -299,21 +299,7 @@ rules: Content-Type: - "application/json" body: |- - [{"contentType":"Audit.General","contentId":"gen-content-1","contentUri":"http://{{ hostname }}:{{ env "PORT" }}/api/v1.0/test-cel-tenant-id/activity/feed/audit/gen-content-1","contentCreated":"{{ .request.vars.endTime }}","contentExpiration":"2199-12-31T23:59:59.000Z"},{"contentType":"Audit.General","contentId":"gen-content-2","contentUri":"http://{{ hostname }}:{{ env "PORT" }}/api/v1.0/test-cel-tenant-id/activity/feed/audit/gen-content-2","contentCreated":"{{ .request.vars.endTime }}","contentExpiration":"2199-12-31T23:59:59.000Z"}] - - # Fetch SharePoint content (original). - - path: /api/v1.0/test-cel-tenant-id/activity/feed/audit/sp-content-1 - methods: [GET] - request_headers: - Authorization: - - "Bearer test-token" - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: |- - [{"Id":"sp-event-001","CreationTime":"2020-02-07T16:43:53","Workload":"SharePoint","Operation":"PageViewed","RecordType":4,"OrganizationId":"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd","UserId":"user@test.onmicrosoft.com","ClientIP":"192.0.2.1"}] + [{"contentType":"Audit.General","contentId":"gen-content-2","contentUri":"http://{{ hostname }}:{{ env "PORT" }}/api/v1.0/test-cel-tenant-id/activity/feed/audit/gen-content-2","contentCreated":"{{ .request.vars.endTime }}","contentExpiration":"2199-12-31T23:59:59.000Z"}] # Fetch SharePoint content (new, post-upgrade). - path: /api/v1.0/test-cel-tenant-id/activity/feed/audit/sp-content-2 @@ -329,20 +315,6 @@ rules: body: |- [{"Id":"sp-event-002","CreationTime":"2020-02-07T17:10:22","Workload":"SharePoint","Operation":"FileUploaded","RecordType":6,"OrganizationId":"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd","UserId":"user@test.onmicrosoft.com","ClientIP":"192.0.2.1"}] - # Fetch General content (original). - - path: /api/v1.0/test-cel-tenant-id/activity/feed/audit/gen-content-1 - methods: [GET] - request_headers: - Authorization: - - "Bearer test-token" - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: |- - [{"Id":"gen-event-001","CreationTime":"2020-02-28T09:42:45","Workload":"Yammer","Operation":"GroupCreation","RecordType":22,"OrganizationId":"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd","UserId":"user@test.onmicrosoft.com","ClientIP":"192.0.2.2"}] - # Fetch General content (new, post-upgrade). - path: /api/v1.0/test-cel-tenant-id/activity/feed/audit/gen-content-2 methods: [GET] diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 50388ac1dca..b0bef17b824 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -73,6 +73,19 @@ processors: ctx.o365audit[field] = new BigDecimal(value).toBigIntegerExact().toString(); } } + - rename: + tag: rename_sender_object_to_sender_entity + description: >- + o365audit.Sender is polymorphic across workloads: it can be reported as + a plain email address (keyword), while other records report it as a + structured identity object. A single field can't be mapped as both, + so structured Sender values are moved to SenderEntity, keeping Sender + consistently a keyword. + field: o365audit.Sender + target_field: o365audit.SenderEntity + if: ctx.o365audit?.Sender instanceof Map + ignore_missing: true + - fingerprint: tag: fingerprint_443be558 fields: diff --git a/packages/o365/data_stream/audit/fields/fields.yml b/packages/o365/data_stream/audit/fields/fields.yml index 0c162b51473..23f7fd8f32a 100644 --- a/packages/o365/data_stream/audit/fields/fields.yml +++ b/packages/o365/data_stream/audit/fields/fields.yml @@ -557,6 +557,8 @@ - name: SecurityComplianceCenterEventType type: keyword - name: Sender + type: keyword + - name: SenderEntity type: object object_type: keyword object_type_mapping_type: '*' diff --git a/packages/o365/docs/README.md b/packages/o365/docs/README.md index fa4f0095804..f260c60f0e2 100644 --- a/packages/o365/docs/README.md +++ b/packages/o365/docs/README.md @@ -526,7 +526,8 @@ An example event for `audit` looks as following: | o365.audit.ResultStatus | | keyword | | o365.audit.RunningTime | | keyword | | o365.audit.SecurityComplianceCenterEventType | | keyword | -| o365.audit.Sender | | object | +| o365.audit.Sender | | keyword | +| o365.audit.SenderEntity | | object | | o365.audit.SenderIP | | keyword | | o365.audit.SenderIp | | keyword | | o365.audit.SensitiveInfoDetectionIsIncluded | | boolean | diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml index 892aab756c7..4d8b5e8ee68 100644 --- a/packages/o365/manifest.yml +++ b/packages/o365/manifest.yml @@ -1,6 +1,6 @@ name: o365 title: Microsoft Office 365 -version: "3.10.3" +version: "3.10.4" description: Collect logs from Microsoft Office 365 with Elastic Agent. type: integration format_version: "3.2.3"