From 4e0bc4f55297f193e5b5426449209be2317de6de Mon Sep 17 00:00:00 2001 From: Matthew Date: Mon, 12 Jan 2026 12:57:10 +0000 Subject: [PATCH 1/5] Add kernel_process_events fields (#716) (cherry picked from commit e745043e004708f4416b50b7c0b1fbe0b4a7e86c) --- custom_documentation/doc/endpoint/metrics/metrics.md | 2 ++ .../src/endpoint/data_stream/metrics/metrics.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/custom_documentation/doc/endpoint/metrics/metrics.md b/custom_documentation/doc/endpoint/metrics/metrics.md index 084108c9e..f67c737d1 100644 --- a/custom_documentation/doc/endpoint/metrics/metrics.md +++ b/custom_documentation/doc/endpoint/metrics/metrics.md @@ -130,6 +130,8 @@ This is an internal state management document that includes metrics on Endpoint' | Endpoint.metrics.system_impact.file_events.week_ms | | Endpoint.metrics.system_impact.kernel_api_events.week_idle_ms | | Endpoint.metrics.system_impact.kernel_api_events.week_ms | +| Endpoint.metrics.system_impact.kernel_process_events.week_idle_ms | +| Endpoint.metrics.system_impact.kernel_process_events.week_ms | | Endpoint.metrics.system_impact.ldap_client_events.week_idle_ms | | Endpoint.metrics.system_impact.ldap_client_events.week_ms | | Endpoint.metrics.system_impact.library_load_events.week_idle_ms | diff --git a/custom_documentation/src/endpoint/data_stream/metrics/metrics.yaml b/custom_documentation/src/endpoint/data_stream/metrics/metrics.yaml index 6a2a62d0c..9606b0e4a 100644 --- a/custom_documentation/src/endpoint/data_stream/metrics/metrics.yaml +++ b/custom_documentation/src/endpoint/data_stream/metrics/metrics.yaml @@ -137,6 +137,8 @@ fields: - Endpoint.metrics.system_impact.file_events.week_ms - Endpoint.metrics.system_impact.kernel_api_events.week_idle_ms - Endpoint.metrics.system_impact.kernel_api_events.week_ms + - Endpoint.metrics.system_impact.kernel_process_events.week_idle_ms + - Endpoint.metrics.system_impact.kernel_process_events.week_ms - Endpoint.metrics.system_impact.ldap_client_events.week_idle_ms - Endpoint.metrics.system_impact.ldap_client_events.week_ms - Endpoint.metrics.system_impact.library_load_events.week_idle_ms From d0c01c9268490afeed54706365f1a444fed01434 Mon Sep 17 00:00:00 2001 From: Asuka Nakajima Date: Wed, 11 Mar 2026 19:50:07 +0900 Subject: [PATCH 2/5] Add missing field in the ASM API events custom documentation (#726) * add missing field * add generated file (cherry picked from commit 89c1a228089dfdaebc938a5986685683ab793b64) --- custom_documentation/doc/endpoint/api/windows/windows_api_asm.md | 1 + .../src/endpoint/data_stream/api/windows/windows_api_asm.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/custom_documentation/doc/endpoint/api/windows/windows_api_asm.md b/custom_documentation/doc/endpoint/api/windows/windows_api_asm.md index 4fac792b4..f9ba5fce1 100644 --- a/custom_documentation/doc/endpoint/api/windows/windows_api_asm.md +++ b/custom_documentation/doc/endpoint/api/windows/windows_api_asm.md @@ -50,6 +50,7 @@ This event is generated when ETW AttackSurfaceMonitor events are generated. | host.os.version | | message | | process.Ext.api.behaviors | +| process.Ext.api.metadata.security_descriptor | | process.Ext.api.name | | process.Ext.api.parameters.device | | process.Ext.api.parameters.io_control_code | diff --git a/custom_documentation/src/endpoint/data_stream/api/windows/windows_api_asm.yaml b/custom_documentation/src/endpoint/data_stream/api/windows/windows_api_asm.yaml index 6b6bdff94..f3a55d288 100644 --- a/custom_documentation/src/endpoint/data_stream/api/windows/windows_api_asm.yaml +++ b/custom_documentation/src/endpoint/data_stream/api/windows/windows_api_asm.yaml @@ -54,6 +54,7 @@ fields: - host.os.version - message - process.Ext.api.behaviors + - process.Ext.api.metadata.security_descriptor - process.Ext.api.name - process.Ext.api.parameters.device - process.Ext.api.parameters.io_control_code From 880100eed46359c0215461c438759ec69c147a47 Mon Sep 17 00:00:00 2001 From: Nicholas Berlin <56366649+nicholasberlin@users.noreply.github.com> Date: Mon, 4 May 2026 09:40:59 -0400 Subject: [PATCH 3/5] Fix Linux and macOS behavior alert custom doc for subtechnique fields (#740) Some rules emit populated threat.technique.subtechnique.{id,name,reference} fields. The Linux and macOS malicious_behavior_alert custom_documentation only listed the parent threat.technique.subtechnique path, causing the EAF custom-documentation validator to fail with undocumented keys. Mirrors the shape already present in the Windows alert (PR #614 era). Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit 332151dc32820f68e6014d80baf0080e48486914) --- .../endpoint/alerts/linux/linux_malicious_behavior_alert.md | 3 +++ .../endpoint/alerts/macos/macos_malicious_behavior_alert.md | 3 +++ .../alerts/linux/linux_malicious_behavior_alert.yaml | 3 +++ .../alerts/macos/macos_malicious_behavior_alert.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/custom_documentation/doc/endpoint/alerts/linux/linux_malicious_behavior_alert.md b/custom_documentation/doc/endpoint/alerts/linux/linux_malicious_behavior_alert.md index 4706516b2..d4a7b0409 100644 --- a/custom_documentation/doc/endpoint/alerts/linux/linux_malicious_behavior_alert.md +++ b/custom_documentation/doc/endpoint/alerts/linux/linux_malicious_behavior_alert.md @@ -111,6 +111,9 @@ This alert is generated when a Malicious Behavior alert occurs. | threat.technique.name | | threat.technique.reference | | threat.technique.subtechnique | +| threat.technique.subtechnique.id | +| threat.technique.subtechnique.name | +| threat.technique.subtechnique.reference | | user.Ext.real.id | | user.Ext.real.name | | user.id | diff --git a/custom_documentation/doc/endpoint/alerts/macos/macos_malicious_behavior_alert.md b/custom_documentation/doc/endpoint/alerts/macos/macos_malicious_behavior_alert.md index c3601b527..27517de50 100644 --- a/custom_documentation/doc/endpoint/alerts/macos/macos_malicious_behavior_alert.md +++ b/custom_documentation/doc/endpoint/alerts/macos/macos_malicious_behavior_alert.md @@ -98,6 +98,9 @@ This alert is generated when a Malicious Behavior alert occurs. | threat.technique.name | | threat.technique.reference | | threat.technique.subtechnique | +| threat.technique.subtechnique.id | +| threat.technique.subtechnique.name | +| threat.technique.subtechnique.reference | | user.Ext.real.id | | user.Ext.real.name | | user.id | diff --git a/custom_documentation/src/endpoint/data_stream/alerts/linux/linux_malicious_behavior_alert.yaml b/custom_documentation/src/endpoint/data_stream/alerts/linux/linux_malicious_behavior_alert.yaml index 61d1ea637..1494bb99e 100644 --- a/custom_documentation/src/endpoint/data_stream/alerts/linux/linux_malicious_behavior_alert.yaml +++ b/custom_documentation/src/endpoint/data_stream/alerts/linux/linux_malicious_behavior_alert.yaml @@ -116,6 +116,9 @@ fields: - threat.technique.name - threat.technique.reference - threat.technique.subtechnique + - threat.technique.subtechnique.id + - threat.technique.subtechnique.name + - threat.technique.subtechnique.reference - user.Ext.real.id - user.Ext.real.name - user.id diff --git a/custom_documentation/src/endpoint/data_stream/alerts/macos/macos_malicious_behavior_alert.yaml b/custom_documentation/src/endpoint/data_stream/alerts/macos/macos_malicious_behavior_alert.yaml index afebb3fa8..7e62a9da7 100644 --- a/custom_documentation/src/endpoint/data_stream/alerts/macos/macos_malicious_behavior_alert.yaml +++ b/custom_documentation/src/endpoint/data_stream/alerts/macos/macos_malicious_behavior_alert.yaml @@ -103,6 +103,9 @@ fields: - threat.technique.name - threat.technique.reference - threat.technique.subtechnique + - threat.technique.subtechnique.id + - threat.technique.subtechnique.name + - threat.technique.subtechnique.reference - user.Ext.real.id - user.Ext.real.name - user.id From 7a8ca5fcc212e98576cfc58373f74a46ee788bf9 Mon Sep 17 00:00:00 2001 From: Nicholas Berlin Date: Fri, 10 Jul 2026 10:33:45 -0400 Subject: [PATCH 4/5] Add mappings for process and security event sources metrics Endpoint.metrics.documents_volume.{process,security}_events.sources.* were added to custom_documentation only (#652, #653), so the fields were never mapped in the package. The metrics index template sets dynamic: false, so these fields are currently not indexed on any stack. Define them in custom_schemas/custom_endpoint.yml mirroring the existing api_events.sources.* mappings, and regenerate with make. (cherry picked from commit 322d76a96c4f5a887425da3df7b3487f03199cef) --- custom_schemas/custom_endpoint.yml | 60 ++++++++++ .../data_stream/metrics/fields/fields.yml | 62 ++++++++++ package/endpoint/docs/README.md | 12 ++ schemas/v1/metrics/metrics.yaml | 110 ++++++++++++++++++ 4 files changed, 244 insertions(+) diff --git a/custom_schemas/custom_endpoint.yml b/custom_schemas/custom_endpoint.yml index afd61bdf7..30ab95f33 100644 --- a/custom_schemas/custom_endpoint.yml +++ b/custom_schemas/custom_endpoint.yml @@ -570,6 +570,36 @@ type: long description: Total size of suppressed documents + - name: metrics.documents_volume.process_events.sources + level: custom + type: object + description: An array of Process Event document statistics per source + + - name: metrics.documents_volume.process_events.sources.source + level: custom + type: keyword + description: Process Event document source name + + - name: metrics.documents_volume.process_events.sources.sent_count + level: custom + type: long + description: Number of sent Process Event documents from source + + - name: metrics.documents_volume.process_events.sources.sent_bytes + level: custom + type: long + description: Total size of Process Event sent documents from source + + - name: metrics.documents_volume.process_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Process Event documents from source + + - name: metrics.documents_volume.process_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Process Event documents from source + - name: metrics.documents_volume.library_events.sent_count level: custom type: long @@ -610,6 +640,36 @@ type: long description: Total size of suppressed documents + - name: metrics.documents_volume.security_events.sources + level: custom + type: object + description: An array of Security Event document statistics per source + + - name: metrics.documents_volume.security_events.sources.source + level: custom + type: keyword + description: Security Event document source name + + - name: metrics.documents_volume.security_events.sources.sent_count + level: custom + type: long + description: Number of sent Security Event documents from source + + - name: metrics.documents_volume.security_events.sources.sent_bytes + level: custom + type: long + description: Total size of Security Event sent documents from source + + - name: metrics.documents_volume.security_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Security Event documents from source + + - name: metrics.documents_volume.security_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Security Event documents from source + - name: metrics.documents_volume.dns_events.sent_count level: custom type: long diff --git a/package/endpoint/data_stream/metrics/fields/fields.yml b/package/endpoint/data_stream/metrics/fields/fields.yml index 0d4203dd9..cae8747c8 100644 --- a/package/endpoint/data_stream/metrics/fields/fields.yml +++ b/package/endpoint/data_stream/metrics/fields/fields.yml @@ -303,6 +303,37 @@ type: long description: Number of sent documents default_field: false + - name: metrics.documents_volume.process_events.sources + level: custom + type: object + description: An array of Process Event document statistics per source + default_field: false + - name: metrics.documents_volume.process_events.sources.sent_bytes + level: custom + type: long + description: Total size of Process Event sent documents from source + default_field: false + - name: metrics.documents_volume.process_events.sources.sent_count + level: custom + type: long + description: Number of sent Process Event documents from source + default_field: false + - name: metrics.documents_volume.process_events.sources.source + level: custom + type: keyword + ignore_above: 1024 + description: Process Event document source name + default_field: false + - name: metrics.documents_volume.process_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Process Event documents from source + default_field: false + - name: metrics.documents_volume.process_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Process Event documents from source + default_field: false - name: metrics.documents_volume.process_events.suppressed_bytes level: custom type: long @@ -343,6 +374,37 @@ type: long description: Number of sent documents default_field: false + - name: metrics.documents_volume.security_events.sources + level: custom + type: object + description: An array of Security Event document statistics per source + default_field: false + - name: metrics.documents_volume.security_events.sources.sent_bytes + level: custom + type: long + description: Total size of Security Event sent documents from source + default_field: false + - name: metrics.documents_volume.security_events.sources.sent_count + level: custom + type: long + description: Number of sent Security Event documents from source + default_field: false + - name: metrics.documents_volume.security_events.sources.source + level: custom + type: keyword + ignore_above: 1024 + description: Security Event document source name + default_field: false + - name: metrics.documents_volume.security_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Security Event documents from source + default_field: false + - name: metrics.documents_volume.security_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Security Event documents from source + default_field: false - name: metrics.documents_volume.security_events.suppressed_bytes level: custom type: long diff --git a/package/endpoint/docs/README.md b/package/endpoint/docs/README.md index a66bcea06..ec12287b1 100644 --- a/package/endpoint/docs/README.md +++ b/package/endpoint/docs/README.md @@ -3059,6 +3059,12 @@ Metrics documents contain performance information about the endpoint executable | Endpoint.metrics.documents_volume.overall.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.documents_volume.process_events.sent_bytes | Total size of sent documents | long | | Endpoint.metrics.documents_volume.process_events.sent_count | Number of sent documents | long | +| Endpoint.metrics.documents_volume.process_events.sources | An array of Process Event document statistics per source | object | +| Endpoint.metrics.documents_volume.process_events.sources.sent_bytes | Total size of Process Event sent documents from source | long | +| Endpoint.metrics.documents_volume.process_events.sources.sent_count | Number of sent Process Event documents from source | long | +| Endpoint.metrics.documents_volume.process_events.sources.source | Process Event document source name | keyword | +| Endpoint.metrics.documents_volume.process_events.sources.suppressed_bytes | Total size of suppressed Process Event documents from source | long | +| Endpoint.metrics.documents_volume.process_events.sources.suppressed_count | Number of suppressed Process Event documents from source | long | | Endpoint.metrics.documents_volume.process_events.suppressed_bytes | Total size of suppressed documents | long | | Endpoint.metrics.documents_volume.process_events.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.documents_volume.registry_events.sent_bytes | Total size of sent documents | long | @@ -3067,6 +3073,12 @@ Metrics documents contain performance information about the endpoint executable | Endpoint.metrics.documents_volume.registry_events.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.documents_volume.security_events.sent_bytes | Total size of sent documents | long | | Endpoint.metrics.documents_volume.security_events.sent_count | Number of sent documents | long | +| Endpoint.metrics.documents_volume.security_events.sources | An array of Security Event document statistics per source | object | +| Endpoint.metrics.documents_volume.security_events.sources.sent_bytes | Total size of Security Event sent documents from source | long | +| Endpoint.metrics.documents_volume.security_events.sources.sent_count | Number of sent Security Event documents from source | long | +| Endpoint.metrics.documents_volume.security_events.sources.source | Security Event document source name | keyword | +| Endpoint.metrics.documents_volume.security_events.sources.suppressed_bytes | Total size of suppressed Security Event documents from source | long | +| Endpoint.metrics.documents_volume.security_events.sources.suppressed_count | Number of suppressed Security Event documents from source | long | | Endpoint.metrics.documents_volume.security_events.suppressed_bytes | Total size of suppressed documents | long | | Endpoint.metrics.documents_volume.security_events.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.event_filter.active_global_count | The number of active global event filters | long | diff --git a/schemas/v1/metrics/metrics.yaml b/schemas/v1/metrics/metrics.yaml index 949cbbf46..1cf63acbb 100644 --- a/schemas/v1/metrics/metrics.yaml +++ b/schemas/v1/metrics/metrics.yaml @@ -513,6 +513,61 @@ Endpoint.metrics.documents_volume.process_events.sent_count: normalize: [] short: Number of sent documents type: long +Endpoint.metrics.documents_volume.process_events.sources: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources + description: An array of Process Event document statistics per source + flat_name: Endpoint.metrics.documents_volume.process_events.sources + level: custom + name: metrics.documents_volume.process_events.sources + normalize: [] + short: An array of Process Event document statistics per source + type: object +Endpoint.metrics.documents_volume.process_events.sources.sent_bytes: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-sent-bytes + description: Total size of Process Event sent documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.sent_bytes + level: custom + name: metrics.documents_volume.process_events.sources.sent_bytes + normalize: [] + short: Total size of Process Event sent documents from source + type: long +Endpoint.metrics.documents_volume.process_events.sources.sent_count: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-sent-count + description: Number of sent Process Event documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.sent_count + level: custom + name: metrics.documents_volume.process_events.sources.sent_count + normalize: [] + short: Number of sent Process Event documents from source + type: long +Endpoint.metrics.documents_volume.process_events.sources.source: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-source + description: Process Event document source name + flat_name: Endpoint.metrics.documents_volume.process_events.sources.source + ignore_above: 1024 + level: custom + name: metrics.documents_volume.process_events.sources.source + normalize: [] + short: Process Event document source name + type: keyword +Endpoint.metrics.documents_volume.process_events.sources.suppressed_bytes: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-suppressed-bytes + description: Total size of suppressed Process Event documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.suppressed_bytes + level: custom + name: metrics.documents_volume.process_events.sources.suppressed_bytes + normalize: [] + short: Total size of suppressed Process Event documents from source + type: long +Endpoint.metrics.documents_volume.process_events.sources.suppressed_count: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-suppressed-count + description: Number of suppressed Process Event documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.suppressed_count + level: custom + name: metrics.documents_volume.process_events.sources.suppressed_count + normalize: [] + short: Number of suppressed Process Event documents from source + type: long Endpoint.metrics.documents_volume.process_events.suppressed_bytes: dashed_name: Endpoint-metrics-documents-volume-process-events-suppressed-bytes description: Total size of suppressed documents @@ -585,6 +640,61 @@ Endpoint.metrics.documents_volume.security_events.sent_count: normalize: [] short: Number of sent documents type: long +Endpoint.metrics.documents_volume.security_events.sources: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources + description: An array of Security Event document statistics per source + flat_name: Endpoint.metrics.documents_volume.security_events.sources + level: custom + name: metrics.documents_volume.security_events.sources + normalize: [] + short: An array of Security Event document statistics per source + type: object +Endpoint.metrics.documents_volume.security_events.sources.sent_bytes: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-sent-bytes + description: Total size of Security Event sent documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.sent_bytes + level: custom + name: metrics.documents_volume.security_events.sources.sent_bytes + normalize: [] + short: Total size of Security Event sent documents from source + type: long +Endpoint.metrics.documents_volume.security_events.sources.sent_count: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-sent-count + description: Number of sent Security Event documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.sent_count + level: custom + name: metrics.documents_volume.security_events.sources.sent_count + normalize: [] + short: Number of sent Security Event documents from source + type: long +Endpoint.metrics.documents_volume.security_events.sources.source: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-source + description: Security Event document source name + flat_name: Endpoint.metrics.documents_volume.security_events.sources.source + ignore_above: 1024 + level: custom + name: metrics.documents_volume.security_events.sources.source + normalize: [] + short: Security Event document source name + type: keyword +Endpoint.metrics.documents_volume.security_events.sources.suppressed_bytes: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-suppressed-bytes + description: Total size of suppressed Security Event documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.suppressed_bytes + level: custom + name: metrics.documents_volume.security_events.sources.suppressed_bytes + normalize: [] + short: Total size of suppressed Security Event documents from source + type: long +Endpoint.metrics.documents_volume.security_events.sources.suppressed_count: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-suppressed-count + description: Number of suppressed Security Event documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.suppressed_count + level: custom + name: metrics.documents_volume.security_events.sources.suppressed_count + normalize: [] + short: Number of suppressed Security Event documents from source + type: long Endpoint.metrics.documents_volume.security_events.suppressed_bytes: dashed_name: Endpoint-metrics-documents-volume-security-events-suppressed-bytes description: Total size of suppressed documents From 50bc156e7212e2f5512ab9333d1b96e20ed0d43b Mon Sep 17 00:00:00 2001 From: Nicholas Berlin Date: Fri, 10 Jul 2026 13:51:20 -0400 Subject: [PATCH 5/5] Add sources arrays to metrics sample_event.json Requested in review: sample values for the new documents_volume.{process,security}_events.sources.* mappings so the automated mapping tests exercise them. Source values reflect what the endpoint emits: process event sources are opcode names (load_module), security event sources are diagnostic_ strings. (cherry picked from commit 52535bc916bcf0c3423b060c998e7aebf55f7c6d) --- .../data_stream/metrics/sample_event.json | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/package/endpoint/data_stream/metrics/sample_event.json b/package/endpoint/data_stream/metrics/sample_event.json index 83513f790..fe05b16a2 100644 --- a/package/endpoint/data_stream/metrics/sample_event.json +++ b/package/endpoint/data_stream/metrics/sample_event.json @@ -658,7 +658,16 @@ "suppressed_count": 0, "suppressed_bytes": 0, "sent_count": 259, - "sent_bytes": 422247 + "sent_bytes": 422247, + "sources": [ + { + "source": "diagnostic_4624", + "sent_count": 4, + "sent_bytes": 10240, + "suppressed_count": 0, + "suppressed_bytes": 0 + } + ] }, "library_events": { "suppressed_count": 0, @@ -670,7 +679,16 @@ "suppressed_count": 0, "suppressed_bytes": 0, "sent_count": 201, - "sent_bytes": 602914 + "sent_bytes": 602914, + "sources": [ + { + "source": "load_module", + "sent_count": 2, + "sent_bytes": 5000, + "suppressed_count": 1, + "suppressed_bytes": 2500 + } + ] }, "registry_events": { "suppressed_count": 0,