From f5de46559e04c55b69f2e9619c3027467e9fc1d4 Mon Sep 17 00:00:00 2001 From: Nicholas Berlin Date: Fri, 10 Jul 2026 10:33:45 -0400 Subject: [PATCH 1/2] Add mappings for process and security event sources metrics Endpoint.metrics.documents_volume.{process,security}_events.sources.* were added to custom_documentation only (#652, #653), so the fields were never mapped in the package. The metrics index template sets dynamic: false, so these fields are currently not indexed on any stack. Define them in custom_schemas/custom_endpoint.yml mirroring the existing api_events.sources.* mappings, and regenerate with make. (cherry picked from commit 322d76a96c4f5a887425da3df7b3487f03199cef) --- custom_schemas/custom_endpoint.yml | 60 ++++++++++ .../data_stream/metrics/fields/fields.yml | 62 ++++++++++ package/endpoint/docs/README.md | 12 ++ schemas/v1/metrics/metrics.yaml | 110 ++++++++++++++++++ 4 files changed, 244 insertions(+) diff --git a/custom_schemas/custom_endpoint.yml b/custom_schemas/custom_endpoint.yml index f6ddb2d34..228e873b6 100644 --- a/custom_schemas/custom_endpoint.yml +++ b/custom_schemas/custom_endpoint.yml @@ -570,6 +570,36 @@ type: long description: Total size of suppressed documents + - name: metrics.documents_volume.process_events.sources + level: custom + type: object + description: An array of Process Event document statistics per source + + - name: metrics.documents_volume.process_events.sources.source + level: custom + type: keyword + description: Process Event document source name + + - name: metrics.documents_volume.process_events.sources.sent_count + level: custom + type: long + description: Number of sent Process Event documents from source + + - name: metrics.documents_volume.process_events.sources.sent_bytes + level: custom + type: long + description: Total size of Process Event sent documents from source + + - name: metrics.documents_volume.process_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Process Event documents from source + + - name: metrics.documents_volume.process_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Process Event documents from source + - name: metrics.documents_volume.library_events.sent_count level: custom type: long @@ -610,6 +640,36 @@ type: long description: Total size of suppressed documents + - name: metrics.documents_volume.security_events.sources + level: custom + type: object + description: An array of Security Event document statistics per source + + - name: metrics.documents_volume.security_events.sources.source + level: custom + type: keyword + description: Security Event document source name + + - name: metrics.documents_volume.security_events.sources.sent_count + level: custom + type: long + description: Number of sent Security Event documents from source + + - name: metrics.documents_volume.security_events.sources.sent_bytes + level: custom + type: long + description: Total size of Security Event sent documents from source + + - name: metrics.documents_volume.security_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Security Event documents from source + + - name: metrics.documents_volume.security_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Security Event documents from source + - name: metrics.documents_volume.dns_events.sent_count level: custom type: long diff --git a/package/endpoint/data_stream/metrics/fields/fields.yml b/package/endpoint/data_stream/metrics/fields/fields.yml index 2f99185f4..a1d6ed426 100644 --- a/package/endpoint/data_stream/metrics/fields/fields.yml +++ b/package/endpoint/data_stream/metrics/fields/fields.yml @@ -303,6 +303,37 @@ type: long description: Number of sent documents default_field: false + - name: metrics.documents_volume.process_events.sources + level: custom + type: object + description: An array of Process Event document statistics per source + default_field: false + - name: metrics.documents_volume.process_events.sources.sent_bytes + level: custom + type: long + description: Total size of Process Event sent documents from source + default_field: false + - name: metrics.documents_volume.process_events.sources.sent_count + level: custom + type: long + description: Number of sent Process Event documents from source + default_field: false + - name: metrics.documents_volume.process_events.sources.source + level: custom + type: keyword + ignore_above: 1024 + description: Process Event document source name + default_field: false + - name: metrics.documents_volume.process_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Process Event documents from source + default_field: false + - name: metrics.documents_volume.process_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Process Event documents from source + default_field: false - name: metrics.documents_volume.process_events.suppressed_bytes level: custom type: long @@ -343,6 +374,37 @@ type: long description: Number of sent documents default_field: false + - name: metrics.documents_volume.security_events.sources + level: custom + type: object + description: An array of Security Event document statistics per source + default_field: false + - name: metrics.documents_volume.security_events.sources.sent_bytes + level: custom + type: long + description: Total size of Security Event sent documents from source + default_field: false + - name: metrics.documents_volume.security_events.sources.sent_count + level: custom + type: long + description: Number of sent Security Event documents from source + default_field: false + - name: metrics.documents_volume.security_events.sources.source + level: custom + type: keyword + ignore_above: 1024 + description: Security Event document source name + default_field: false + - name: metrics.documents_volume.security_events.sources.suppressed_bytes + level: custom + type: long + description: Total size of suppressed Security Event documents from source + default_field: false + - name: metrics.documents_volume.security_events.sources.suppressed_count + level: custom + type: long + description: Number of suppressed Security Event documents from source + default_field: false - name: metrics.documents_volume.security_events.suppressed_bytes level: custom type: long diff --git a/package/endpoint/docs/README.md b/package/endpoint/docs/README.md index aa2cb20b8..dcdab4d01 100644 --- a/package/endpoint/docs/README.md +++ b/package/endpoint/docs/README.md @@ -3214,6 +3214,12 @@ Metrics documents contain performance information about the endpoint executable | Endpoint.metrics.documents_volume.overall.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.documents_volume.process_events.sent_bytes | Total size of sent documents | long | | Endpoint.metrics.documents_volume.process_events.sent_count | Number of sent documents | long | +| Endpoint.metrics.documents_volume.process_events.sources | An array of Process Event document statistics per source | object | +| Endpoint.metrics.documents_volume.process_events.sources.sent_bytes | Total size of Process Event sent documents from source | long | +| Endpoint.metrics.documents_volume.process_events.sources.sent_count | Number of sent Process Event documents from source | long | +| Endpoint.metrics.documents_volume.process_events.sources.source | Process Event document source name | keyword | +| Endpoint.metrics.documents_volume.process_events.sources.suppressed_bytes | Total size of suppressed Process Event documents from source | long | +| Endpoint.metrics.documents_volume.process_events.sources.suppressed_count | Number of suppressed Process Event documents from source | long | | Endpoint.metrics.documents_volume.process_events.suppressed_bytes | Total size of suppressed documents | long | | Endpoint.metrics.documents_volume.process_events.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.documents_volume.registry_events.sent_bytes | Total size of sent documents | long | @@ -3222,6 +3228,12 @@ Metrics documents contain performance information about the endpoint executable | Endpoint.metrics.documents_volume.registry_events.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.documents_volume.security_events.sent_bytes | Total size of sent documents | long | | Endpoint.metrics.documents_volume.security_events.sent_count | Number of sent documents | long | +| Endpoint.metrics.documents_volume.security_events.sources | An array of Security Event document statistics per source | object | +| Endpoint.metrics.documents_volume.security_events.sources.sent_bytes | Total size of Security Event sent documents from source | long | +| Endpoint.metrics.documents_volume.security_events.sources.sent_count | Number of sent Security Event documents from source | long | +| Endpoint.metrics.documents_volume.security_events.sources.source | Security Event document source name | keyword | +| Endpoint.metrics.documents_volume.security_events.sources.suppressed_bytes | Total size of suppressed Security Event documents from source | long | +| Endpoint.metrics.documents_volume.security_events.sources.suppressed_count | Number of suppressed Security Event documents from source | long | | Endpoint.metrics.documents_volume.security_events.suppressed_bytes | Total size of suppressed documents | long | | Endpoint.metrics.documents_volume.security_events.suppressed_count | Number of suppressed documents | long | | Endpoint.metrics.event_filter.active_global_count | The number of active global event filters | long | diff --git a/schemas/v1/metrics/metrics.yaml b/schemas/v1/metrics/metrics.yaml index a6ec90c2e..020d4a868 100644 --- a/schemas/v1/metrics/metrics.yaml +++ b/schemas/v1/metrics/metrics.yaml @@ -513,6 +513,61 @@ Endpoint.metrics.documents_volume.process_events.sent_count: normalize: [] short: Number of sent documents type: long +Endpoint.metrics.documents_volume.process_events.sources: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources + description: An array of Process Event document statistics per source + flat_name: Endpoint.metrics.documents_volume.process_events.sources + level: custom + name: metrics.documents_volume.process_events.sources + normalize: [] + short: An array of Process Event document statistics per source + type: object +Endpoint.metrics.documents_volume.process_events.sources.sent_bytes: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-sent-bytes + description: Total size of Process Event sent documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.sent_bytes + level: custom + name: metrics.documents_volume.process_events.sources.sent_bytes + normalize: [] + short: Total size of Process Event sent documents from source + type: long +Endpoint.metrics.documents_volume.process_events.sources.sent_count: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-sent-count + description: Number of sent Process Event documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.sent_count + level: custom + name: metrics.documents_volume.process_events.sources.sent_count + normalize: [] + short: Number of sent Process Event documents from source + type: long +Endpoint.metrics.documents_volume.process_events.sources.source: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-source + description: Process Event document source name + flat_name: Endpoint.metrics.documents_volume.process_events.sources.source + ignore_above: 1024 + level: custom + name: metrics.documents_volume.process_events.sources.source + normalize: [] + short: Process Event document source name + type: keyword +Endpoint.metrics.documents_volume.process_events.sources.suppressed_bytes: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-suppressed-bytes + description: Total size of suppressed Process Event documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.suppressed_bytes + level: custom + name: metrics.documents_volume.process_events.sources.suppressed_bytes + normalize: [] + short: Total size of suppressed Process Event documents from source + type: long +Endpoint.metrics.documents_volume.process_events.sources.suppressed_count: + dashed_name: Endpoint-metrics-documents-volume-process-events-sources-suppressed-count + description: Number of suppressed Process Event documents from source + flat_name: Endpoint.metrics.documents_volume.process_events.sources.suppressed_count + level: custom + name: metrics.documents_volume.process_events.sources.suppressed_count + normalize: [] + short: Number of suppressed Process Event documents from source + type: long Endpoint.metrics.documents_volume.process_events.suppressed_bytes: dashed_name: Endpoint-metrics-documents-volume-process-events-suppressed-bytes description: Total size of suppressed documents @@ -585,6 +640,61 @@ Endpoint.metrics.documents_volume.security_events.sent_count: normalize: [] short: Number of sent documents type: long +Endpoint.metrics.documents_volume.security_events.sources: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources + description: An array of Security Event document statistics per source + flat_name: Endpoint.metrics.documents_volume.security_events.sources + level: custom + name: metrics.documents_volume.security_events.sources + normalize: [] + short: An array of Security Event document statistics per source + type: object +Endpoint.metrics.documents_volume.security_events.sources.sent_bytes: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-sent-bytes + description: Total size of Security Event sent documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.sent_bytes + level: custom + name: metrics.documents_volume.security_events.sources.sent_bytes + normalize: [] + short: Total size of Security Event sent documents from source + type: long +Endpoint.metrics.documents_volume.security_events.sources.sent_count: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-sent-count + description: Number of sent Security Event documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.sent_count + level: custom + name: metrics.documents_volume.security_events.sources.sent_count + normalize: [] + short: Number of sent Security Event documents from source + type: long +Endpoint.metrics.documents_volume.security_events.sources.source: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-source + description: Security Event document source name + flat_name: Endpoint.metrics.documents_volume.security_events.sources.source + ignore_above: 1024 + level: custom + name: metrics.documents_volume.security_events.sources.source + normalize: [] + short: Security Event document source name + type: keyword +Endpoint.metrics.documents_volume.security_events.sources.suppressed_bytes: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-suppressed-bytes + description: Total size of suppressed Security Event documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.suppressed_bytes + level: custom + name: metrics.documents_volume.security_events.sources.suppressed_bytes + normalize: [] + short: Total size of suppressed Security Event documents from source + type: long +Endpoint.metrics.documents_volume.security_events.sources.suppressed_count: + dashed_name: Endpoint-metrics-documents-volume-security-events-sources-suppressed-count + description: Number of suppressed Security Event documents from source + flat_name: Endpoint.metrics.documents_volume.security_events.sources.suppressed_count + level: custom + name: metrics.documents_volume.security_events.sources.suppressed_count + normalize: [] + short: Number of suppressed Security Event documents from source + type: long Endpoint.metrics.documents_volume.security_events.suppressed_bytes: dashed_name: Endpoint-metrics-documents-volume-security-events-suppressed-bytes description: Total size of suppressed documents From 659b07fda80846eaf7c4c23b55f6507a9d3e1ae7 Mon Sep 17 00:00:00 2001 From: Nicholas Berlin Date: Fri, 10 Jul 2026 13:51:20 -0400 Subject: [PATCH 2/2] Add sources arrays to metrics sample_event.json Requested in review: sample values for the new documents_volume.{process,security}_events.sources.* mappings so the automated mapping tests exercise them. Source values reflect what the endpoint emits: process event sources are opcode names (load_module), security event sources are diagnostic_ strings. (cherry picked from commit 52535bc916bcf0c3423b060c998e7aebf55f7c6d) --- .../data_stream/metrics/sample_event.json | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/package/endpoint/data_stream/metrics/sample_event.json b/package/endpoint/data_stream/metrics/sample_event.json index a7ca50b13..877e0eb82 100644 --- a/package/endpoint/data_stream/metrics/sample_event.json +++ b/package/endpoint/data_stream/metrics/sample_event.json @@ -664,7 +664,16 @@ "suppressed_count": 0, "suppressed_bytes": 0, "sent_count": 259, - "sent_bytes": 422247 + "sent_bytes": 422247, + "sources": [ + { + "source": "diagnostic_4624", + "sent_count": 4, + "sent_bytes": 10240, + "suppressed_count": 0, + "suppressed_bytes": 0 + } + ] }, "library_events": { "suppressed_count": 0, @@ -676,7 +685,16 @@ "suppressed_count": 0, "suppressed_bytes": 0, "sent_count": 201, - "sent_bytes": 602914 + "sent_bytes": 602914, + "sources": [ + { + "source": "load_module", + "sent_count": 2, + "sent_bytes": 5000, + "suppressed_count": 1, + "suppressed_bytes": 2500 + } + ] }, "registry_events": { "suppressed_count": 0,