From a2a2f5599c89e3a5ae5b39243cd9e4ad0f091675 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:01:08 +0530 Subject: [PATCH 01/14] update ECS version to 9.3.0 and format_version to 3.3.2 --- .../cloudflare_logpush/_dev/build/build.yml | 2 +- .../access_request/fields/agent.yml | 39 - .../access_request/fields/base-fields.yml | 22 +- .../access_request/fields/beats.yml | 6 + .../data_stream/audit/fields/agent.yml | 39 - .../data_stream/audit/fields/base-fields.yml | 22 +- .../data_stream/audit/fields/beats.yml | 6 + .../data_stream/casb/fields/agent.yml | 39 - .../data_stream/casb/fields/base-fields.yml | 22 +- .../data_stream/casb/fields/beats.yml | 6 + .../device_posture/fields/agent.yml | 39 - .../device_posture/fields/base-fields.yml | 22 +- .../device_posture/fields/beats.yml | 6 + .../dlp_forensic_copies/fields/agent.yml | 39 - .../fields/base-fields.yml | 22 +- .../dlp_forensic_copies/fields/beats.yml | 6 + .../data_stream/dns/fields/agent.yml | 39 - .../data_stream/dns/fields/base-fields.yml | 22 +- .../data_stream/dns/fields/beats.yml | 6 + .../data_stream/dns_firewall/fields/agent.yml | 39 - .../dns_firewall/fields/base-fields.yml | 22 +- .../data_stream/dns_firewall/fields/beats.yml | 6 + .../email_security_alerts/fields/agent.yml | 39 - .../fields/base-fields.yml | 22 +- .../email_security_alerts/fields/beats.yml | 6 + .../firewall_event/fields/agent.yml | 39 - .../firewall_event/fields/base-fields.yml | 22 +- .../firewall_event/fields/beats.yml | 6 + .../data_stream/gateway_dns/fields/agent.yml | 39 - .../gateway_dns/fields/base-fields.yml | 22 +- .../data_stream/gateway_dns/fields/beats.yml | 6 + .../data_stream/gateway_http/fields/agent.yml | 39 - .../gateway_http/fields/base-fields.yml | 22 +- .../data_stream/gateway_http/fields/beats.yml | 6 + .../gateway_network/fields/agent.yml | 39 - .../gateway_network/fields/base-fields.yml | 22 +- .../gateway_network/fields/beats.yml | 6 + .../data_stream/http_request/fields/agent.yml | 39 - .../http_request/fields/base-fields.yml | 22 +- .../data_stream/http_request/fields/beats.yml | 6 + .../data_stream/magic_ids/fields/agent.yml | 39 - .../magic_ids/fields/base-fields.yml | 22 +- .../data_stream/magic_ids/fields/beats.yml | 6 + .../data_stream/nel_report/fields/agent.yml | 39 - .../nel_report/fields/base-fields.yml | 22 +- .../data_stream/nel_report/fields/beats.yml | 6 + .../network_analytics/fields/agent.yml | 39 - .../network_analytics/fields/base-fields.yml | 22 +- .../network_analytics/fields/beats.yml | 6 + .../network_session/fields/agent.yml | 39 - .../network_session/fields/base-fields.yml | 22 +- .../network_session/fields/beats.yml | 6 + .../page_shield_events/fields/agent.yml | 39 - .../page_shield_events/fields/base-fields.yml | 22 +- .../page_shield_events/fields/beats.yml | 6 + .../sinkhole_http/fields/agent.yml | 39 - .../sinkhole_http/fields/base-fields.yml | 22 +- .../sinkhole_http/fields/beats.yml | 6 + .../spectrum_event/fields/agent.yml | 39 - .../spectrum_event/fields/base-fields.yml | 22 +- .../spectrum_event/fields/beats.yml | 6 + .../workers_trace/fields/agent.yml | 39 - .../workers_trace/fields/base-fields.yml | 22 +- .../workers_trace/fields/beats.yml | 6 + packages/cloudflare_logpush/docs/README.md | 1665 ++++++++--------- packages/cloudflare_logpush/manifest.yml | 4 +- 66 files changed, 1140 insertions(+), 1938 deletions(-) delete mode 100644 packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/audit/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/casb/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/dns/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml delete mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml diff --git a/packages/cloudflare_logpush/_dev/build/build.yml b/packages/cloudflare_logpush/_dev/build/build.yml index 2bfcfc223b0..0435c2b2300 100644 --- a/packages/cloudflare_logpush/_dev/build/build.yml +++ b/packages/cloudflare_logpush/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: "git@v8.11.0" + reference: 'git@v9.3.0' diff --git a/packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml b/packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/access_request/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/access_request/fields/base-fields.yml index bdfe12e2287..9be04129d86 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/access_request/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.access_request -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/access_request/fields/beats.yml b/packages/cloudflare_logpush/data_stream/access_request/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/access_request/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/audit/fields/agent.yml b/packages/cloudflare_logpush/data_stream/audit/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/audit/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/audit/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/audit/fields/base-fields.yml index d59dd058877..58a378335f4 100644 --- a/packages/cloudflare_logpush/data_stream/audit/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/audit/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.audit -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/audit/fields/beats.yml b/packages/cloudflare_logpush/data_stream/audit/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/audit/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/audit/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/casb/fields/agent.yml b/packages/cloudflare_logpush/data_stream/casb/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/casb/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/casb/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/casb/fields/base-fields.yml index 06d3deff01d..25ec719369b 100644 --- a/packages/cloudflare_logpush/data_stream/casb/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/casb/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.casb -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/casb/fields/beats.yml b/packages/cloudflare_logpush/data_stream/casb/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/casb/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/casb/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml b/packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/device_posture/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/device_posture/fields/base-fields.yml index f080a805ae0..b4cdab9ef0a 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/device_posture/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.device_posture -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/device_posture/fields/beats.yml b/packages/cloudflare_logpush/data_stream/device_posture/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/device_posture/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/agent.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/base-fields.yml index 0352e83f179..657c27fced4 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.dlp_forensic_copies -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/beats.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/dns/fields/agent.yml b/packages/cloudflare_logpush/data_stream/dns/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/dns/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/dns/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/dns/fields/base-fields.yml index 7cd21a55f79..03fd5ef9891 100644 --- a/packages/cloudflare_logpush/data_stream/dns/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/dns/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.dns -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/dns/fields/beats.yml b/packages/cloudflare_logpush/data_stream/dns/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/dns/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/dns/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/base-fields.yml index e2b28b23c46..37720c57a7d 100644 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.dns_firewall -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/beats.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/agent.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/base-fields.yml index 3dfbaf7efeb..7ba31f164a2 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.email_security_alerts -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/beats.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/base-fields.yml index 958bad1989e..2993499140f 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.firewall_event -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/beats.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/base-fields.yml index 1ff2e065e51..e14c61755cf 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.gateway_dns -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/beats.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml b/packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/gateway_http/fields/base-fields.yml index db66c18a119..fc8053d692c 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.gateway_http -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/fields/beats.yml b/packages/cloudflare_logpush/data_stream/gateway_http/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml b/packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/gateway_network/fields/base-fields.yml index 29627fa4c0c..8669e4d71ba 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_network/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.gateway_network -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/fields/beats.yml b/packages/cloudflare_logpush/data_stream/gateway_network/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_network/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml b/packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/http_request/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/http_request/fields/base-fields.yml index 6f4d0762cae..2e14006104a 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.http_request -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/http_request/fields/beats.yml b/packages/cloudflare_logpush/data_stream/http_request/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml b/packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/magic_ids/fields/base-fields.yml index b1337fa77ec..d794123892f 100644 --- a/packages/cloudflare_logpush/data_stream/magic_ids/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/magic_ids/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.magic_ids -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/fields/beats.yml b/packages/cloudflare_logpush/data_stream/magic_ids/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/magic_ids/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/magic_ids/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml b/packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/nel_report/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/nel_report/fields/base-fields.yml index 90a63ff65c1..d8a1bebb060 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/nel_report/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.nel_report -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/nel_report/fields/beats.yml b/packages/cloudflare_logpush/data_stream/nel_report/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/nel_report/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml b/packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/network_analytics/fields/base-fields.yml index f25938756c8..f53e42d0436 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/network_analytics/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.network_analytics -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/fields/beats.yml b/packages/cloudflare_logpush/data_stream/network_analytics/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/network_analytics/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml b/packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/network_session/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/network_session/fields/base-fields.yml index 6d65cb54b28..52899501604 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/network_session/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.network_session -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/network_session/fields/beats.yml b/packages/cloudflare_logpush/data_stream/network_session/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/network_session/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/fields/agent.yml b/packages/cloudflare_logpush/data_stream/page_shield_events/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/page_shield_events/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/page_shield_events/fields/base-fields.yml index 2c82e7b3417..872a6dad9b0 100644 --- a/packages/cloudflare_logpush/data_stream/page_shield_events/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.page_shield_events -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/fields/beats.yml b/packages/cloudflare_logpush/data_stream/page_shield_events/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/page_shield_events/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/base-fields.yml index efce1ba57fe..79bc1b0980b 100644 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.sinkhole_http -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/beats.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/base-fields.yml index 135b81f388f..97433eff6be 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.spectrum_event -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/beats.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml b/packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml deleted file mode 100644 index 894e6f12be2..00000000000 --- a/packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/fields/base-fields.yml b/packages/cloudflare_logpush/data_stream/workers_trace/fields/base-fields.yml index 773a75b6497..f21585ba728 100644 --- a/packages/cloudflare_logpush/data_stream/workers_trace/fields/base-fields.yml +++ b/packages/cloudflare_logpush/data_stream/workers_trace/fields/base-fields.yml @@ -1,20 +1,16 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace + external: ecs +- name: event.module type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. + external: ecs + value: cloudflare_logpush - name: event.dataset type: constant_keyword - description: Event dataset. + external: ecs value: cloudflare_logpush.workers_trace -- name: event.module - type: constant_keyword - description: Event module. - value: cloudflare_logpush +- name: '@timestamp' + external: ecs diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/fields/beats.yml b/packages/cloudflare_logpush/data_stream/workers_trace/fields/beats.yml index 4bfc83d5cae..72efc55e081 100644 --- a/packages/cloudflare_logpush/data_stream/workers_trace/fields/beats.yml +++ b/packages/cloudflare_logpush/data_stream/workers_trace/fields/beats.yml @@ -43,3 +43,9 @@ - name: object.content_type type: keyword description: The content type of the Google Cloud Storage object. +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/cloudflare_logpush/docs/README.md b/packages/cloudflare_logpush/docs/README.md index 4e662c7af19..6622394d961 100644 --- a/packages/cloudflare_logpush/docs/README.md +++ b/packages/cloudflare_logpush/docs/README.md @@ -207,9 +207,9 @@ An example event for `access_request` looks as following: { "@timestamp": "2023-05-23T17:18:33.000Z", "agent": { - "ephemeral_id": "3c43110d-da0b-4e1b-adec-7031cdfb87a1", - "id": "57b2b3df-7f1f-49a9-8b35-90731f9b1b4e", - "name": "elastic-agent-50154", + "ephemeral_id": "a4b006f5-ea6d-4027-8aa3-2c81ff622f0d", + "id": "39bb77e3-2cdc-4e93-9302-147cd7c45658", + "name": "elastic-agent-84026", "type": "filebeat", "version": "8.17.1" }, @@ -228,15 +228,22 @@ An example event for `access_request` looks as following: }, "ip": "67.43.156.93" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "access_request": { + "action": "login", + "allowed": true, "app": { + "domain": "partner-zt-logs.cloudflareaccess.com/warp", "uuid": "123e4567-e89b-12d3-a456-426614174000" }, + "client": { + "ip": "67.43.156.93" + }, "connection": "onetimepin", + "country": "us", + "ray": { + "id": "00c0ffeeabc12345" + }, "request": { "prompt": "Please provide your reason for accessing the application.", "response": "I need to access the application for work purposes." @@ -247,19 +254,24 @@ An example event for `access_request` looks as following: "approver2@example.com" ], "duration": 7200 + }, + "timestamp": "2023-05-23T17:18:33.000Z", + "user": { + "email": "user@example.com", + "id": "166befbb-00e3-5e20-bd6e-27245333949f" } } }, "data_stream": { "dataset": "cloudflare_logpush.access_request", - "namespace": "33755", + "namespace": "88021", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "57b2b3df-7f1f-49a9-8b35-90731f9b1b4e", + "id": "39bb77e3-2cdc-4e93-9302-147cd7c45658", "snapshot": false, "version": "8.17.1" }, @@ -271,32 +283,16 @@ An example event for `access_request` looks as following: ], "dataset": "cloudflare_logpush.access_request", "id": "00c0ffeeabc12345", - "ingested": "2025-12-12T07:31:27Z", + "ingested": "2026-04-07T10:14:21Z", "kind": "event", + "original": "{\"Action\":\"login\",\"Allowed\":true,\"AppDomain\":\"partner-zt-logs.cloudflareaccess.com/warp\",\"AppUUID\":\"123e4567-e89b-12d3-a456-426614174000\",\"Connection\":\"onetimepin\",\"Country\":\"us\",\"CreatedAt\":1684862313000000000,\"Email\":\"user@example.com\",\"IPAddress\":\"67.43.156.93\",\"PurposeJustificationPrompt\":\"Please provide your reason for accessing the application.\",\"PurposeJustificationResponse\":\"I need to access the application for work purposes.\",\"RayID\":\"00c0ffeeabc12345\",\"TemporaryAccessApprovers\":[\"approver1@example.com\",\"approver2@example.com\"],\"TemporaryAccessDuration\":7200,\"UserUID\":\"166befbb-00e3-5e20-bd6e-27245333949f\"}", "type": [ "access", "allowed" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "access_request.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/access_request.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "ip": [ @@ -310,6 +306,8 @@ An example event for `access_request` looks as following: ] }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-access_request" ], @@ -327,14 +325,13 @@ An example event for `access_request` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.access_request.action | What type of record is this. login | logout. | keyword | | cloudflare_logpush.access_request.allowed | If request was allowed or denied. | boolean | | cloudflare_logpush.access_request.app.domain | The domain of the Application that Access is protecting. | keyword | @@ -350,18 +347,15 @@ An example event for `access_request` looks as following: | cloudflare_logpush.access_request.timestamp | The date and time the corresponding access request was made. | date | | cloudflare_logpush.access_request.user.email | Email of the user who logged in. | keyword | | cloudflare_logpush.access_request.user.id | The uid of the user who logged in. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -380,20 +374,26 @@ An example event for `audit` looks as following: { "@timestamp": "2021-11-30T20:19:48.000Z", "agent": { - "ephemeral_id": "f6aa0050-d066-4e9f-ad0e-44e29655cb0a", - "id": "c88b6a8a-5b51-4f19-9386-d141ad8d5fd7", - "name": "elastic-agent-42667", + "ephemeral_id": "a95e5d2e-311a-4aae-ae73-f9e157308c0e", + "id": "8992a634-8664-45e9-9581-cd467d2590e0", + "name": "elastic-agent-87538", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "audit": { + "action": { + "result": "success", + "type": "token_create" + }, "actor": { + "email": "user@example.com", + "id": "enl3j9du8rnx2swwd9l32qots7l54t9s", + "ip": "81.2.69.142", "type": "user" }, + "id": "73fd39ed-5aab-4a2a-b93c-c9a4abf0c425", + "interface": "UI", "metadata": { "token_name": "test", "token_tag": "b7261c49a793a82678d12285f0bc1401" @@ -412,19 +412,20 @@ An example event for `audit` looks as following: "resource": { "id": "enl3j9du8rnx2swwd9l32qots7l54t9s", "type": "account" - } + }, + "timestamp": "2021-11-30T20:19:48.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.audit", - "namespace": "27343", + "namespace": "77347", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "c88b6a8a-5b51-4f19-9386-d141ad8d5fd7", + "id": "8992a634-8664-45e9-9581-cd467d2590e0", "snapshot": false, "version": "8.17.1" }, @@ -436,46 +437,33 @@ An example event for `audit` looks as following: ], "dataset": "cloudflare_logpush.audit", "id": "73fd39ed-5aab-4a2a-b93c-c9a4abf0c425", - "ingested": "2025-12-12T07:37:27Z", + "ingested": "2026-04-07T10:15:20Z", "kind": "event", + "original": "{\"ActionResult\":true,\"ActionType\":\"token_create\",\"ActorEmail\":\"user@example.com\",\"ActorID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ActorIP\":\"81.2.69.142\",\"ActorType\":\"user\",\"ID\":\"73fd39ed-5aab-4a2a-b93c-c9a4abf0c425\",\"Interface\":\"UI\",\"Metadata\":{\"token_name\":\"test\",\"token_tag\":\"b7261c49a793a82678d12285f0bc1401\"},\"NewValue\":{\"key1\":\"value1\",\"key2\":\"value2\"},\"OldValue\":{\"key3\":\"value4\",\"key4\":\"value4\"},\"OwnerID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ResourceID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ResourceType\":\"account\",\"When\":\"2021-11-30T20:19:48Z\"}", "outcome": "success", "provider": "UI", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "audit.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/audit.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "ip": [ "81.2.69.142" ], "user": [ - "enl3j9du8rnx2swwd9l32qots7l54t9s" + "enl3j9du8rnx2swwd9l32qots7l54t9s", + "user@example.com" ] }, "source": { "ip": "81.2.69.142" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-audit" ], @@ -490,14 +478,13 @@ An example event for `audit` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.audit.action.result | Whether the action was successful. | keyword | | cloudflare_logpush.audit.action.type | Type of action taken. | keyword | | cloudflare_logpush.audit.actor.email | Email of the actor. | keyword | @@ -513,18 +500,15 @@ An example event for `audit` looks as following: | cloudflare_logpush.audit.resource.id | Unique identifier of the resource within Cloudflare system. | keyword | | cloudflare_logpush.audit.resource.type | The type of resource that was changed. | keyword | | cloudflare_logpush.audit.timestamp | When the change happened. | date | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -542,15 +526,12 @@ An example event for `casb` looks as following: { "@timestamp": "2023-05-16T10:00:00.000Z", "agent": { - "ephemeral_id": "941f6da3-676d-466a-817c-3d2b60cb0da8", - "id": "082d25a2-4d08-4619-9273-1bbf8ecda1ac", - "name": "elastic-agent-85424", + "ephemeral_id": "e6196296-ea3c-4d44-8154-c7c5bc5bd548", + "id": "17cbd457-fcca-44a8-9448-2fdec84f262b", + "name": "elastic-agent-21002", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "casb": { "asset": { @@ -604,31 +585,35 @@ An example event for `casb` looks as following: "url": "/services/data/userID" } }, - "name": "John Doe" + "name": "John Doe", + "url": "https://example.com/resource" }, "finding": { + "id": "6b187be4-2dd5-42c5-a37b-111111111111", "type": { "id": "a2790c4f-03f5-449f-b209-5f4447f417aa", - "name": "Salesforce User Sending Email with Different Email Address" + "name": "Salesforce User Sending Email with Different Email Address", + "severity": "Medium" } }, "integration": { "id": "c772678d-5cf1-4c73-bf3f-111111111111", "name": "Salesforce Testing", "policy_vendor": "Salesforce Connection" - } + }, + "timestamp": "2023-05-16T10:00:00.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.casb", - "namespace": "79671", + "namespace": "21142", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "082d25a2-4d08-4619-9273-1bbf8ecda1ac", + "id": "17cbd457-fcca-44a8-9448-2fdec84f262b", "snapshot": false, "version": "8.17.1" }, @@ -639,34 +624,20 @@ An example event for `casb` looks as following: ], "dataset": "cloudflare_logpush.casb", "id": "6b187be4-2dd5-42c5-a37b-111111111111", - "ingested": "2025-12-12T11:49:37Z", + "ingested": "2026-04-07T10:16:20Z", "kind": "event", + "original": "{\"AssetDisplayName\":\"John Doe\",\"AssetExternalID\":\"0051N000004mG2LAAA\",\"AssetLink\":\"https://example.com/resource\",\"AssetMetadata\":{\"AccountId\":null,\"Address\":{\"city\":\"Singapore\",\"country\":\"Singapore\",\"countryCode\":\"SG\",\"geocodeAccuracy\":null,\"latitude\":null,\"longitude\":null,\"postalCode\":null,\"state\":null,\"stateCode\":null,\"street\":null},\"Alias\":\"JDoe\",\"BadgeText\":\"\",\"BannerPhotoUrl\":\"/profilephoto/001\",\"CallCenterId\":null,\"CommunityNickname\":\"Doe.John\",\"CompanyName\":\"MyCompany\",\"ContactId\":null,\"DefaultGroupNotificationFrequency\":\"N\",\"Department\":\"521\",\"DigestFrequency\":\"D\",\"Division\":null,\"Email\":\"user@example.com\",\"EmailEncodingKey\":\"UTF-8\",\"EmailPreferencesAutoBcc\":true,\"EmployeeNumber\":\"18124\",\"Extension\":null,\"Fax\":null,\"FederationIdentifier\":null,\"FirstName\":\"John\",\"ForecastEnabled\":false,\"FullPhotoUrl\":\"https://photos.com/profilephoto/001\",\"Id\":\"0051N000004mG2LAAA\",\"IsActive\":false,\"IsProfilePhotoActive\":false,\"LanguageLocaleKey\":\"en_US\",\"LastLoginDate\":\"2021-10-06T06:32:09.000+0000\",\"LastName\":\"Doe\",\"LastReferencedDate\":null,\"LastViewedDate\":null,\"LocaleSidKey\":\"en_SG\",\"MediumBannerPhotoUrl\":\"/profilephoto/001/E\",\"MobilePhone\":null,\"Name\":\"John Doe\",\"OfflineTrialExpirationDate\":null,\"Phone\":\"+3460000000\",\"ReceivesAdminInfoEmails\":true,\"ReceivesInfoEmails\":true,\"SenderEmail\":\"sender@example.com\",\"SenderName\":null,\"Signature\":null,\"SmallBannerPhotoUrl\":\"/profilephoto/001/D\",\"SmallPhotoUrl\":\"https://photos.com/photo/001\",\"TimeZoneSidKey\":\"Asia/Singapore\",\"Title\":\"Customer Solutions Engineer\",\"UserPermissionsCallCenterAutoLogin\":false,\"UserPermissionsInteractionUser\":true,\"UserPermissionsMarketingUser\":false,\"UserPermissionsOfflineUser\":false,\"UserPermissionsSupportUser\":false,\"UserRoleId\":\"00E2G000001E\",\"UserType\":\"Standard\",\"attributes\":{\"type\":\"User\",\"url\":\"/services/data/userID\"}},\"DetectedTimestamp\":\"2023-05-16T10:00:00Z\",\"FindingTypeDisplayName\":\"Salesforce User Sending Email with Different Email Address\",\"FindingTypeID\":\"a2790c4f-03f5-449f-b209-5f4447f417aa\",\"FindingTypeSeverity\":\"Medium\",\"InstanceID\":\"6b187be4-2dd5-42c5-a37b-111111111111\",\"IntegrationDisplayName\":\"Salesforce Testing\",\"IntegrationID\":\"c772678d-5cf1-4c73-bf3f-111111111111\",\"IntegrationPolicyVendor\":\"Salesforce Connection\"}", "severity": 2, "type": [ "access" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "casb.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/casb.log" - }, - "offset": 0 + "type": "http_endpoint" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-casb" ], @@ -683,14 +654,13 @@ An example event for `casb` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.casb.asset.id | Unique identifier for an asset of this type. Format will vary by policy vendor. | keyword | | cloudflare_logpush.casb.asset.metadata | Metadata associated with the asset. Structure will vary by policy vendor. | flattened | | cloudflare_logpush.casb.asset.name | Asset display name. | keyword | @@ -703,18 +673,15 @@ An example event for `casb` looks as following: | cloudflare_logpush.casb.integration.name | Human-readable name of the integration. | keyword | | cloudflare_logpush.casb.integration.policy_vendor | Human-readable vendor name of the integration´s policy. | keyword | | cloudflare_logpush.casb.timestamp | Date and time the finding was first identified. | date | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -732,26 +699,12 @@ An example event for `device_posture` looks as following: { "@timestamp": "2023-05-17T12:00:00.000Z", "agent": { - "ephemeral_id": "edffd861-c70c-4011-9b8e-4bb37d718964", - "id": "c1433c6f-9bf4-4f42-b22e-10ef701c8969", - "name": "elastic-agent-95387", + "ephemeral_id": "697666b7-2c61-4b7e-8a91-8582b66162ab", + "id": "3452c3cf-0cc4-408d-a6b0-f55c7f306781", + "name": "elastic-agent-71222", "type": "filebeat", "version": "8.17.1" }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-device-posture-bucket-31257", - "name": "elastic-package-device-posture-bucket-31257" - }, - "object": { - "key": "test-device-posture.log" - } - } - }, - "cloud": { - "region": "us-east-1" - }, "cloudflare_logpush": { "device_posture": { "eval": { @@ -770,22 +723,48 @@ An example event for `device_posture` looks as following: "result": true }, "host": { + "id": "083a8354-d56c-11ed-9771-111111111", "manufacturer": "Google Compute Engine", "model": "Google Compute Engine", + "name": "zt-test-vm1", + "os": { + "family": "linux", + "version": "5.15.0" + }, "serial": "GoogleCloud-ABCD1234567890" - } + }, + "rule": { + "category": "os_version", + "id": "policy-abcdefgh", + "name": "Ubuntu" + }, + "timestamp": "2023-05-17T12:00:00.000Z", + "user": { + "email": "user@example.com", + "id": "user-abcdefgh" + }, + "version": "2023.3.258" } }, "data_stream": { "dataset": "cloudflare_logpush.device_posture", - "namespace": "33464", + "namespace": "60433", "type": "logs" }, + "device": { + "id": "083a8354-d56c-11ed-9771-111111111", + "manufacturer": "Google Compute Engine", + "model": { + "name": "Google Compute Engine" + }, + "serial_number": "GoogleCloud-ABCD1234567890", + "type": "linux" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "c1433c6f-9bf4-4f42-b22e-10ef701c8969", + "id": "3452c3cf-0cc4-408d-a6b0-f55c7f306781", "snapshot": false, "version": "8.17.1" }, @@ -795,9 +774,9 @@ An example event for `device_posture` looks as following: "host" ], "dataset": "cloudflare_logpush.device_posture", - "ingested": "2025-12-12T08:15:20Z", + "ingested": "2026-04-07T10:17:21Z", "kind": "event", - "original": "{\"ClientVersion\":\"2023.3.258\",\"DeviceID\":\"083a8354-d56c-11ed-9771-111111111\",\"DeviceManufacturer\":\"Google Compute Engine\",\"DeviceModel\":\"Google Compute Engine\",\"DeviceName\":\"zt-test-vm1\",\"DeviceSerialNumber\":\"GoogleCloud-ABCD1234567890\",\"DeviceType\":\"linux\",\"Email\":\"user@example.com\",\"OSVersion\":\"5.15.0\",\"PolicyID\":\"policy-abcdefgh\",\"PostureCheckName\":\"Ubuntu\",\"PostureCheckType\":\"os_version\",\"PostureEvaluatedResult\":true,\"PostureExpectedJSON\":{\"version\":\"5.15.0-1025-gcp\",\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\"},\"PostureReceivedJSON\":{\"version\":\"5.15.0-1025-gcp\",\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\"},\"Timestamp\":\"2023-05-17T12:00:00Z\",\"UserUID\":\"user-abcdefgh\"}", + "original": "{\"ClientVersion\":\"2023.3.258\",\"DeviceID\":\"083a8354-d56c-11ed-9771-111111111\",\"DeviceManufacturer\":\"Google Compute Engine\",\"DeviceModel\":\"Google Compute Engine\",\"DeviceName\":\"zt-test-vm1\",\"DeviceSerialNumber\":\"GoogleCloud-ABCD1234567890\",\"DeviceType\":\"linux\",\"Email\":\"user@example.com\",\"OSVersion\":\"5.15.0\",\"PolicyID\":\"policy-abcdefgh\",\"PostureCheckName\":\"Ubuntu\",\"PostureCheckType\":\"os_version\",\"PostureEvaluatedResult\":true,\"PostureExpectedJSON\":{\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\",\"version\":\"5.15.0-1025-gcp\"},\"PostureReceivedJSON\":{\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\",\"version\":\"5.15.0-1025-gcp\"},\"Timestamp\":\"2023-05-17T12:00:00Z\",\"UserUID\":\"user-abcdefgh\"}", "outcome": "success", "type": [ "info" @@ -812,13 +791,7 @@ An example event for `device_posture` looks as following: } }, "input": { - "type": "aws-s3" - }, - "log": { - "file": { - "path": "https://elastic-package-device-posture-bucket-31257.s3.us-east-1.amazonaws.com/test-device-posture.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hosts": [ @@ -836,8 +809,8 @@ An example event for `device_posture` looks as following: "name": "Ubuntu" }, "tags": [ - "collect_sqs_logs", "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-device_posture" ], @@ -855,14 +828,13 @@ An example event for `device_posture` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.device_posture.eval.expected | JSON object of what the posture check expects from the Zero Trust client. | flattened | | cloudflare_logpush.device_posture.eval.received | JSON object of what the Zero Trust client actually uploads. | flattened | | cloudflare_logpush.device_posture.eval.result | Whether this posture upload passes the associated posture check, given the requirements posture check at the time of the timestamp. | boolean | @@ -873,6 +845,7 @@ An example event for `device_posture` looks as following: | cloudflare_logpush.device_posture.host.os.family | The Zero Trust client operating system type. | keyword | | cloudflare_logpush.device_posture.host.os.version | The operating system version at the time of upload. | keyword | | cloudflare_logpush.device_posture.host.serial | The serial number of the device that the Zero Trust client is running on. | keyword | +| cloudflare_logpush.device_posture.registration_id | The UUID of the device registration associated with this posture result. | keyword | | cloudflare_logpush.device_posture.rule.category | The type of the Zero Trust client check or service provider check. | keyword | | cloudflare_logpush.device_posture.rule.id | The posture check ID associated with this device posture result. | keyword | | cloudflare_logpush.device_posture.rule.name | The name of the posture check associated with this device posture result. | keyword | @@ -880,18 +853,15 @@ An example event for `device_posture` looks as following: | cloudflare_logpush.device_posture.user.email | The email used to register the device with the Zero Trust client. | keyword | | cloudflare_logpush.device_posture.user.id | The uid of the user who registered the device. | keyword | | cloudflare_logpush.device_posture.version | The Zero Trust client version at the time of upload. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -909,26 +879,12 @@ An example event for `dlp_forensic_copies` looks as following: { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "ac5a4097-8089-4aaa-86b6-7a2807d34abf", - "id": "f5cd4a36-a09b-49fe-8b0c-0148d4e98428", - "name": "elastic-agent-81829", + "ephemeral_id": "5eca1e85-a7a9-4c98-920e-c688eddd8dfa", + "id": "c938d3c0-71e7-4d2d-9f16-6c3193069652", + "name": "elastic-agent-17032", "type": "filebeat", "version": "8.17.1" }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-dlp-forensic-copies-bucket-19014", - "name": "elastic-package-dlp-forensic-copies-bucket-19014" - }, - "object": { - "key": "test-dlp-forensic-copies.log" - } - } - }, - "cloud": { - "region": "us-east-1" - }, "cloudflare_logpush": { "dlp_forensic_copies": { "account_id": "acc-id", @@ -946,14 +902,14 @@ An example event for `dlp_forensic_copies` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.dlp_forensic_copies", - "namespace": "64194", + "namespace": "36828", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "f5cd4a36-a09b-49fe-8b0c-0148d4e98428", + "id": "c938d3c0-71e7-4d2d-9f16-6c3193069652", "snapshot": false, "version": "8.17.1" }, @@ -963,28 +919,22 @@ An example event for `dlp_forensic_copies` looks as following: "network" ], "dataset": "cloudflare_logpush.dlp_forensic_copies", - "ingested": "2025-12-12T08:39:29Z", + "ingested": "2026-04-07T10:18:20Z", "kind": "event", - "original": "{\"AccountID\":\"acc-id\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"}}", + "original": "{\"AccountID\":\"acc-id\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\"}", "type": [ "info" ] }, "input": { - "type": "aws-s3" - }, - "log": { - "file": { - "path": "https://elastic-package-dlp-forensic-copies-bucket-19014.s3.us-east-1.amazonaws.com/test-dlp-forensic-copies.log" - }, - "offset": 0 + "type": "http_endpoint" }, "rule": { "id": "9" }, "tags": [ - "collect_sqs_logs", "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-dlp_forensic_copies" ] @@ -995,14 +945,13 @@ An example event for `dlp_forensic_copies` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.dlp_forensic_copies.account_id | Cloudflare account ID. | keyword | | cloudflare_logpush.dlp_forensic_copies.datetime | The date and time the corresponding HTTP request was made. | date | | cloudflare_logpush.dlp_forensic_copies.forensic_copy_id | The unique ID for this particular forensic copy. | keyword | @@ -1011,18 +960,15 @@ An example event for `dlp_forensic_copies` looks as following: | cloudflare_logpush.dlp_forensic_copies.payload | Captured request/response data, base64-encoded. | keyword | | cloudflare_logpush.dlp_forensic_copies.phase | Phase of the HTTP request this forensic copy was captured from (i.e. "request" or "response"). | keyword | | cloudflare_logpush.dlp_forensic_copies.triggered_rule_id | The ID of the Gateway firewall rule that triggered this forensic copy. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -1040,15 +986,12 @@ An example event for `dns` looks as following: { "@timestamp": "2022-05-26T09:23:54.000Z", "agent": { - "ephemeral_id": "28ac7554-1d13-49b2-a05e-c28fff86aed2", - "id": "90e30034-7fa9-4654-9018-4c3f3605c19e", - "name": "elastic-agent-61553", + "ephemeral_id": "52bd7d3d-8a51-4486-b6c3-bdc863b853d2", + "id": "29ff4554-a2a6-43c4-a7c6-a69bdd20aad3", + "name": "elastic-agent-20485", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "dns": { "colo": { @@ -1059,29 +1002,35 @@ An example event for `dns` looks as following: "subnet_length": 0 }, "query": { + "name": "example.com", "type": 65535 }, "response": { "cached": false, "code": 0 - } + }, + "source": { + "ip": "175.16.199.0" + }, + "timestamp": "2022-05-26T09:23:54.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.dns", - "namespace": "24004", + "namespace": "29720", "type": "logs" }, "dns": { "question": { "name": "example.com" - } + }, + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "90e30034-7fa9-4654-9018-4c3f3605c19e", + "id": "29ff4554-a2a6-43c4-a7c6-a69bdd20aad3", "snapshot": false, "version": "8.17.1" }, @@ -1091,31 +1040,15 @@ An example event for `dns` looks as following: "network" ], "dataset": "cloudflare_logpush.dns", - "ingested": "2025-12-12T08:46:14Z", + "ingested": "2026-04-07T10:19:17Z", "kind": "event", + "original": "{\"ColoCode\":\"MRS\",\"EDNSSubnet\":\"1.128.0.0\",\"EDNSSubnetLength\":0,\"QueryName\":\"example.com\",\"QueryType\":65535,\"ResponseCached\":false,\"ResponseCode\":0,\"SourceIP\":\"175.16.199.0\",\"Timestamp\":\"2022-05-26T09:23:54Z\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "dns.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/dns.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "ip": [ @@ -1127,6 +1060,8 @@ An example event for `dns` looks as following: "ip": "175.16.199.0" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-dns" ] @@ -1137,14 +1072,13 @@ An example event for `dns` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.dns.colo.code | IATA airport code of data center that received the request. | keyword | | cloudflare_logpush.dns.edns.subnet | EDNS Client Subnet (IPv4 or IPv6). | ip | | cloudflare_logpush.dns.edns.subnet_length | EDNS Client Subnet length. | long | @@ -1154,18 +1088,15 @@ An example event for `dns` looks as following: | cloudflare_logpush.dns.response.code | Integer value of response code. | long | | cloudflare_logpush.dns.source.ip | IP address of the client (IPv4 or IPv6). | ip | | cloudflare_logpush.dns.timestamp | Timestamp at which the query occurred. | date | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -1183,15 +1114,12 @@ An example event for `dns_firewall` looks as following: { "@timestamp": "2023-09-19T12:30:00.000Z", "agent": { - "ephemeral_id": "2f6c2025-b827-4b56-b9d7-5515c0e8ec22", - "id": "9cf56aad-c149-4ec4-b610-b508a6f3812d", - "name": "elastic-agent-80366", + "ephemeral_id": "0bae3f0e-8910-457f-815b-c9c7c798fded", + "id": "71e660c4-04ce-40ed-b44d-69ba6c7bcf60", + "name": "elastic-agent-48112", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "dns_firewall": { "cluster_id": "CLUSTER-001", @@ -1204,6 +1132,7 @@ An example event for `dns_firewall` looks as following: }, "question": { "dnssec_ok": true, + "name": "example.com", "recursion_desired": true, "size": 60, "tcp": false, @@ -1211,8 +1140,13 @@ An example event for `dns_firewall` looks as following: }, "response": { "cached": true, - "cached_stale": false + "cached_stale": false, + "code": "0" }, + "source": { + "ip": "67.43.156.2" + }, + "timestamp": "2023-09-19T12:30:00.000Z", "upstream": { "ip": "81.2.69.144", "response_code": "0", @@ -1222,20 +1156,21 @@ An example event for `dns_firewall` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.dns_firewall", - "namespace": "97058", + "namespace": "87794", "type": "logs" }, "dns": { "question": { - "name": "example.com" + "name": "example.com", + "type": "A" }, - "response_code": "0" + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "9cf56aad-c149-4ec4-b610-b508a6f3812d", + "id": "71e660c4-04ce-40ed-b44d-69ba6c7bcf60", "snapshot": false, "version": "8.17.1" }, @@ -1245,31 +1180,15 @@ An example event for `dns_firewall` looks as following: "network" ], "dataset": "cloudflare_logpush.dns_firewall", - "ingested": "2025-12-12T08:52:06Z", + "ingested": "2026-04-07T10:20:19Z", "kind": "event", + "original": "{\"ClientResponseCode\":0,\"ClusterID\":\"CLUSTER-001\",\"ColoCode\":\"SFO\",\"EDNSSubnet\":\"67.43.156.0\",\"EDNSSubnetLength\":24,\"QueryDO\":true,\"QueryName\":\"example.com\",\"QueryRD\":true,\"QuerySize\":60,\"QueryTCP\":false,\"QueryType\":1,\"ResponseCached\":true,\"ResponseCachedStale\":false,\"SourceIP\":\"67.43.156.2\",\"Timestamp\":\"2023-09-19T12:30:00Z\",\"UpstreamIP\":\"81.2.69.144\",\"UpstreamResponseCode\":0,\"UpstreamResponseTimeMs\":30}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "dns_firewall.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/dns_firewall.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "udp" @@ -1297,6 +1216,8 @@ An example event for `dns_firewall` looks as following: "ip": "67.43.156.2" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-dns_firewall" ] @@ -1307,14 +1228,13 @@ An example event for `dns_firewall` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.dns_firewall.cluster_id | The ID of the cluster which handled this request. | keyword | | cloudflare_logpush.dns_firewall.colo.code | IATA airport code of data center that received the request. | keyword | | cloudflare_logpush.dns_firewall.edns.subnet | EDNS Client Subnet (IPv4 or IPv6). | ip | @@ -1334,18 +1254,15 @@ An example event for `dns_firewall` looks as following: | cloudflare_logpush.dns_firewall.upstream.ip | IP of the upstream nameserver (IPv4 or IPv6). | ip | | cloudflare_logpush.dns_firewall.upstream.response_code | Response code from the upstream nameserver. | keyword | | cloudflare_logpush.dns_firewall.upstream.response_time_ms | Upstream response time in milliseconds. | long | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -1363,15 +1280,12 @@ An example event for `email_security_alerts` looks as following: { "@timestamp": "2024-08-28T15:32:35.000Z", "agent": { - "ephemeral_id": "e482a27d-3e24-4212-8a06-620ac597d5af", - "id": "7bc49be7-d33e-43a3-bfec-1d4c34fbf105", - "name": "elastic-agent-95627", + "ephemeral_id": "c0ef4c22-6924-4e42-86cf-e213e8c61ba4", + "id": "e991eee8-5ba2-4777-bbe2-f414cda793b8", + "name": "elastic-agent-79774", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "email_security_alerts": { "alert_id": "4WtWkr6nlBz9sNH-2024-08-28T15:32:35", @@ -1414,6 +1328,7 @@ An example event for `email_security_alerts` looks as following: "smtp_envelope_to": [ "firstlast+env_to@cloudflare.com" ], + "smtp_helo_server_ip": "81.2.69.144", "smtp_helo_server_ip_as_name": "asn", "smtp_helo_server_ip_as_number": "42", "smtp_helo_server_ip_geo": "US/NV/Las Vegas", @@ -1423,23 +1338,60 @@ An example event for `email_security_alerts` looks as following: "CredentialHarvester", "Dropper" ], + "timestamp": "2024-08-28T15:32:35.000Z", "to": "firstlast+to@cloudflare.com", "to_name": "First Last (to)" } }, "data_stream": { "dataset": "cloudflare_logpush.email_security_alerts", - "namespace": "91726", + "namespace": "89350", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "7bc49be7-d33e-43a3-bfec-1d4c34fbf105", + "id": "e991eee8-5ba2-4777-bbe2-f414cda793b8", "snapshot": false, "version": "8.17.1" }, + "email": { + "attachments": [ + { + "file": { + "hash": { + "md5": "91f073bd208689ddbd248e8989ecae90", + "sha1": "62b77e14e2c43049c45b5725018e78d0f9986930", + "sha256": "3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06", + "ssdeep": "McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3" + }, + "mime_type": "application/x-msi", + "name": "attachment.gif" + } + } + ], + "cc": { + "address": [ + "firstlast+cc@cloudflare.com" + ] + }, + "from": { + "address": [ + "firstlast+from@cloudflare.com" + ] + }, + "message_id": "", + "reply_to": { + "address": [ + "firstlast+reply@cloudflare.com" + ] + }, + "sender": { + "address": "firstlast+env_from@cloudflare.com" + }, + "subject": "innocuous message: please read" + }, "event": { "agent_id_status": "verified", "category": [ @@ -1447,8 +1399,9 @@ An example event for `email_security_alerts` looks as following: "network" ], "dataset": "cloudflare_logpush.email_security_alerts", - "ingested": "2025-12-12T08:58:27Z", + "ingested": "2026-04-07T10:21:21Z", "kind": "alert", + "original": "{\"AlertID\":\"4WtWkr6nlBz9sNH-2024-08-28T15:32:35\",\"AlertReasons\":[\"because\",\"said-so\"],\"Attachments\":[{\"ContentTypeComputed\":\"application/x-msi\",\"ContentTypeProvided\":\"image/gif\",\"Decrypted\":true,\"Encrypted\":true,\"Md5\":\"91f073bd208689ddbd248e8989ecae90\",\"Name\":\"attachment.gif\",\"Sha1\":\"62b77e14e2c43049c45b5725018e78d0f9986930\",\"Sha256\":\"3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06\",\"Ssdeep\":\"McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3\"}],\"CC\":[\"firstlast+cc@cloudflare.com\"],\"CCName\":[\"First Last (cc)\"],\"FinalDisposition\":\"malicious\",\"From\":\"firstlast+from@cloudflare.com\",\"FromName\":\"First Last (from)\",\"Links\":[\"https://example.com\"],\"MessageDeliveryMode\":\"unset\",\"MessageID\":\"\\u003cMessage-ID\\u003e\",\"Origin\":\"unset\",\"OriginalSender\":\"firstlast+origin@cloudflare.com\",\"ReplyTo\":\"firstlast+reply@cloudflare.com\",\"ReplyToName\":\"First Last (reply)\",\"SMTPEnvelopeFrom\":\"firstlast+env_from@cloudflare.com\",\"SMTPEnvelopeTo\":[\"firstlast+env_to@cloudflare.com\"],\"SMTPHeloServerIP\":\"81.2.69.144\",\"SMTPHeloServerIPAsName\":\"asn\",\"SMTPHeloServerIPAsNumber\":\"42\",\"SMTPHeloServerIPGeo\":\"US/NV/Las Vegas\",\"SMTPHeloServerName\":\"servername\",\"Subject\":\"innocuous message: please read\",\"ThreatCategories\":[\"CredentialHarvester\",\"Dropper\"],\"Timestamp\":\"2024-08-28T15:32:35Z\",\"To\":\"firstlast+to@cloudflare.com\",\"ToName\":\"First Last (to)\"}", "type": [ "info" ] @@ -1465,25 +1418,8 @@ An example event for `email_security_alerts` looks as following: "name": "attachment.gif" } ], - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "email_security_alerts.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/email_security_alerts.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hash": [ @@ -1504,6 +1440,9 @@ An example event for `email_security_alerts` looks as following: "firstlast+from@cloudflare.com", "First Last (from)", "firstlast+env_from@cloudflare.com", + "firstlast+origin@cloudflare.com", + "firstlast+reply@cloudflare.com", + "First Last (reply)", "firstlast+env_to@cloudflare.com", "firstlast+cc@cloudflare.com", "First Last (cc)" @@ -1527,6 +1466,8 @@ An example event for `email_security_alerts` looks as following: "ip": "81.2.69.144" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-email_security_alerts" ] @@ -1537,14 +1478,13 @@ An example event for `email_security_alerts` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.email_security_alerts.alert_id | The canonical ID for an Email Security Alert. | keyword | | cloudflare_logpush.email_security_alerts.alert_reasons | Human-readable list of findings which contributed to this message's final disposition. | keyword | | cloudflare_logpush.email_security_alerts.attachments.\* | Metadata of attachments contained in this message. | keyword | @@ -1574,18 +1514,15 @@ An example event for `email_security_alerts` looks as following: | cloudflare_logpush.email_security_alerts.timestamp | Start time of message processing. | date | | cloudflare_logpush.email_security_alerts.to | Email address portions of the To header provided by the sender. | keyword | | cloudflare_logpush.email_security_alerts.to_name | Name portions of the To header provided by the sender. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -1603,32 +1540,22 @@ An example event for `firewall_event` looks as following: { "@timestamp": "2022-05-31T05:23:43.000Z", "agent": { - "ephemeral_id": "37b1d591-989b-4113-b104-1f4212137e5a", - "id": "c39dd230-1d6b-4fa2-a12b-ae61fb5e7f5f", - "name": "elastic-agent-20484", + "ephemeral_id": "4a6896a5-f0b1-4275-9bf9-e6e92c06a166", + "id": "dc6bddca-eefa-4068-adc7-79eda79dc040", + "name": "elastic-agent-39254", "type": "filebeat", "version": "8.17.1" }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-firewall-event-bucket-13780", - "name": "elastic-package-firewall-event-bucket-13780" - }, - "object": { - "key": "test-firewall-event.log" - } - } - }, - "cloud": { - "region": "us-east-1" - }, "cloudflare_logpush": { "firewall_event": { + "action": "block", "client": { "asn": { - "description": "CLOUDFLARENET" + "description": "CLOUDFLARENET", + "value": 15169 }, + "country": "us", + "ip": "175.16.199.0", "ip_class": "searchEngine", "referer": { "host": "abc.example.com", @@ -1638,6 +1565,7 @@ An example event for `firewall_event` looks as following: }, "request": { "host": "xyz.example.com", + "method": "GET", "path": "/abc/checkout", "protocol": "HTTP/1.1", "query": "?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))", @@ -1650,6 +1578,9 @@ An example event for `firewall_event` looks as following: "edge": { "colo": { "code": "IAD" + }, + "response": { + "status": 403 } }, "kind": "firewall", @@ -1669,19 +1600,23 @@ An example event for `firewall_event` looks as following: "ray": { "id": "713d477539b55c29" }, - "source": "firewallrules" + "rule": { + "id": "7dc666e026974dab84884c73b3e2afe1" + }, + "source": "firewallrules", + "timestamp": "2022-05-31T05:23:43.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.firewall_event", - "namespace": "63366", + "namespace": "93638", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "c39dd230-1d6b-4fa2-a12b-ae61fb5e7f5f", + "id": "dc6bddca-eefa-4068-adc7-79eda79dc040", "snapshot": false, "version": "8.17.1" }, @@ -1693,9 +1628,9 @@ An example event for `firewall_event` looks as following: ], "dataset": "cloudflare_logpush.firewall_event", "id": "713d477539b55c29", - "ingested": "2025-12-12T09:03:48Z", + "ingested": "2026-04-07T10:22:20Z", "kind": "event", - "original": "{\"ClientRequestScheme\":\"https\",\"MatchIndex\":1,\"ClientRefererHost\":\"abc.example.com\",\"Source\":\"firewallrules\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\",\"ClientRefererPath\":\"/abc/checkout\",\"Metadata\":{\"filter\":\"1ced07e066a34abf8b14f2a99593bc8d\",\"type\":\"customer\"},\"EdgeResponseStatus\":403,\"ClientRequestProtocol\":\"HTTP/1.1\",\"OriginatorRayID\":\"00\",\"RayID\":\"713d477539b55c29\",\"ClientRequestMethod\":\"GET\",\"ClientIP\":\"175.16.199.0\",\"ClientRequestPath\":\"/abc/checkout\",\"Action\":\"block\",\"Kind\":\"firewall\",\"RuleID\":\"7dc666e026974dab84884c73b3e2afe1\",\"ClientIPClass\":\"searchEngine\",\"ClientASNDescription\":\"CLOUDFLARENET\",\"ClientCountry\":\"us\",\"ClientRefererQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRequestQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"OriginResponseStatus\":0,\"EdgeColoCode\":\"IAD\",\"ClientRefererScheme\":\"referer URL scheme\",\"Datetime\":\"2022-05-31T05:23:43Z\",\"ClientRequestHost\":\"xyz.example.com\",\"ClientASN\":15169}", + "original": "{\"Action\":\"block\",\"ClientASN\":15169,\"ClientASNDescription\":\"CLOUDFLARENET\",\"ClientCountry\":\"us\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"searchEngine\",\"ClientRefererHost\":\"abc.example.com\",\"ClientRefererPath\":\"/abc/checkout\",\"ClientRefererQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))\\u0026timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRefererScheme\":\"referer URL scheme\",\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"GET\",\"ClientRequestPath\":\"/abc/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))\\u0026timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRequestScheme\":\"https\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\",\"Datetime\":\"2022-05-31T05:23:43Z\",\"EdgeColoCode\":\"IAD\",\"EdgeResponseStatus\":403,\"Kind\":\"firewall\",\"MatchIndex\":1,\"Metadata\":{\"filter\":\"1ced07e066a34abf8b14f2a99593bc8d\",\"type\":\"customer\"},\"OriginResponseStatus\":0,\"OriginatorRayID\":\"00\",\"RayID\":\"713d477539b55c29\",\"RuleID\":\"7dc666e026974dab84884c73b3e2afe1\",\"Source\":\"firewallrules\"}", "type": [ "info" ] @@ -1710,13 +1645,7 @@ An example event for `firewall_event` looks as following: "version": "1.1" }, "input": { - "type": "aws-s3" - }, - "log": { - "file": { - "path": "https://elastic-package-firewall-event-bucket-13780.s3.us-east-1.amazonaws.com/test-firewall-event.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "protocol": "http" @@ -1743,8 +1672,8 @@ An example event for `firewall_event` looks as following: "ip": "175.16.199.0" }, "tags": [ - "collect_sqs_logs", "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-firewall_event" ], @@ -1774,14 +1703,13 @@ An example event for `firewall_event` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.firewall_event.action | The code of the first-class action the Cloudflare Firewall took on this request. | keyword | | cloudflare_logpush.firewall_event.client.asn.description | The ASN of the visitor as string. | keyword | | cloudflare_logpush.firewall_event.client.asn.value | The ASN number of the visitor. | long | @@ -1804,12 +1732,13 @@ An example event for `firewall_event` looks as following: | cloudflare_logpush.firewall_event.content_scan.types | List of content types. | keyword | | cloudflare_logpush.firewall_event.edge.colo.code | The airport code of the Cloudflare datacenter that served this request. | keyword | | cloudflare_logpush.firewall_event.edge.response.status | HTTP response status code returned to browser. | long | +| cloudflare_logpush.firewall_event.fraud.user_id | A unique identifier generated by the Fraud Detection system for each user. | keyword | | cloudflare_logpush.firewall_event.kind | The kind of event, currently only possible values are. | keyword | | cloudflare_logpush.firewall_event.leaked_credential_check | Result of the check for leaked credentials. Possible results are: password_leaked | username_and_password_leaked | username_password_similar | username_leaked | clean. | keyword | | cloudflare_logpush.firewall_event.match_index | Rules match index in the chain. | long | | cloudflare_logpush.firewall_event.meta_data | Additional product-specific information. | flattened | -| cloudflare_logpush.firewall_event.origin.ray.id | HTTP origin response status code returned to browser. | keyword | -| cloudflare_logpush.firewall_event.origin.response.status | The RayID of the request that issued the challenge/jschallenge. | long | +| cloudflare_logpush.firewall_event.origin.ray.id | The RayID of the request that issued the challenge/jschallenge. | keyword | +| cloudflare_logpush.firewall_event.origin.response.status | HTTP origin response status code returned to browser. | long | | cloudflare_logpush.firewall_event.ray.id | The RayID of the request. | keyword | | cloudflare_logpush.firewall_event.ref | The user-defined identifier for the rule triggered by this request. | keyword | | cloudflare_logpush.firewall_event.rule.description | The Cloudflare security product-specific Description of the rule triggered by this request. | keyword | @@ -1817,18 +1746,15 @@ An example event for `firewall_event` looks as following: | cloudflare_logpush.firewall_event.source | The Cloudflare security product triggered by this request. | keyword | | cloudflare_logpush.firewall_event.timestamp | The date and time the event occurred at the edge. | date | | cloudflare_logpush.firewall_event.zone.name | The human-readable name of the zone. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -1846,22 +1772,41 @@ An example event for `gateway_dns` looks as following: { "@timestamp": "2023-05-02T22:49:53.000Z", "agent": { - "ephemeral_id": "4b6b3e0f-122c-4c43-bc90-343c619d6b41", - "id": "57130e12-3aa9-4de1-8bab-eed6847b485d", - "name": "elastic-agent-76347", + "ephemeral_id": "78dc937a-0f0c-4fb7-9945-2e49bbc091c6", + "id": "b69b3b71-ded8-4572-b93d-1ef026594275", + "name": "elastic-agent-96379", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "gateway_dns": { + "answers": [ + { + "data": "CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=", + "type": "1" + }, + { + "data": "CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=", + "type": "1" + }, + { + "data": "CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=", + "type": "1" + } + ], "application_id": 0, "colo": { "code": "ORD", "id": 14 }, + "destination": { + "ip": "89.160.20.129", + "port": 443 + }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b111aaa", + "name": "zt-test-vm1" + }, "location": { "id": "f233bd67-78c7-4050-9aff-ad63cce25732", "name": "GCP default" @@ -1882,6 +1827,7 @@ An example event for `gateway_dns` looks as following: "id": "1412", "name": "7bdc7a9c-81d3-4816-8e56-de1acad3dec5" }, + "protocol": "https", "question": { "category": { "ids": [ @@ -1893,17 +1839,37 @@ An example event for `gateway_dns` looks as following: "Technology" ] }, + "name": "security.ubuntu.com", "reversed": "com.ubuntu.security", "size": 48, + "type": "A", "type_id": 1 }, + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", - "timezone_inferred_method": "fromLocalTime" + "response_code": "0", + "source": { + "ip": "67.43.156.2", + "port": 0 + }, + "timestamp": "2023-05-02T22:49:53.000Z", + "timezone": "UTC", + "timezone_inferred_method": "fromLocalTime", + "user": { + "email": "user@test.com", + "id": "166befbb-00e3-5e20-bd6e-27245000000" + } } }, "data_stream": { "dataset": "cloudflare_logpush.gateway_dns", - "namespace": "69154", + "namespace": "45109", "type": "logs" }, "destination": { @@ -1928,6 +1894,9 @@ An example event for `gateway_dns` looks as following: "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b111aaa" + }, "dns": { "answers": [ { @@ -1955,10 +1924,10 @@ An example event for `gateway_dns` looks as following: "response_code": "0" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "57130e12-3aa9-4de1-8bab-eed6847b485d", + "id": "b69b3b71-ded8-4572-b93d-1ef026594275", "snapshot": false, "version": "8.17.1" }, @@ -1968,37 +1937,21 @@ An example event for `gateway_dns` looks as following: "network" ], "dataset": "cloudflare_logpush.gateway_dns", - "ingested": "2025-12-12T09:10:08Z", + "ingested": "2026-04-07T10:23:18Z", "kind": "event", + "original": "{\"ApplicationID\":0,\"ColoCode\":\"ORD\",\"ColoID\":14,\"Datetime\":\"2023-05-02T22:49:53Z\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b111aaa\",\"DeviceName\":\"zt-test-vm1\",\"DstIP\":\"89.160.20.129\",\"DstPort\":443,\"Email\":\"user@test.com\",\"Location\":\"GCP default\",\"LocationID\":\"f233bd67-78c7-4050-9aff-ad63cce25732\",\"MatchedCategoryIDs\":[7,163],\"MatchedCategoryNames\":[\"Photography\",\"Weather\"],\"Policy\":\"7bdc7a9c-81d3-4816-8e56-de1acad3dec5\",\"PolicyID\":\"1412\",\"Protocol\":\"https\",\"QueryCategoryIDs\":[26,155],\"QueryCategoryNames\":[\"Technology\",\"Technology\"],\"QueryName\":\"security.ubuntu.com\",\"QueryNameReversed\":\"com.ubuntu.security\",\"QuerySize\":48,\"QueryType\":1,\"QueryTypeName\":\"A\",\"RCode\":0,\"RData\":[{\"data\":\"CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=\",\"type\":\"1\"},{\"data\":\"CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=\",\"type\":\"1\"},{\"data\":\"CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=\",\"type\":\"1\"}],\"ResolvedIPs\":[\"67.43.156.1\",\"67.43.156.2\",\"67.43.156.3\"],\"ResolverDecision\":\"allowedOnNoPolicyMatch\",\"SrcIP\":\"67.43.156.2\",\"SrcPort\":0,\"TimeZone\":\"UTC\",\"TimeZoneInferredMethod\":\"fromLocalTime\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245000000\"}", "outcome": "success", "timezone": "UTC", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "gateway_dns.log" - } - } - }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b111aaa", "name": "zt-test-vm1" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/gateway_dns.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "protocol": "https" @@ -2009,12 +1962,14 @@ An example event for `gateway_dns` looks as following: "zt-test-vm1" ], "ip": [ + "89.160.20.129", "67.43.156.2", - "89.160.20.129" + "67.43.156.1", + "67.43.156.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245000000", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245000000" ] }, "source": { @@ -2034,6 +1989,8 @@ An example event for `gateway_dns` looks as following: "port": 0 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-gateway_dns" ], @@ -2048,14 +2005,13 @@ An example event for `gateway_dns` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.gateway_dns.account_id | Cloudflare account ID. | keyword | | cloudflare_logpush.gateway_dns.answers | The response data objects. | flattened | | cloudflare_logpush.gateway_dns.application_id | ID of the application the domain belongs to. | long | @@ -2081,6 +2037,12 @@ An example event for `gateway_dns` looks as following: | cloudflare_logpush.gateway_dns.host.name | The name of the device where the HTTP request originated from. | keyword | | cloudflare_logpush.gateway_dns.initial_category.ids | ID or IDs of category that the queried domains belongs to. | keyword | | cloudflare_logpush.gateway_dns.initial_category.names | Name or names of category that the queried domains belongs to. | keyword | +| cloudflare_logpush.gateway_dns.initial_resolved_ips | The IPs used to correlate existing FQDN matching policy between Gateway DNS and Gateway proxy. | keyword | +| cloudflare_logpush.gateway_dns.internal_dns.duration_ms | The time it took for the internal DNS to respond. | long | +| cloudflare_logpush.gateway_dns.internal_dns.fallback_strategy | The fallback strategy applied over the internal DNS response. | keyword | +| cloudflare_logpush.gateway_dns.internal_dns.rcode | The return code sent back by the internal DNS service. | long | +| cloudflare_logpush.gateway_dns.internal_dns.view_id | The DNS internal view identifier that was sent to the internal DNS service. | keyword | +| cloudflare_logpush.gateway_dns.internal_dns.zone_id | The DNS zone identifier returned by the internal DNS service. | keyword | | cloudflare_logpush.gateway_dns.is_response_cached | Response comes from cache or not. | boolean | | cloudflare_logpush.gateway_dns.location.id | UUID of the location the DNS request is coming from. | keyword | | cloudflare_logpush.gateway_dns.location.name | Name of the location the DNS request is coming from. | keyword | @@ -2089,8 +2051,10 @@ An example event for `gateway_dns` looks as following: | cloudflare_logpush.gateway_dns.matched.indicator_feed.ids | ID or IDs of indicator feed(s) that the domain was matched with the policy. | keyword | | cloudflare_logpush.gateway_dns.matched.indicator_feed.names | Name or names of indicator feed(s) that the domain was matched with the policy. | keyword | | cloudflare_logpush.gateway_dns.policy.id | ID of the policy/rule that was applied (if any). | keyword | -| cloudflare_logpush.gateway_dns.policy.name | Name of the policy that was applied (if any) | keyword | +| cloudflare_logpush.gateway_dns.policy.name | Name of the policy that was applied (if any). | keyword | | cloudflare_logpush.gateway_dns.protocol | The protocol used for the DNS query by the client. | keyword | +| cloudflare_logpush.gateway_dns.question.application.ids | ID or IDs of applications the queried domain belongs to. | long | +| cloudflare_logpush.gateway_dns.question.application.names | Name or names of applications the queried domain belongs to. | keyword | | cloudflare_logpush.gateway_dns.question.category.ids | ID or IDs of category that the domain belongs to. | long | | cloudflare_logpush.gateway_dns.question.category.names | Name or names of category that the domain belongs to. | keyword | | cloudflare_logpush.gateway_dns.question.id | Globally unique identifier of the query. | keyword | @@ -2101,13 +2065,16 @@ An example event for `gateway_dns` looks as following: | cloudflare_logpush.gateway_dns.question.size | The size of the DNS request in bytes. | long | | cloudflare_logpush.gateway_dns.question.type | The type of DNS query. | keyword | | cloudflare_logpush.gateway_dns.question.type_id | ID of the type of DNS query. | long | -| cloudflare_logpush.gateway_dns.resolved_ip | The resolved IPs in the response, if any. | ip | +| cloudflare_logpush.gateway_dns.redirect_target_uri | Custom URI to which the user was redirected, if any. | keyword | +| cloudflare_logpush.gateway_dns.registration_id | The UUID of the device registration from which the HTTP request originated. | keyword | +| cloudflare_logpush.gateway_dns.request_context_category.ids | ID or IDs of the category that was sent to gateway in the EDNS request for filtering. | long | +| cloudflare_logpush.gateway_dns.request_context_category.names | Name or names of the category that was sent to gateway in the EDNS request for filtering. | keyword | | cloudflare_logpush.gateway_dns.resolved_ip_details.category.ids | ID or IDs of category that the IPs in the response belongs to. | keyword | | cloudflare_logpush.gateway_dns.resolved_ip_details.category.names | Name or names of category that the IPs in the response belongs to. | keyword | | cloudflare_logpush.gateway_dns.resolved_ip_details.continent_codes | Continent code of each resolved IP, if any. | keyword | | cloudflare_logpush.gateway_dns.resolved_ip_details.country_codes | Country code of each resolved IP, if any. | keyword | | cloudflare_logpush.gateway_dns.resolved_ip_details.ips | The resolved IPs in the response, if any. | ip | -| cloudflare_logpush.gateway_dns.resolver.policy.id | Resolver policy UUID, if any matched. | keyword | +| cloudflare_logpush.gateway_dns.resolver.policy.ids | Resolver policy UUID, if any matched. | keyword | | cloudflare_logpush.gateway_dns.resolver.policy.names | Resolver policy name, if any matched. | keyword | | cloudflare_logpush.gateway_dns.resolver_decision | Result of the DNS query. | keyword | | cloudflare_logpush.gateway_dns.resource_records.json | String that represents the JSON array with the returned resource records. | match_only_text | @@ -2122,18 +2089,15 @@ An example event for `gateway_dns` looks as following: | cloudflare_logpush.gateway_dns.timezone_inferred_method | Method used to pick the time zone for the schedule. | keyword | | cloudflare_logpush.gateway_dns.user.email | Email used to authenticate the client. | keyword | | cloudflare_logpush.gateway_dns.user.id | User identity where the HTTP request originated from. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -2151,18 +2115,16 @@ An example event for `gateway_http` looks as following: { "@timestamp": "2023-05-03T20:55:05.000Z", "agent": { - "ephemeral_id": "48652a2c-a69f-44aa-8c1c-cadb9863f0c9", - "id": "48f0311d-16dc-41e5-b2cf-40ae0743d746", - "name": "elastic-agent-46882", + "ephemeral_id": "16e90d1c-6204-4d75-99a4-1b3952547172", + "id": "7871aadb-a497-48a4-a492-94f7e6e190e0", + "name": "elastic-agent-69130", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "gateway_http": { "account_id": "e1836771179f98aabb828da5ea69a348", + "action": "block", "blocked_file": { "hash": "91dc1db739a705105e1c763bfdbdaa84c0de8", "name": "downloaded_test", @@ -2170,6 +2132,10 @@ An example event for `gateway_http` looks as following: "size": 43, "type": "bin" }, + "destination": { + "ip": "89.160.20.129", + "port": 443 + }, "downloaded_files": [ "downloaded_file", "downloaded_test" @@ -2186,28 +2152,47 @@ An example event for `gateway_http` looks as following: } ] }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff", + "name": "zt-test-vm1" + }, "isolated": false, "policy": { "id": "85063bec-74cb-4546-85a3-e0cde2cdfda2", "name": "Block Yahoo" }, "request": { - "host": "guce.yahoo.com" + "host": "guce.yahoo.com", + "method": "GET", + "referrer": "https://www.example.com/", + "version": "HTTP/2" }, "request_id": "1884fec9b600007fb06a299400000001", + "response": { + "status_code": 302 + }, "source": { - "internal_ip": "192.168.1.123" + "internal_ip": "192.168.1.123", + "ip": "67.43.156.2", + "port": 47924 }, + "timestamp": "2023-05-03T20:55:05.000Z", "untrusted_certificate_action": "none", "uploaded_files": [ "uploaded_file", "uploaded_test" - ] + ], + "url": "https://test.com", + "user": { + "email": "user@example.com", + "id": "166befbb-00e3-5e20-bd6e-27245723949f" + }, + "user_agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0" } }, "data_stream": { "dataset": "cloudflare_logpush.gateway_http", - "namespace": "27028", + "namespace": "70767", "type": "logs" }, "destination": { @@ -2232,11 +2217,14 @@ An example event for `gateway_http` looks as following: "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "48f0311d-16dc-41e5-b2cf-40ae0743d746", + "id": "7871aadb-a497-48a4-a492-94f7e6e190e0", "snapshot": false, "version": "8.17.1" }, @@ -2247,23 +2235,21 @@ An example event for `gateway_http` looks as following: "network" ], "dataset": "cloudflare_logpush.gateway_http", - "ingested": "2025-12-12T09:16:06Z", + "ingested": "2026-04-07T10:24:18Z", "kind": "event", + "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a348\",\"Action\":\"block\",\"BlockedFileHash\":\"91dc1db739a705105e1c763bfdbdaa84c0de8\",\"BlockedFileName\":\"downloaded_test\",\"BlockedFileReason\":\"malware\",\"BlockedFileSize\":43,\"BlockedFileType\":\"bin\",\"Datetime\":\"2023-05-03T20:55:05Z\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"DownloadedFileNames\":[\"downloaded_file\",\"downloaded_test\"],\"Email\":\"user@example.com\",\"FileInfo\":{\"files\":[{\"name\":\"downloaded_file\",\"size\":43},{\"name\":\"downloaded_test\",\"size\":341}]},\"HTTPHost\":\"guce.yahoo.com\",\"HTTPMethod\":\"GET\",\"HTTPStatusCode\":302,\"HTTPVersion\":\"HTTP/2\",\"IsIsolated\":false,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"Block Yahoo\",\"Referer\":\"https://www.example.com/\",\"RequestID\":\"1884fec9b600007fb06a299400000001\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.123\",\"SourcePort\":47924,\"URL\":\"https://test.com\",\"UntrustedCertificateAction\":\"none\",\"UploadedFileNames\":[\"uploaded_file\",\"uploaded_test\"],\"UserAgent\":\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ "info", "denied" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "gateway_http.log" - } - } + "file": { + "extension": "bin", + "hash": { + "sha256": "91dc1db739a705105e1c763bfdbdaa84c0de8" + }, + "name": "downloaded_test", + "size": 43 }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", @@ -2280,27 +2266,25 @@ An example event for `gateway_http` looks as following: "version": "HTTP/2" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/gateway_http.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { + "hash": [ + "91dc1db739a705105e1c763bfdbdaa84c0de8" + ], "hosts": [ "083a8354-d56c-11ed-9771-6a842b100cff", - "zt-test-vm1" + "zt-test-vm1", + "guce.yahoo.com" ], "ip": [ - "67.43.156.2", "89.160.20.129", + "67.43.156.2", "192.168.1.123" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@example.com" + "user@example.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -2320,6 +2304,8 @@ An example event for `gateway_http` looks as following: "port": 47924 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-gateway_http" ], @@ -2342,18 +2328,19 @@ An example event for `gateway_http` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.gateway_http.account_id | Cloudflare account tag. | keyword | | cloudflare_logpush.gateway_http.action | Action performed by gateway on the HTTP request. | keyword | +| cloudflare_logpush.gateway_http.app_control_info | Information about application control operations, APIs, and groups that matched the HTTP request. | flattened | | cloudflare_logpush.gateway_http.application.ids | IDs of the applications that matched the session parameters. | keyword | | cloudflare_logpush.gateway_http.application.names | Names of the applications that matched the session parameters. | keyword | +| cloudflare_logpush.gateway_http.application.statuses | Statuses of the applications that matched the HTTP request parameters. | keyword | | cloudflare_logpush.gateway_http.blocked_file.hash | Hash of the file blocked in the response, if any. | keyword | | cloudflare_logpush.gateway_http.blocked_file.name | File name blocked in the request, if any. | keyword | | cloudflare_logpush.gateway_http.blocked_file.reason | Reason file was blocked in the response, if any. | keyword | @@ -2377,7 +2364,9 @@ An example event for `gateway_http` looks as following: | cloudflare_logpush.gateway_http.policy.name | The name of the gateway policy applied to the request, if any. | keyword | | cloudflare_logpush.gateway_http.private_app_aud | The private app AUD, if any. | keyword | | cloudflare_logpush.gateway_http.proxy_endpoint | The proxy endpoint used on this network session, if any. | keyword | -| cloudflare_logpush.gateway_http.quarantined | If the request content was quarantined. | keyword | +| cloudflare_logpush.gateway_http.quarantined | If the request content was quarantined. | boolean | +| cloudflare_logpush.gateway_http.redirect_target_uri | Custom URI to which the user was redirected, if any. | keyword | +| cloudflare_logpush.gateway_http.registration_id | The UUID of the device registration from which the HTTP request originated. | keyword | | cloudflare_logpush.gateway_http.request.host | Content of the host header in the HTTP request. | keyword | | cloudflare_logpush.gateway_http.request.method | HTTP request method. | keyword | | cloudflare_logpush.gateway_http.request.referrer | Contents of the referer header in the HTTP request. | keyword | @@ -2401,18 +2390,15 @@ An example event for `gateway_http` looks as following: | cloudflare_logpush.gateway_http.user_agent | Contents of the user agent header in the HTTP request. | keyword | | cloudflare_logpush.gateway_http.virtual_network.id | The identifier of the virtual network the device was connected to, if any. | keyword | | cloudflare_logpush.gateway_http.virtual_network.name | The name of the virtual network the device was connected to, if any. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -2430,18 +2416,24 @@ An example event for `gateway_network` looks as following: { "@timestamp": "2023-05-18T21:12:57.058Z", "agent": { - "ephemeral_id": "e97e2537-55fc-411f-9d94-ee80a16b9840", - "id": "b7375e7f-8998-462f-8a35-45412fd644da", - "name": "elastic-agent-36424", + "ephemeral_id": "84fcdf72-331c-4a7f-823e-539c5ea00ede", + "id": "39b947da-4405-47cb-9077-a8d58d0ff74b", + "name": "elastic-agent-16155", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "gateway_network": { "account_id": "e1836771179f98aabb828da5ea69a111", + "action": "allowedOnNoRuleMatch", + "destination": { + "ip": "89.160.20.129", + "port": 443 + }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff", + "name": "zt-test-vm1" + }, "override": { "ip": "175.16.199.4", "port": 8080 @@ -2450,14 +2442,24 @@ An example event for `gateway_network` looks as following: "id": "85063bec-74cb-4546-85a3-e0cde2cdfda2", "name": "My policy" }, + "session_id": "5f2d04be-3512-11e8-b467-0ed5f89f718b", + "sni": "www.elastic.co", "source": { - "internal_ip": "192.168.1.3" + "internal_ip": "192.168.1.3", + "ip": "67.43.156.2", + "port": 47924 + }, + "timestamp": "2023-05-18T21:12:57.058Z", + "transport": "tcp", + "user": { + "email": "user@test.com", + "id": "166befbb-00e3-5e20-bd6e-27245723949f" } } }, "data_stream": { "dataset": "cloudflare_logpush.gateway_network", - "namespace": "25906", + "namespace": "62524", "type": "logs" }, "destination": { @@ -2483,11 +2485,14 @@ An example event for `gateway_network` looks as following: "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "b7375e7f-8998-462f-8a35-45412fd644da", + "id": "39b947da-4405-47cb-9077-a8d58d0ff74b", "snapshot": false, "version": "8.17.1" }, @@ -2499,54 +2504,37 @@ An example event for `gateway_network` looks as following: ], "dataset": "cloudflare_logpush.gateway_network", "id": "5f2d04be-3512-11e8-b467-0ed5f89f718b", - "ingested": "2025-12-12T09:23:26Z", + "ingested": "2026-04-07T10:25:18Z", "kind": "event", + "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"Action\":\"allowedOnNoRuleMatch\",\"Datetime\":1684444377058000000,\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"Email\":\"user@test.com\",\"OverrideIP\":\"175.16.199.4\",\"OverridePort\":8080,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"My policy\",\"SNI\":\"www.elastic.co\",\"SessionID\":\"5f2d04be-3512-11e8-b467-0ed5f89f718b\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.3\",\"SourcePort\":47924,\"Transport\":\"tcp\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "gateway_network.log" - } - } - }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", "name": "zt-test-vm1" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/gateway_network.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "tcp" }, "related": { "hosts": [ - "www.elastic.co", "083a8354-d56c-11ed-9771-6a842b100cff", "zt-test-vm1" ], "ip": [ - "67.43.156.2", "89.160.20.129", "175.16.199.4", + "67.43.156.2", "192.168.1.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -2566,6 +2554,8 @@ An example event for `gateway_network` looks as following: "port": 47924 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-gateway_network" ], @@ -2585,14 +2575,13 @@ An example event for `gateway_network` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.gateway_network.account_id | Cloudflare account tag. | keyword | | cloudflare_logpush.gateway_network.action | Action performed by gateway on the session. | keyword | | cloudflare_logpush.gateway_network.application.ids | IDs of the applications that matched the session parameters. | keyword | @@ -2606,11 +2595,12 @@ An example event for `gateway_network` looks as following: | cloudflare_logpush.gateway_network.detected_protocol | Detected traffic protocol of the network session. | keyword | | cloudflare_logpush.gateway_network.host.id | UUID of the device where the network session originated from. | keyword | | cloudflare_logpush.gateway_network.host.name | The name of the device where the network session originated from. | keyword | -| cloudflare_logpush.gateway_network.override.ip | Overriden IP of the network session, if any. | ip | -| cloudflare_logpush.gateway_network.override.port | Overriden port of the network session, if any. | long | +| cloudflare_logpush.gateway_network.override.ip | Overridden IP of the network session, if any. | ip | +| cloudflare_logpush.gateway_network.override.port | Overridden port of the network session, if any. | long | | cloudflare_logpush.gateway_network.policy.id | Identifier of the policy/rule that was applied, if any. | keyword | | cloudflare_logpush.gateway_network.policy.name | The name of the gateway policy applied to the session, if any. | keyword | | cloudflare_logpush.gateway_network.proxy_endpoint | The proxy endpoint used on this network session, if any. | keyword | +| cloudflare_logpush.gateway_network.registration_id | The UUID of the device registration from which the network session originated. | keyword | | cloudflare_logpush.gateway_network.session_id | The session identifier of this network session. | keyword | | cloudflare_logpush.gateway_network.sni | Content of the SNI (Server Name Indication) for the TLS network session, if any. | keyword | | cloudflare_logpush.gateway_network.source.internal_ip | Local LAN IP of the device. Only available when connected via a GRE/IPsec tunnel on-ramp. | ip | @@ -2620,22 +2610,19 @@ An example event for `gateway_network` looks as following: | cloudflare_logpush.gateway_network.source_ip.country_code | Country code of the source IP of the network session. | keyword | | cloudflare_logpush.gateway_network.timestamp | The date and time the corresponding network session was made. | date | | cloudflare_logpush.gateway_network.transport | Transport protocol used for this session. | keyword | -| cloudflare_logpush.gateway_network.user.email | Email associated with the user identity where the network sesion originated from. | keyword | +| cloudflare_logpush.gateway_network.user.email | Email associated with the user identity where the network session originated from. | keyword | | cloudflare_logpush.gateway_network.user.id | User identity where the network session originated from. | keyword | | cloudflare_logpush.gateway_network.virtual_network.id | The identifier of the virtual network the device was connected to, if any. | keyword | | cloudflare_logpush.gateway_network.virtual_network.name | The name of the virtual network the device was connected to, if any. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -2653,9 +2640,9 @@ An example event for `http_request` looks as following: { "@timestamp": "2022-05-25T13:25:26.000Z", "agent": { - "ephemeral_id": "3cd010e7-a0bc-4a53-aff5-3af3c380387f", - "id": "3ce9b73c-faf9-42ad-a254-eb64a83a369c", - "name": "elastic-agent-80039", + "ephemeral_id": "63db57a5-406d-44bc-836c-da04e2acb85e", + "id": "1b015357-e819-4d54-9d04-e72849d9691b", + "name": "elastic-agent-32793", "type": "filebeat", "version": "8.17.1" }, @@ -2843,17 +2830,20 @@ An example event for `http_request` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.http_request", - "namespace": "73428", + "namespace": "38039", "type": "logs" }, "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "3ce9b73c-faf9-42ad-a254-eb64a83a369c", + "id": "1b015357-e819-4d54-9d04-e72849d9691b", "snapshot": false, "version": "8.17.1" }, @@ -2864,7 +2854,7 @@ An example event for `http_request` looks as following: ], "dataset": "cloudflare_logpush.http_request", "id": "710e98d9367f357d", - "ingested": "2026-01-20T11:31:07Z", + "ingested": "2026-04-07T10:26:19Z", "kind": "event", "original": "{\"BotDetectionIDs\":[7,8,9],\"BotScore\":20,\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":[\"bing\",\"api\"],\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityAction\":\"unknown\",\"SecurityLevel\":\"off\",\"SecurityRuleDescription\":\"matchad variable message\",\"SecurityRuleID\":\"98d93d5\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAttackScore\":50,\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRCEAttackScore\":1,\"WAFSQLiAttackScore\":99,\"WAFXSSAttackScore\":90,\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}", "type": [ @@ -2873,9 +2863,11 @@ An example event for `http_request` looks as following: }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -2888,9 +2880,17 @@ An example event for `http_request` looks as following: "protocol": "http" }, "related": { + "hash": [ + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -2900,7 +2900,8 @@ An example event for `http_request` looks as following: "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", @@ -2939,14 +2940,13 @@ An example event for `http_request` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.http_request.bot.detection_ids | List of IDs that correlate to the Bot Management Heuristic detections made on a request. Available in Logpush v2 only. | long | | cloudflare_logpush.http_request.bot.detection_tags | List of tags that correlate to the Bot Management Heuristic detections made on a request. Available only for Bot Management customers. To enable this feature, contact your account team. | keyword | | cloudflare_logpush.http_request.bot.score.src | Detection engine responsible for generating the Bot Score. Possible values are Not Computed, Heuristics, Machine Learning, Behavioral Analysis, Verified Bot, JS Fingerprinting, Cloudflare Service. | text | @@ -2954,8 +2954,8 @@ An example event for `http_request` looks as following: | cloudflare_logpush.http_request.bot.tag | Type of bot traffic (if available). Available in Logpush v2 only. | text | | cloudflare_logpush.http_request.cache.reserve_used | Cache Reserve was used to serve this request. | boolean | | cloudflare_logpush.http_request.cache.response.bytes | Number of bytes returned by the cache. | long | -| cloudflare_logpush.http_request.cache.response.status | Cache status. | long | -| cloudflare_logpush.http_request.cache.status | HTTP status code returned by the cache to the edge. | keyword | +| cloudflare_logpush.http_request.cache.response.status | HTTP status code returned by the cache to the edge. | long | +| cloudflare_logpush.http_request.cache.status | Cache status. | keyword | | cloudflare_logpush.http_request.cache.tiered_fill | Tiered Cache was used to serve this request. | boolean | | cloudflare_logpush.http_request.client.asn | Client AS number. | long | | cloudflare_logpush.http_request.client.city | Approximate city of the client. | keyword | @@ -2987,7 +2987,7 @@ An example event for `http_request` looks as following: | cloudflare_logpush.http_request.content_scan.sizes | List of content object sizes. | long | | cloudflare_logpush.http_request.content_scan.types | List of content types. | keyword | | cloudflare_logpush.http_request.cookies | String key-value pairs for Cookies. | flattened | -| cloudflare_logpush.http_request.datetime | Timestamp when the request was received | date | +| cloudflare_logpush.http_request.datetime | Timestamp when the request was received. | date | | cloudflare_logpush.http_request.edge.cf_connecting_o2o | True if the request looped through multiple zones on the Cloudflare edge. | boolean | | cloudflare_logpush.http_request.edge.colo.code | IATA airport code of data center that received the request. | keyword | | cloudflare_logpush.http_request.edge.colo.id | Cloudflare edge colo id. | long | @@ -3009,9 +3009,16 @@ An example event for `http_request` looks as following: | cloudflare_logpush.http_request.firewall.matches.action | Array of actions the Cloudflare firewall products performed on this request. | keyword | | cloudflare_logpush.http_request.firewall.matches.rule_id | Array of RuleIDs of the firewall product that has matched the request. | keyword | | cloudflare_logpush.http_request.firewall.matches.sources | The firewall products that matched the request. | keyword | +| cloudflare_logpush.http_request.fraud.attack | The primary attack or use case detected by Fraud detections. | keyword | +| cloudflare_logpush.http_request.fraud.detection_ids | List of IDs correlating to Fraud detections. | long | +| cloudflare_logpush.http_request.fraud.detection_tags | List of tags correlating to Fraud detections. | keyword | +| cloudflare_logpush.http_request.fraud.email_risk | Risk of a specific email address (low/medium/high). | keyword | +| cloudflare_logpush.http_request.fraud.user_id | A unique identifier generated by the Fraud Detection system. | keyword | | cloudflare_logpush.http_request.ja3_hash | The MD5 hash of the JA3 fingerprint used to profile SSL/TLS clients. | keyword | | cloudflare_logpush.http_request.ja4 | The JA4 fingerprint used to profile SSL/TLS clients. Available only for Bot Management customers. | keyword | | cloudflare_logpush.http_request.ja4_signals | Inter-request statistics computed for this JA4 fingerprint. JA4Signals field is organized in key:value pairs, where values are numbers. Available only for Bot Management customers. | flattened | +| cloudflare_logpush.http_request.js_detection_passed | Whether the request passed background JS Detection (passed/failed/missing). | keyword | +| cloudflare_logpush.http_request.leaked_credential_check | Result of the check for leaked credentials. | keyword | | cloudflare_logpush.http_request.origin.dns_response_time.ms | Time taken to receive a DNS response for an origin name. | long | | cloudflare_logpush.http_request.origin.ip | IP of the origin server. | ip | | cloudflare_logpush.http_request.origin.request_header_send_duration.ms | Time taken to send request headers to origin after establishing a connection. | long | @@ -3026,12 +3033,14 @@ An example event for `http_request` looks as following: | cloudflare_logpush.http_request.origin.tcp_handshake_duration.ms | Time taken to complete TCP handshake with origin. | long | | cloudflare_logpush.http_request.origin.tls_handshake_duration.ms | Time taken to complete TLS handshake with origin. | long | | cloudflare_logpush.http_request.parent_ray.id | Ray ID of the parent request if this request was made using a Worker script. | keyword | +| cloudflare_logpush.http_request.pay_per_crawl_status | Pay Per Crawl outcome. | keyword | | cloudflare_logpush.http_request.ray.id | ID of the request. | keyword | | cloudflare_logpush.http_request.request.headers | String key-value pairs for RequestHeaders. | flattened | | cloudflare_logpush.http_request.response.headers | String key-value pairs for ResponseHeaders. | flattened | | cloudflare_logpush.http_request.security_level | The security level configured at the time of this request. This is used to determine the sensitivity of the IP Reputation system. | text | | cloudflare_logpush.http_request.smart_route.colo.id | The Cloudflare datacenter used to connect to the origin server if Argo Smart Routing is used. Available in Logpush v2 only. | long | | cloudflare_logpush.http_request.upper_tier.colo.id | The “upper tier” datacenter that was checked for a cached copy if Tiered Cache is used. Available in Logpush v2 only. | long | +| cloudflare_logpush.http_request.verified_bot_category | The category of verified bot. | keyword | | cloudflare_logpush.http_request.waf.action | Action taken by the WAF, if triggered. | text | | cloudflare_logpush.http_request.waf.flag | Additional configuration flags. | text | | cloudflare_logpush.http_request.waf.matched_var | The full name of the most-recently matched variable. | text | @@ -3042,25 +3051,25 @@ An example event for `http_request` looks as following: | cloudflare_logpush.http_request.waf.score.rce | WAF score for a Remote Code Execution (RCE) attack. | long | | cloudflare_logpush.http_request.waf.score.sqli | WAF score for an SQL injection (SQLi) attack. | long | | cloudflare_logpush.http_request.waf.score.xss | WAF score for a Cross-site scripting (XSS) attack. | long | +| cloudflare_logpush.http_request.web_assets.labels_managed | Cloudflare-defined labels matched for the request. | keyword | +| cloudflare_logpush.http_request.web_assets.operation_id | UUID of the matched web asset operation. | keyword | | cloudflare_logpush.http_request.worker.cpu_time | Amount of time in microseconds spent executing a worker, if any. | long | +| cloudflare_logpush.http_request.worker.script_name | The Worker script name that made the request. | keyword | | cloudflare_logpush.http_request.worker.status | Status returned from worker daemon. | text | | cloudflare_logpush.http_request.worker.subrequest.count | Number of subrequests issued by a worker when handling this request. | long | | cloudflare_logpush.http_request.worker.subrequest.value | Whether or not this request was a worker subrequest. | boolean | | cloudflare_logpush.http_request.worker.wall_time_us | Real-time in microseconds elapsed between start and end of worker invocation. | long | | cloudflare_logpush.http_request.zone.id | Internal zone ID. | long | | cloudflare_logpush.http_request.zone.name | The human-readable name of the zone. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -3078,31 +3087,39 @@ An example event for `magic_ids` looks as following: { "@timestamp": "2023-09-11T03:02:57.000Z", "agent": { - "ephemeral_id": "b4087b6e-9b59-4f1e-b4a1-8be9334146fb", - "id": "01cff4ad-3e8d-4819-a226-ad295d9fcb79", - "name": "elastic-agent-65074", + "ephemeral_id": "59364f67-62d2-4044-970d-40ba6542ffdc", + "id": "33f5db7e-7e2e-4e82-8a91-c5694677f76d", + "name": "elastic-agent-64158", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "magic_ids": { + "action": "pass", "colo": { "city": "Tokyo", "code": "NRT" }, + "destination": { + "ip": "89.160.20.129", + "port": 80 + }, "signature": { "id": 2031296, "message": "ET CURRENT_EVENTS [Fireeye] POSSIBLE HackTool.TCP.Rubeus.[User32LogonProcesss]", "revision": 1 - } + }, + "source": { + "ip": "67.43.156.2", + "port": 44667 + }, + "timestamp": "2023-09-11T03:02:57.000Z", + "transport": "tcp" } }, "data_stream": { "dataset": "cloudflare_logpush.magic_ids", - "namespace": "22953", + "namespace": "77401", "type": "logs" }, "destination": { @@ -3128,10 +3145,10 @@ An example event for `magic_ids` looks as following: "port": 80 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "01cff4ad-3e8d-4819-a226-ad295d9fcb79", + "id": "33f5db7e-7e2e-4e82-8a91-c5694677f76d", "snapshot": false, "version": "8.17.1" }, @@ -3143,32 +3160,16 @@ An example event for `magic_ids` looks as following: "intrusion_detection" ], "dataset": "cloudflare_logpush.magic_ids", - "ingested": "2025-12-12T09:39:17Z", + "ingested": "2026-04-07T10:27:20Z", "kind": "event", + "original": "{\"Action\":\"pass\",\"ColoCity\":\"Tokyo\",\"ColoCode\":\"NRT\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":80,\"Protocol\":\"tcp\",\"SignatureID\":2031296,\"SignatureMessage\":\"ET CURRENT_EVENTS [Fireeye] POSSIBLE HackTool.TCP.Rubeus.[User32LogonProcesss]\",\"SignatureRevision\":1,\"SourceIP\":\"67.43.156.2\",\"SourcePort\":44667,\"Timestamp\":\"2023-09-11T03:02:57Z\"}", "type": [ "info", "allowed" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "magic_ids.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/magic_ids.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "tcp" @@ -3196,6 +3197,8 @@ An example event for `magic_ids` looks as following: "port": 44667 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-magic_ids" ] @@ -3206,14 +3209,13 @@ An example event for `magic_ids` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.magic_ids.action | What action was taken on the packet. Possible values are pass | block. | keyword | | cloudflare_logpush.magic_ids.colo.city | The city where the detection occurred. | keyword | | cloudflare_logpush.magic_ids.colo.code | The IATA airport code corresponding to where the detection occurred. | keyword | @@ -3226,18 +3228,15 @@ An example event for `magic_ids` looks as following: | cloudflare_logpush.magic_ids.source.port | The source port of the packet which triggered the detection. It is set to 0 if the protocol field is set to any. | long | | cloudflare_logpush.magic_ids.timestamp | A timestamp of when the detection occurred. | date | | cloudflare_logpush.magic_ids.transport | The layer 4 protocol of the packet which triggered the detection. Possible values are tcp | udp | any. Variant any means a detection occurred at a lower layer (such as IP). | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -3255,15 +3254,12 @@ An example event for `nel_report` looks as following: { "@timestamp": "2021-07-27T00:01:07.000Z", "agent": { - "ephemeral_id": "268e4658-b07c-4510-b437-dcbb422583b5", - "id": "7ccbc26d-a497-4fcf-8a84-93b0b1f3c120", - "name": "elastic-agent-51944", + "ephemeral_id": "6d48242a-56b5-4837-b56e-33babdc5e374", + "id": "ecb544ed-5183-4910-8bf4-e1d26c69e8fd", + "name": "elastic-agent-98632", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "nel_report": { "client": { @@ -3275,24 +3271,28 @@ An example event for `nel_report` looks as following: "country": "US" } }, + "error": { + "type": "network-error" + }, "last_known_good": { "colo": { "code": "SJC" } }, - "phase": "connection" + "phase": "connection", + "timestamp": "2021-07-27T00:01:07.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.nel_report", - "namespace": "32139", + "namespace": "90511", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "7ccbc26d-a497-4fcf-8a84-93b0b1f3c120", + "id": "ecb544ed-5183-4910-8bf4-e1d26c69e8fd", "snapshot": false, "version": "8.17.1" }, @@ -3305,33 +3305,19 @@ An example event for `nel_report` looks as following: "network" ], "dataset": "cloudflare_logpush.nel_report", - "ingested": "2025-12-12T09:45:27Z", + "ingested": "2026-04-07T10:28:19Z", "kind": "event", + "original": "{\"ClientIPASN\":\"13335\",\"ClientIPASNDescription\":\"CLOUDFLARENET\",\"ClientIPCountry\":\"US\",\"LastKnownGoodColoCode\":\"SJC\",\"Phase\":\"connection\",\"Timestamp\":\"2021-07-27T00:01:07Z\",\"Type\":\"network-error\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "nel_report.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/nel_report.log" - }, - "offset": 0 + "type": "http_endpoint" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-nel_report" ] @@ -3342,14 +3328,13 @@ An example event for `nel_report` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.nel_report.client.ip.asn.description | Client ASN description. | keyword | | cloudflare_logpush.nel_report.client.ip.asn.value | Client ASN. | long | | cloudflare_logpush.nel_report.client.ip.country | Client country. | keyword | @@ -3357,18 +3342,15 @@ An example event for `nel_report` looks as following: | cloudflare_logpush.nel_report.last_known_good.colo.code | IATA airport code of colo client connected to. | keyword | | cloudflare_logpush.nel_report.phase | The phase of connection the error occurred in. | keyword | | cloudflare_logpush.nel_report.timestamp | Timestamp for error report. | date | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -3386,15 +3368,12 @@ An example event for `network_analytics` looks as following: { "@timestamp": "2021-07-27T00:01:07.000Z", "agent": { - "ephemeral_id": "0ed44bac-a86d-417a-83f6-06250aca48c5", - "id": "47e353e4-8ed7-44c4-b167-ad9581cbdeaf", - "name": "elastic-agent-13604", + "ephemeral_id": "5e8103cc-39cd-49c1-adb5-76502c3887c3", + "id": "841ed7ef-7586-4df8-980f-703f36dd251e", + "name": "elastic-agent-32455", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "network_analytics": { "attack": { @@ -3416,10 +3395,14 @@ An example event for `network_analytics` looks as following: "description": "asn description" } }, + "asn": 1900, "country": "AD", "geo_hash": "gbuun", - "geo_location": "gbuun" + "geo_location": "gbuun", + "ip": "175.16.199.0", + "port": 1234 }, + "direction": "ingress", "gre": { "checksum": 10, "ether": { @@ -3453,6 +3436,7 @@ An example event for `network_analytics` looks as following: "fragments": 1480 }, "protocol": { + "name": "tcp", "value": 6 }, "source": { @@ -3489,8 +3473,10 @@ An example event for `network_analytics` looks as following: "scope": "local", "system": "flowtrackd" }, + "outcome": "success", "protocol_state": "OPEN", "rule": { + "id": "rule1", "set": { "id": "3b64149bfa6e4220bbbc2bd6db589552", "override": { @@ -3505,9 +3491,12 @@ An example event for `network_analytics` looks as following: "description": "Source ASN Description" } }, + "asn": 1500, "country": "AD", "geo_hash": "gbuun", - "geo_location": "gbuun" + "geo_location": "gbuun", + "ip": "67.43.156.0", + "port": 5678 }, "tcp": { "acknowledgement_number": 1000, @@ -3536,6 +3525,7 @@ An example event for `network_analytics` looks as following: "size": 10 } }, + "timestamp": "2021-07-27T00:01:07.000Z", "udp": { "checksum": 10, "payload_length": 10 @@ -3545,7 +3535,7 @@ An example event for `network_analytics` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.network_analytics", - "namespace": "16796", + "namespace": "91714", "type": "logs" }, "destination": { @@ -3553,13 +3543,13 @@ An example event for `network_analytics` looks as following: "number": 1900 }, "ip": "175.16.199.0", - "port": 0 + "port": 1234 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "47e353e4-8ed7-44c4-b167-ad9581cbdeaf", + "id": "841ed7ef-7586-4df8-980f-703f36dd251e", "snapshot": false, "version": "8.17.1" }, @@ -3569,34 +3559,19 @@ An example event for `network_analytics` looks as following: "network" ], "dataset": "cloudflare_logpush.network_analytics", - "ingested": "2025-12-12T09:51:57Z", + "ingested": "2026-04-07T10:44:19Z", "kind": "event", + "original": "{\"AttackCampaignID\":\"xyz987\",\"AttackID\":\"abc777\",\"ColoCountry\":\"AD\",\"ColoGeoHash\":\"gbuun\",\"ColoID\":46,\"ColoName\":\"SJC\",\"Datetime\":\"2021-07-27T00:01:07Z\",\"DestinationASN\":1900,\"DestinationASNDescription\":\"asn description\",\"DestinationCountry\":\"AD\",\"DestinationGeoHash\":\"gbuun\",\"DestinationPort\":1234,\"Direction\":\"ingress\",\"GREChecksum\":10,\"GREEthertype\":10,\"GREHeaderLength\":1024,\"GREKey\":10,\"GRESequenceNumber\":10,\"GREVersion\":10,\"ICMPChecksum\":10,\"ICMPCode\":10,\"ICMPType\":10,\"IPDestinationAddress\":\"175.16.199.0\",\"IPDestinationSubnet\":\"/24\",\"IPFragmentOffset\":1480,\"IPHeaderLength\":20,\"IPMoreFragments\":1480,\"IPProtocol\":6,\"IPProtocolName\":\"tcp\",\"IPSourceAddress\":\"67.43.156.0\",\"IPSourceSubnet\":\"/24\",\"IPTotalLength\":1024,\"IPTotalLengthBuckets\":10,\"IPTtl\":240,\"IPTtlBuckets\":2,\"IPv4Checksum\":0,\"IPv4DontFragment\":0,\"IPv4Dscp\":46,\"IPv4Ecn\":1,\"IPv4Identification\":1,\"IPv4Options\":1,\"IPv6Dscp\":46,\"IPv6Ecn\":1,\"IPv6ExtensionHeaders\":\"header\",\"IPv6FlowLabel\":1,\"IPv6Identification\":1,\"MitigationReason\":\"BLOCKED\",\"MitigationScope\":\"local\",\"MitigationSystem\":\"flowtrackd\",\"Outcome\":\"pass\",\"ProtocolState\":\"OPEN\",\"RuleID\":\"rule1\",\"RulesetID\":\"3b64149bfa6e4220bbbc2bd6db589552\",\"RulesetOverrideID\":\"id1\",\"SampleInterval\":1,\"SourceASN\":1500,\"SourceASNDescription\":\"Source ASN Description\",\"SourceCountry\":\"AD\",\"SourceGeoHash\":\"gbuun\",\"SourcePort\":5678,\"TCPAcknowledgementNumber\":1000,\"TCPChecksum\":10,\"TCPDataOffset\":0,\"TCPFlags\":1,\"TCPFlagsString\":\"Human-readable flags string\",\"TCPMss\":512,\"TCPOptions\":\"mss\",\"TCPSackBlocks\":1,\"TCPSacksPermitted\":1,\"TCPSequenceNumber\":100,\"TCPTimestampEcr\":100,\"TCPTimestampValue\":100,\"TCPUrgentPointer\":10,\"TCPWindowScale\":10,\"TCPWindowSize\":10,\"UDPChecksum\":10,\"UDPPayloadLength\":10,\"Verdict\":\"pass\"}", "outcome": "success", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "network_analytics.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/network_analytics.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { + "community_id": "1:FRJ1/ev5eXNHmcCgyev/H8HrvH4=", "direction": "ingress", "transport": "tcp" }, @@ -3604,6 +3579,9 @@ An example event for `network_analytics` looks as following: "hash": [ "gbuun" ], + "hosts": [ + "SJC" + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -3617,12 +3595,24 @@ An example event for `network_analytics` looks as following: "number": 1500 }, "ip": "67.43.156.0", - "port": 0 + "port": 5678 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-network_analytics" - ] + ], + "threat": { + "group": { + "id": "xyz987" + }, + "indicator": { + "id": [ + "abc777" + ] + } + } } ``` @@ -3630,14 +3620,13 @@ An example event for `network_analytics` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.network_analytics.attack.campaign.id | Unique identifier of the attack campaign that this packet was a part of, if any. | keyword | | cloudflare_logpush.network_analytics.attack.id | Unique identifier of the mitigation that matched the packet, if any. | keyword | | cloudflare_logpush.network_analytics.attack.vector | Descriptive name of the type of attack that this packet was a part of, if any. Only for packets matching rules contained within the Cloudflare L3/4 managed ruleset. | keyword | @@ -3657,15 +3646,17 @@ An example event for `network_analytics` looks as following: | cloudflare_logpush.network_analytics.destination.ip | Value of the Destination Address header field in the IPv4 or IPv6 packet. | ip | | cloudflare_logpush.network_analytics.destination.port | Value of the Destination Port header field in the TCP or UDP packet. | long | | cloudflare_logpush.network_analytics.direction | The direction in relation to customer network. | keyword | +| cloudflare_logpush.network_analytics.dns.query.name | The DNS query name (domain) if the packet is a DNS query. | keyword | +| cloudflare_logpush.network_analytics.dns.query.type | The DNS query type if the packet is a DNS query. | keyword | | cloudflare_logpush.network_analytics.gre.checksum | Value of the Checksum header field in the GRE packet. | long | | cloudflare_logpush.network_analytics.gre.ether.type | Value of the Ethertype header field in the GRE packet. | long | | cloudflare_logpush.network_analytics.gre.header.length | Length of the GRE packet header, in bytes. | long | | cloudflare_logpush.network_analytics.gre.key | Value of the Key header field in the GRE packet. | long | | cloudflare_logpush.network_analytics.gre.sequence.number | Value of the Sequence Number header field in the GRE packet. | long | | cloudflare_logpush.network_analytics.gre.version | Value of the Version header field in the GRE packet. | long | -| cloudflare_logpush.network_analytics.icmp.checksum | Value of the Checksum header field in the ICMP packet | long | -| cloudflare_logpush.network_analytics.icmp.code | Value of the Code header field in the ICMP packet | long | -| cloudflare_logpush.network_analytics.icmp.type | Value of the Type header field in the ICMP packet | long | +| cloudflare_logpush.network_analytics.icmp.checksum | Value of the Checksum header field in the ICMP packet. | long | +| cloudflare_logpush.network_analytics.icmp.code | Value of the Code header field in the ICMP packet. | long | +| cloudflare_logpush.network_analytics.icmp.type | Value of the Type header field in the ICMP packet. | long | | cloudflare_logpush.network_analytics.ip.destination.subnet | Computed subnet of the Destination Address header field in the IPv4 or IPv6 packet. | keyword | | cloudflare_logpush.network_analytics.ip.fragment.offset | Value of the Fragment Offset header field in the IPv4 or IPv6 packet. | long | | cloudflare_logpush.network_analytics.ip.header.length | Length of the IPv4 or IPv6 packet header, in bytes. | long | @@ -3692,6 +3683,7 @@ An example event for `network_analytics` looks as following: | cloudflare_logpush.network_analytics.mitigation.scope | Whether the packet matched a local or global mitigation, if any. | keyword | | cloudflare_logpush.network_analytics.mitigation.system | Which Cloudflare system dropped the packet, if any. | keyword | | cloudflare_logpush.network_analytics.outcome | The action that Cloudflare systems took on the packet. | keyword | +| cloudflare_logpush.network_analytics.pfp_custom_tag | The custom network analytics tag set by Programmable Flow Protection. | long | | cloudflare_logpush.network_analytics.protocol_state | State of the packet in the context of the protocol, if any. | keyword | | cloudflare_logpush.network_analytics.rule.id | Unique identifier of the rule contained with the Cloudflare L3/4 managed ruleset that this packet matched, if any. | text | | cloudflare_logpush.network_analytics.rule.name | Human-readable name of the rule contained within the Cloudflare L3/4 managed ruleset that this packet matched, if any. | text | @@ -3725,18 +3717,15 @@ An example event for `network_analytics` looks as following: | cloudflare_logpush.network_analytics.udp.checksum | Value of the Checksum header field in the UDP packet. | long | | cloudflare_logpush.network_analytics.udp.payload_length | Value of the Payload Length header field in the UDP packet. | long | | cloudflare_logpush.network_analytics.verdict | The action that Cloudflare systems think should be taken on the packet (pass | drop). | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -3754,19 +3743,19 @@ An example event for `network_session` looks as following: { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "ba4a058c-c545-42a7-a7fa-e33b4ff32fda", - "id": "70c5474a-4b38-40f2-a354-bf18bcdc45c9", - "name": "elastic-agent-55862", + "ephemeral_id": "2944bb96-75f0-470e-b5e7-15a6cbce31e6", + "id": "5f315a59-b129-4ec3-9025-50eac9ca468f", + "name": "elastic-agent-59563", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "network_session": { "account_id": "e1836771179f98aabb828da5ea69a111", "destination": { + "bytes": 679, + "ip": "89.160.20.129", + "port": 80, "tunnel_id": "00000000-0000-0000-0000-000000000000" }, "egress": { @@ -3778,6 +3767,10 @@ An example event for `network_session` looks as following: "name": "Egress Rule 1" } }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff", + "name": "zt-test-vm1" + }, "ingress": { "colo_name": "ORD" }, @@ -3785,8 +3778,16 @@ An example event for `network_session` looks as following: "rule_evaluation": { "time_ms": 10 }, + "session": { + "end": "2023-05-04T11:29:14.000Z", + "id": "18881f179300007fb0d06d6400000001", + "start": "2023-05-04T11:29:14.000Z" + }, "source": { - "internal_ip": "1.128.0.1" + "bytes": 2333, + "internal_ip": "1.128.0.1", + "ip": "67.43.156.2", + "port": 52994 }, "tcp": { "client": { @@ -3797,6 +3798,7 @@ An example event for `network_session` looks as following: "reuse": false } }, + "timestamp": "2023-05-04T11:29:14.000Z", "tls": { "client": { "cipher": "TLS_AES_128_GCM_SHA256", @@ -3805,18 +3807,27 @@ An example event for `network_session` looks as following: }, "server": { "certificate": { + "issuer": "DigiCert Inc", "validation_result": "VALID" }, "cipher": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "handshake_time_ms": 130, "version": "TLS 1.2" } + }, + "transport": "TCP", + "user": { + "email": "user@test.com", + "id": "166befbb-00e3-5e20-bd6e-27245723949f" + }, + "vlan": { + "id": "0ce99869-63d3-4d5d-bdaf-d4f33df964aa" } } }, "data_stream": { "dataset": "cloudflare_logpush.network_session", - "namespace": "60780", + "namespace": "16918", "type": "logs" }, "destination": { @@ -3842,11 +3853,14 @@ An example event for `network_session` looks as following: "ip": "89.160.20.129", "port": 80 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "70c5474a-4b38-40f2-a354-bf18bcdc45c9", + "id": "5f315a59-b129-4ec3-9025-50eac9ca468f", "snapshot": false, "version": "8.17.1" }, @@ -3857,38 +3871,23 @@ An example event for `network_session` looks as following: "session" ], "dataset": "cloudflare_logpush.network_session", + "duration": 0, "end": "2023-05-04T11:29:14.000Z", "id": "18881f179300007fb0d06d6400000001", - "ingested": "2025-12-12T09:58:37Z", + "ingested": "2026-04-07T10:30:21Z", "kind": "event", + "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"BytesReceived\":679,\"BytesSent\":2333,\"ClientTCPHandshakeDurationMs\":12,\"ClientTLSCipher\":\"TLS_AES_128_GCM_SHA256\",\"ClientTLSHandshakeDurationMs\":125,\"ClientTLSVersion\":\"TLS 1.3\",\"ConnectionCloseReason\":\"CLIENT_CLOSED\",\"ConnectionReuse\":false,\"DestinationTunnelID\":\"00000000-0000-0000-0000-000000000000\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"EgressColoName\":\"ORD\",\"EgressIP\":\"2a02:cf40::23\",\"EgressPort\":41052,\"EgressRuleID\":\"00000000-0000-0000-0000-000000000000\",\"EgressRuleName\":\"Egress Rule 1\",\"Email\":\"user@test.com\",\"IngressColoName\":\"ORD\",\"Offramp\":\"INTERNET\",\"OriginIP\":\"89.160.20.129\",\"OriginPort\":80,\"OriginTLSCertificateIssuer\":\"DigiCert Inc\",\"OriginTLSCertificateValidationResult\":\"VALID\",\"OriginTLSCipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"OriginTLSHandshakeDurationMs\":130,\"OriginTLSVersion\":\"TLS 1.2\",\"Protocol\":\"TCP\",\"RuleEvaluationDurationMs\":10,\"SessionEndTime\":\"2023-05-04T11:29:14Z\",\"SessionID\":\"18881f179300007fb0d06d6400000001\",\"SessionStartTime\":\"2023-05-04T11:29:14Z\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"1.128.0.1\",\"SourcePort\":52994,\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\",\"VirtualNetworkID\":\"0ce99869-63d3-4d5d-bdaf-d4f33df964aa\"}", "start": "2023-05-04T11:29:14.000Z", "type": [ "connection" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "network_session.log" - } - } - }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", "name": "zt-test-vm1" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/network_session.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "TCP", @@ -3904,7 +3903,8 @@ An example event for `network_session` looks as following: "ip": [ "67.43.156.2", "89.160.20.129", - "2a02:cf40::23" + "2a02:cf40::23", + "1.128.0.1" ], "user": [ "166befbb-00e3-5e20-bd6e-27245723949f", @@ -3929,10 +3929,13 @@ An example event for `network_session` looks as following: "port": 52994 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-network_session" ], "tls": { + "cipher": "TLS_AES_128_GCM_SHA256", "server": { "issuer": "DigiCert Inc" } @@ -3948,14 +3951,13 @@ An example event for `network_session` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.network_session.account_id | Cloudflare account ID. | keyword | | cloudflare_logpush.network_session.destination.bytes | The number of bytes sent from the origin to the client during the network session. | long | | cloudflare_logpush.network_session.destination.ip | The IP of the destination (origin) for the network session. | ip | @@ -3970,11 +3972,15 @@ An example event for `network_session` looks as following: | cloudflare_logpush.network_session.host.id | Identifier of the client device which initiated the network session, if applicable. | keyword | | cloudflare_logpush.network_session.host.name | Name of the client device which initiated the network session, if applicable. | keyword | | cloudflare_logpush.network_session.ingress.colo_name | The name of the Cloudflare colo to which traffic ingressed. | keyword | +| cloudflare_logpush.network_session.initial_origin_ip | The IP used to correlate FQDN matching between Gateway DNS and proxy. | ip | | cloudflare_logpush.network_session.offramp | The type of destination to which the network session was routed. | keyword | +| cloudflare_logpush.network_session.registration_id | Identifier of the client registration. | keyword | +| cloudflare_logpush.network_session.resolved_fqdn | The fully qualified domain name of the destination. | keyword | | cloudflare_logpush.network_session.rule_evaluation.time_ms | The duration taken by Secure Web Gateway applying applicable Network, HTTP, and Egress rules to the network session in milliseconds. | long | | cloudflare_logpush.network_session.session.end | The network session end timestamp with nanosecond precision. | date | | cloudflare_logpush.network_session.session.id | The identifier of this network session. | keyword | | cloudflare_logpush.network_session.session.start | The network session start timestamp with nanosecond precision. | date | +| cloudflare_logpush.network_session.sni | The server name indication (SNI) value from the TLS handshake. | keyword | | cloudflare_logpush.network_session.source.bytes | The number of bytes sent from the client to the origin during the network session. | long | | cloudflare_logpush.network_session.source.internal_ip | Local LAN IP of the device. Only available when connected via a GRE/IPsec tunnel on-ramp. | ip | | cloudflare_logpush.network_session.source.ip | Source IP of the network session. | ip | @@ -3995,18 +4001,15 @@ An example event for `network_session` looks as following: | cloudflare_logpush.network_session.user.email | Email address associated with the user identity which initiated the network session. | keyword | | cloudflare_logpush.network_session.user.id | User identity where the network session originated from. | keyword | | cloudflare_logpush.network_session.vlan.id | Identifier of the virtual network configured for the client. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -4024,35 +4027,36 @@ An example event for `page_shield_events` looks as following: { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "1dfb269c-1b6e-443c-8dd2-2cc980f787ff", - "id": "a717a65c-0d5d-44b7-ad3e-8459415f563c", - "name": "elastic-agent-55379", + "ephemeral_id": "ca737b17-cac2-420e-9d87-a75a0ba2f486", + "id": "e5093573-263c-47e2-a874-2d65a7a582ff", + "name": "elastic-agent-41800", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "page_shield_events": { + "action": "log", "csp_directive": "directive", + "host": "hostymchost.face", "page_url": "http://example.com/?query=42", "policy_id": "9", "resource_type": "other", + "timestamp": "2023-05-04T11:29:14.000Z", + "url": "https://example.com/?query=hog", "url_contains_cdn_cgi_path": true, "url_host": "example.com" } }, "data_stream": { "dataset": "cloudflare_logpush.page_shield_events", - "namespace": "48764", + "namespace": "91602", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "a717a65c-0d5d-44b7-ad3e-8459415f563c", + "id": "e5093573-263c-47e2-a874-2d65a7a582ff", "snapshot": false, "version": "8.17.1" }, @@ -4063,41 +4067,28 @@ An example event for `page_shield_events` looks as following: "network" ], "dataset": "cloudflare_logpush.page_shield_events", - "ingested": "2025-12-12T10:04:57Z", + "ingested": "2026-04-07T10:31:19Z", "kind": "event", + "original": "{\"Action\":\"log\",\"CSPDirective\":\"directive\",\"Host\":\"hostymchost.face\",\"PageURL\":\"http://example.com/?query=42\",\"PolicyID\":\"9\",\"ResourceType\":\"other\",\"Timestamp\":\"2023-05-04T11:29:14Z\",\"URL\":\"https://example.com/?query=hog\",\"URLContainsCDNCGIPath\":true,\"URLHost\":\"example.com\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "page_shield_events.log" - } - } - }, "host": { "name": "hostymchost.face" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/page_shield_events.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hosts": [ - "hostymchost.face" + "hostymchost.face", + "example.com" ] }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-page_shield_events" ], @@ -4115,14 +4106,13 @@ An example event for `page_shield_events` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.page_shield_events.action | The action which was taken against the violation. Possible values are (log, allow). | keyword | | cloudflare_logpush.page_shield_events.csp_directive | The violated directive in the report. | keyword | | cloudflare_logpush.page_shield_events.host | The host where the resource was seen. | keyword | @@ -4134,18 +4124,15 @@ An example event for `page_shield_events` looks as following: | cloudflare_logpush.page_shield_events.url | The resource URL. | keyword | | cloudflare_logpush.page_shield_events.url_contains_cdn_cgi_path | Whether the resource URL contains the CDN-CGI path. (deprecated by Cloudflare) | boolean | | cloudflare_logpush.page_shield_events.url_host | The domain host of the URL. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -4163,33 +4150,52 @@ An example event for `sinkhole_http` looks as following: { "@timestamp": "2023-09-19T12:00:00.000Z", "agent": { - "ephemeral_id": "fe6f8be2-37c0-4f74-83b6-d6061fff286d", - "id": "97a57f01-e27c-4ad1-87e6-ef4b6d5e1e0e", - "name": "elastic-agent-25658", + "ephemeral_id": "6db1a86d-5bdd-4757-b7ec-ef46ba70549e", + "id": "334188ef-543b-4b19-a898-ff614abf72dc", + "name": "elastic-agent-88294", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "sinkhole_http": { "account_id": "AC123456", + "destination": { + "ip": "89.160.20.129" + }, + "host": { + "name": "example.com" + }, "request": { + "body": { + "bytes": 39, + "content": "{\"action\": \"login\", \"user\": \"john_doe\"}" + }, "headers": [ "Host: example.com", "User-Agent: Mozilla/5.0", "Accept: */*", "Connection: keep-alive" ], - "password": "password123" + "method": "POST", + "password": "password123", + "referrer": "https://searchengine.com/", + "uri": "/api/v1/login", + "url": "https://example.com/api/v1/login" + }, + "sinkhole_id": "SH001", + "source": { + "ip": "67.43.156.2" }, - "sinkhole_id": "SH001" + "timestamp": "2023-09-19T12:00:00.000Z", + "user": { + "name": "john_doe" + }, + "user_agent": "Mozilla/5.0" } }, "data_stream": { "dataset": "cloudflare_logpush.sinkhole_http", - "namespace": "95284", + "namespace": "31136", "type": "logs" }, "destination": { @@ -4214,10 +4220,10 @@ An example event for `sinkhole_http` looks as following: "ip": "89.160.20.129" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "97a57f01-e27c-4ad1-87e6-ef4b6d5e1e0e", + "id": "334188ef-543b-4b19-a898-ff614abf72dc", "snapshot": false, "version": "8.17.1" }, @@ -4227,23 +4233,13 @@ An example event for `sinkhole_http` looks as following: "network" ], "dataset": "cloudflare_logpush.sinkhole_http", - "ingested": "2025-12-12T10:10:56Z", + "ingested": "2026-04-07T10:32:21Z", "kind": "event", + "original": "{\"AccountID\":\"AC123456\",\"Body\":\"{\\\"action\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\"}\",\"BodyLength\":39,\"DestAddr\":\"89.160.20.129\",\"Headers\":\"Host: example.com\\nUser-Agent: Mozilla/5.0\\nAccept: */*\\nConnection: keep-alive\",\"Host\":\"example.com\",\"Method\":\"POST\",\"Password\":\"password123\",\"R2Path\":\"\",\"Referrer\":\"https://searchengine.com/\",\"SinkholeID\":\"SH001\",\"SrcAddr\":\"67.43.156.2\",\"Timestamp\":\"2023-09-19T12:00:00Z\",\"URI\":\"/api/v1/login\",\"URL\":\"https://example.com/api/v1/login\",\"UserAgent\":\"Mozilla/5.0\",\"Username\":\"john_doe\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "sinkhole_http.log" - } - } - }, "host": { "name": "example.com" }, @@ -4258,13 +4254,7 @@ An example event for `sinkhole_http` looks as following: } }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/sinkhole_http.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hosts": [ @@ -4294,6 +4284,8 @@ An example event for `sinkhole_http` looks as following: "ip": "67.43.156.2" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-sinkhole_http" ], @@ -4320,14 +4312,13 @@ An example event for `sinkhole_http` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.sinkhole_http.account_id | The Account ID. | keyword | | cloudflare_logpush.sinkhole_http.destination.ip | The destination IP address of the request. | ip | | cloudflare_logpush.sinkhole_http.host.name | The host the request was sent to. | keyword | @@ -4345,18 +4336,15 @@ An example event for `sinkhole_http` looks as following: | cloudflare_logpush.sinkhole_http.timestamp | The date and time the sinkhole HTTP request was logged. | date | | cloudflare_logpush.sinkhole_http.user.name | The request username. | keyword | | cloudflare_logpush.sinkhole_http.user_agent | The request user agent. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -4374,20 +4362,23 @@ An example event for `spectrum_event` looks as following: { "@timestamp": "2022-05-26T09:24:00.000Z", "agent": { - "ephemeral_id": "1e145e40-ba54-4666-86af-b8bbda38bfa6", - "id": "e7a01d20-2fff-4feb-bb44-7361b6bc648a", - "name": "elastic-agent-97143", + "ephemeral_id": "e7abae94-0044-4900-9c4b-48c2505ae84c", + "id": "2738c1bb-9aa7-4f2f-a704-fb666fbd0dc1", + "name": "elastic-agent-87759", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "spectrum_event": { "action": "connect", + "application": "7ef659a2f8ef4810a9bade96fdad7c75", "client": { + "asn": 200391, + "bytes": 0, + "country": "bg", + "ip": "67.43.156.0", "matched_ip_firewall": "UNKNOWN", + "port": 40456, "protocol": "tcp", "tcp_rtt": 0, "tls": { @@ -4408,6 +4399,9 @@ An example event for `spectrum_event` looks as following: }, "ip_firewall": false, "origin": { + "bytes": 0, + "ip": "175.16.199.0", + "port": 3389, "protocol": "tcp", "tcp_rtt": 0, "tls": { @@ -4420,12 +4414,14 @@ An example event for `spectrum_event` looks as following: }, "proxy": { "protocol": "off" - } + }, + "status": 0, + "timestamp": "2022-05-26T09:24:00.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.spectrum_event", - "namespace": "19575", + "namespace": "32034", "type": "logs" }, "destination": { @@ -4434,10 +4430,10 @@ An example event for `spectrum_event` looks as following: "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "e7a01d20-2fff-4feb-bb44-7361b6bc648a", + "id": "2738c1bb-9aa7-4f2f-a704-fb666fbd0dc1", "snapshot": false, "version": "8.17.1" }, @@ -4448,45 +4444,33 @@ An example event for `spectrum_event` looks as following: "network" ], "dataset": "cloudflare_logpush.spectrum_event", + "duration": -1653557040000000000, "end": "1970-01-01T00:00:00.000Z", "id": "7ef659a2f8ef4810a9bade96fdad7c75", - "ingested": "2025-12-12T10:17:07Z", + "ingested": "2026-04-07T10:33:21Z", "kind": "event", + "original": "{\"Application\":\"7ef659a2f8ef4810a9bade96fdad7c75\",\"ClientAsn\":200391,\"ClientBytes\":0,\"ClientCountry\":\"bg\",\"ClientIP\":\"67.43.156.0\",\"ClientMatchedIpFirewall\":\"UNKNOWN\",\"ClientPort\":40456,\"ClientProto\":\"tcp\",\"ClientTcpRtt\":0,\"ClientTlsCipher\":\"UNK\",\"ClientTlsClientHelloServerName\":\"server name\",\"ClientTlsProtocol\":\"unknown\",\"ClientTlsStatus\":\"UNKNOWN\",\"ColoCode\":\"SOF\",\"ConnectTimestamp\":\"2022-05-26T09:24:00Z\",\"DisconnectTimestamp\":\"1970-01-01T00:00:00Z\",\"Event\":\"connect\",\"IpFirewall\":false,\"OriginBytes\":0,\"OriginIP\":\"175.16.199.0\",\"OriginPort\":3389,\"OriginProto\":\"tcp\",\"OriginTcpRtt\":0,\"OriginTlsCipher\":\"UNK\",\"OriginTlsFingerprint\":\"0000000000000000000000000000000000000000000000000000000000000000.\",\"OriginTlsMode\":\"off\",\"OriginTlsProtocol\":\"unknown\",\"OriginTlsStatus\":\"UNKNOWN\",\"ProxyProtocol\":\"off\",\"Status\":0,\"Timestamp\":\"2022-05-26T09:24:00Z\"}", "start": "2022-05-26T09:24:00.000Z", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "spectrum_event.log" - } - } - }, "http": { "response": { "status_code": 0 } }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/spectrum_event.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "community_id": "1:X7lywUVKlduqRq5SyCRaBj4hLP0=", "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -4504,9 +4488,14 @@ An example event for `spectrum_event` looks as following: "port": 40456 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-spectrum_event" - ] + ], + "tls": { + "cipher": "UNK" + } } ``` @@ -4514,14 +4503,13 @@ An example event for `spectrum_event` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | | cloudflare_logpush.spectrum_event.action | Event Action. | keyword | | cloudflare_logpush.spectrum_event.application | The unique public ID of the application on which the event occurred. | keyword | | cloudflare_logpush.spectrum_event.client.asn | Client AS number. | long | @@ -4553,18 +4541,15 @@ An example event for `spectrum_event` looks as following: | cloudflare_logpush.spectrum_event.proxy.protocol | Which form of proxy protocol is applied to the given connection. | keyword | | cloudflare_logpush.spectrum_event.status | A code indicating reason for connection closure. | long | | cloudflare_logpush.spectrum_event.timestamp | Timestamp at which the event took place. | date | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -4582,18 +4567,25 @@ An example event for `workers_trace` looks as following: { "@timestamp": "2023-07-20T11:35:46.804Z", "agent": { - "ephemeral_id": "7be35875-0716-4185-a715-56f4262e6677", - "id": "5de705c9-500f-4f23-b85e-94dfb165569e", - "name": "elastic-agent-35180", + "ephemeral_id": "a01887ca-67a7-4a33-819e-cc0cb6df6fbb", + "id": "6016b130-b61b-4830-aef9-3d270a4d8f09", + "name": "elastic-agent-70540", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "workers_trace": { "dispatch_namespace": "my-worker-dispatch", + "event": { + "ray_id": "7e9ae7157ac0c33a", + "request": { + "method": "GET", + "url": "http://chat-gpt-little-butterfly-0c3d.example.workers.dev/v2/_catalog" + }, + "response": { + "status": 404 + } + }, "exceptions": [ { "message": "Uncaught TypeError: Cannot read property 'x' of undefined", @@ -4610,25 +4602,28 @@ An example event for `workers_trace` looks as following: "message": "Something went wrong" } ], + "outcome": "exception", "script": { "name": "chat-gpt-little-butterfly-0c3d", "tags": [ "api", "chatgpt" ] - } + }, + "timestamp": "2023-07-20T11:35:46.804Z", + "type": "fetch" } }, "data_stream": { "dataset": "cloudflare_logpush.workers_trace", - "namespace": "22099", + "namespace": "66050", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "5de705c9-500f-4f23-b85e-94dfb165569e", + "id": "6016b130-b61b-4830-aef9-3d270a4d8f09", "snapshot": false, "version": "8.17.1" }, @@ -4640,25 +4635,15 @@ An example event for `workers_trace` looks as following: ], "dataset": "cloudflare_logpush.workers_trace", "id": "7e9ae7157ac0c33a", - "ingested": "2025-12-12T10:23:03Z", + "ingested": "2026-04-07T10:34:20Z", "kind": "event", + "original": "{\"DispatchNamespace\":\"my-worker-dispatch\",\"Event\":{\"RayID\":\"7e9ae7157ac0c33a\",\"Request\":{\"Method\":\"GET\",\"URL\":\"http://chat-gpt-little-butterfly-0c3d.example.workers.dev/v2/_catalog\"},\"Response\":{\"Status\":404}},\"EventTimestampMs\":1689852946804,\"EventType\":\"fetch\",\"Exceptions\":[{\"Message\":\"Uncaught TypeError: Cannot read property 'x' of undefined\",\"Stack\":\"TypeError: Cannot read property 'x' of undefined\\n at fetchHandler (/workers/script.js:12:27)\\n at handleRequest (/workers/script.js:6:13)\"}],\"Logs\":[{\"level\":\"info\",\"message\":\"Request received for /api/data\"},{\"level\":\"error\",\"message\":\"Something went wrong\"}],\"Outcome\":\"exception\",\"ScriptName\":\"chat-gpt-little-butterfly-0c3d\",\"ScriptTags\":[\"api\",\"chatgpt\"]}", "outcome": "failure", "type": [ "info", "error" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "workers_trace.log" - } - } - }, "http": { "request": { "method": "GET" @@ -4668,15 +4653,11 @@ An example event for `workers_trace` looks as following: } }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/workers_trace.log" - }, - "offset": 0 + "type": "http_endpoint" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-workers_trace" ], @@ -4693,14 +4674,14 @@ An example event for `workers_trace` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | | azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | | azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | | azure.storage.container.name | The name of the Azure Blob Storage container | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloudflare_logpush.workers_trace.cpu_time_ms | The amount of CPU time used by the Worker script, in milliseconds. | long | | cloudflare_logpush.workers_trace.dispatch_namespace | The Cloudflare Worker dispatch namespace. | keyword | | cloudflare_logpush.workers_trace.entrypoint | The name of the entrypoint class in which the Worker began execution. | keyword | | cloudflare_logpush.workers_trace.event | Details about the source event. | flattened | @@ -4712,18 +4693,16 @@ An example event for `workers_trace` looks as following: | cloudflare_logpush.workers_trace.script.version | The version of the script that was invoked. | flattened | | cloudflare_logpush.workers_trace.timestamp | The timestamp of when the event was received. | date | | cloudflare_logpush.workers_trace.type | The event type that triggered the invocation. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset. | constant_keyword | -| event.module | Event module. | constant_keyword | +| cloudflare_logpush.workers_trace.wall_time_ms | The elapsed wall-clock time in milliseconds for the Worker invocation. | long | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | | gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | | gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | | gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | diff --git a/packages/cloudflare_logpush/manifest.yml b/packages/cloudflare_logpush/manifest.yml index e0967576bf1..21ab097008c 100644 --- a/packages/cloudflare_logpush/manifest.yml +++ b/packages/cloudflare_logpush/manifest.yml @@ -1,7 +1,7 @@ -format_version: "3.0.2" +format_version: "3.3.2" name: cloudflare_logpush title: Cloudflare Logpush -version: "1.43.3" +version: "1.44.0" description: Collect and parse logs from Cloudflare API with Elastic Agent. type: integration categories: From d58039349ff2930bd5a6d0714ffa21678e81e4ed Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:18:44 +0530 Subject: [PATCH 02/14] sort fields.yml alphabetically, add new fields, and fix field descriptions Sort all fields.yml entries alphabetically across all data streams for better maintainability. Add support for new fields across device_posture, gateway_dns, gateway_http, gateway_network, http_request, network_analytics, network_session, workers_trace, firewall_event, and email_security_alerts data streams. Fix swapped field descriptions for firewall_event (origin.ray.id/origin.response.status) and http_request (cache.status/cache.response.status). Align header naming between pipeline and fields.yml (singular header to plural headers for RequestHeaders/ResponseHeaders in http_request). --- .../access_request/fields/fields.yml | 1 - .../data_stream/casb/fields/fields.yml | 1 - .../device_posture/fields/fields.yml | 62 ++++---- .../dns_firewall/fields/fields.yml | 1 - .../email_security_alerts/fields/fields.yml | 6 +- .../firewall_event/fields/fields.yml | 16 ++- .../data_stream/gateway_dns/fields/fields.yml | 135 ++++++++++++------ .../gateway_http/fields/fields.yml | 101 +++++++------ .../gateway_network/fields/fields.yml | 54 +++---- .../http_request/fields/fields.yml | 58 ++++++-- .../data_stream/magic_ids/fields/fields.yml | 1 - .../network_analytics/fields/fields.yml | 27 +++- .../network_session/fields/fields.yml | 125 ++++++++-------- .../sinkhole_http/fields/fields.yml | 1 - .../workers_trace/fields/fields.yml | 7 +- 15 files changed, 363 insertions(+), 233 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/access_request/fields/fields.yml b/packages/cloudflare_logpush/data_stream/access_request/fields/fields.yml index 50eca66d75b..51dfe844e8d 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/access_request/fields/fields.yml @@ -1,6 +1,5 @@ - name: cloudflare_logpush.access_request type: group - description: Integration for parsing Cloudflare Access Request logs. fields: - name: timestamp type: date diff --git a/packages/cloudflare_logpush/data_stream/casb/fields/fields.yml b/packages/cloudflare_logpush/data_stream/casb/fields/fields.yml index 2945cceb888..e88ce0ebc42 100644 --- a/packages/cloudflare_logpush/data_stream/casb/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/casb/fields/fields.yml @@ -1,6 +1,5 @@ - name: cloudflare_logpush.casb type: group - description: Integration for parsing Cloudflare CASB Findings logs. fields: - name: timestamp type: date diff --git a/packages/cloudflare_logpush/data_stream/device_posture/fields/fields.yml b/packages/cloudflare_logpush/data_stream/device_posture/fields/fields.yml index 8638369dad8..47468216ba5 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/device_posture/fields/fields.yml @@ -1,31 +1,33 @@ - name: cloudflare_logpush.device_posture type: group - description: Integration for parsing Cloudflare Device Posture Results logs. fields: - - name: timestamp - type: date - description: The date and time the corresponding device posture upload was performed. - - name: version - type: keyword - description: The Zero Trust client version at the time of upload. + - name: eval + type: group + fields: + - name: expected + type: flattened + description: JSON object of what the posture check expects from the Zero Trust client. + - name: received + type: flattened + description: JSON object of what the Zero Trust client actually uploads. + - name: result + type: boolean + description: Whether this posture upload passes the associated posture check, given the requirements posture check at the time of the timestamp. - name: host type: group fields: - name: id type: keyword description: The device ID that performed the posture upload. - - name: name - type: keyword - description: The name of the device that the Zero Trust client is running on. - name: manufacturer type: keyword description: The manufacturer of the device that the Zero Trust client is running on. - name: model type: keyword description: The model of the device that the Zero Trust client is running on. - - name: serial + - name: name type: keyword - description: The serial number of the device that the Zero Trust client is running on. + description: The name of the device that the Zero Trust client is running on. - name: os type: group fields: @@ -35,39 +37,39 @@ - name: version type: keyword description: The operating system version at the time of upload. + - name: serial + type: keyword + description: The serial number of the device that the Zero Trust client is running on. + - name: registration_id + type: keyword + description: The UUID of the device registration associated with this posture result. - name: rule type: group fields: + - name: category + type: keyword + description: The type of the Zero Trust client check or service provider check. - name: id type: keyword description: The posture check ID associated with this device posture result. - name: name type: keyword description: The name of the posture check associated with this device posture result. - - name: category - type: keyword - description: The type of the Zero Trust client check or service provider check. - - name: eval - type: group - fields: - - name: result - type: boolean - description: Whether this posture upload passes the associated posture check, given the requirements posture check at the time of the timestamp. - - name: expected - type: flattened - description: JSON object of what the posture check expects from the Zero Trust client. - - name: received - type: flattened - description: JSON object of what the Zero Trust client actually uploads. + - name: timestamp + type: date + description: The date and time the corresponding device posture upload was performed. - name: user type: group fields: - - name: id - type: keyword - description: The uid of the user who registered the device. - name: email type: keyword description: The email used to register the device with the Zero Trust client. + - name: id + type: keyword + description: The uid of the user who registered the device. + - name: version + type: keyword + description: The Zero Trust client version at the time of upload. - name: log.source.address type: keyword description: Source address from which the log event was read / sent from. diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/fields.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/fields.yml index 937f43773c3..df5c09aef32 100644 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/fields.yml @@ -1,6 +1,5 @@ - name: cloudflare_logpush.dns_firewall type: group - description: Integration for parsing Cloudflare Logpush DNS Firewall logs. fields: - name: cluster_id type: keyword diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/fields.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/fields.yml index 3ff2e2457cc..9f054e5b3f6 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/fields/fields.yml @@ -10,12 +10,12 @@ - name: attachments type: group fields: - - name: Encrypted - type: boolean - description: Whether the attachment was encrypted. - name: Decrypted type: boolean description: Whether the attachment was decrypted. + - name: Encrypted + type: boolean + description: Whether the attachment was encrypted. - name: "*" type: keyword description: Metadata of attachments contained in this message. diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml index 674913a9ec2..5819e4c6494 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml @@ -94,6 +94,12 @@ - name: status type: long description: HTTP response status code returned to browser. + - name: fraud + type: group + fields: + - name: user_id + type: keyword + description: A unique identifier generated by the Fraud Detection system for each user. - name: kind type: keyword description: The kind of event, currently only possible values are. @@ -115,13 +121,13 @@ fields: - name: id type: keyword - description: HTTP origin response status code returned to browser. + description: The RayID of the request that issued the challenge/jschallenge. - name: response type: group fields: - name: status type: long - description: The RayID of the request that issued the challenge/jschallenge. + description: HTTP origin response status code returned to browser. - name: ray type: group fields: @@ -134,12 +140,12 @@ - name: rule type: group fields: - - name: id - type: keyword - description: The Cloudflare security product-specific RuleID triggered by this request. - name: description type: keyword description: The Cloudflare security product-specific Description of the rule triggered by this request. + - name: id + type: keyword + description: The Cloudflare security product-specific RuleID triggered by this request. - name: source type: keyword description: The Cloudflare security product triggered by this request. diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml index 28eb51ed5e0..a88f885f25a 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml @@ -1,10 +1,12 @@ - name: cloudflare_logpush.gateway_dns type: group - description: Integration for parsing Cloudflare Logpush Gateway DNS logs. fields: - name: account_id type: keyword description: Cloudflare account ID. + - name: answers + type: flattened + description: The response data objects. - name: application_id type: long description: ID of the application the domain belongs to. @@ -14,6 +16,9 @@ - name: authoritative_name_server_ip type: ip description: The IPs of the authoritative nameservers that provided the answers, if any. + - name: cname + type: keyword + description: Resolved intermediate cname domains. - name: cname_category type: group fields: @@ -23,9 +28,6 @@ - name: names type: keyword description: Name or names of category that the intermediate cname domains belongs to. - - name: cname - type: keyword - description: Resolved intermediate cname domains. - name: cname_reversed type: keyword description: Resolved intermediate cname domains in reverse. @@ -41,12 +43,12 @@ - name: custom_resolver type: group fields: - - name: duration_milli - type: long - description: The time it took for the custom resolver to respond in milliseconds. - name: address type: keyword description: IP and port combo used to resolve the custom dns resolver query, if any. + - name: duration_milli + type: long + description: The time it took for the custom resolver to respond in milliseconds. - name: policy type: group fields: @@ -59,18 +61,6 @@ - name: response type: keyword description: Status of the custom resolver response. - - name: timestamp - type: date - description: The date and time the corresponding DNS request was made. - - name: host - type: group - fields: - - name: id - type: keyword - description: UUID of the device where the HTTP request originated from. - - name: name - type: keyword - description: The name of the device where the HTTP request originated from. - name: destination type: group fields: @@ -89,6 +79,15 @@ - name: extended_dns_error_codes type: keyword description: List of returned Extended DNS Error Codes. + - name: host + type: group + fields: + - name: id + type: keyword + description: UUID of the device where the HTTP request originated from. + - name: name + type: keyword + description: The name of the device where the HTTP request originated from. - name: initial_category type: group fields: @@ -98,18 +97,39 @@ - name: names type: keyword description: Name or names of category that the queried domains belongs to. + - name: initial_resolved_ips + type: keyword + description: The IPs used to correlate existing FQDN matching policy between Gateway DNS and Gateway proxy. + - name: internal_dns + type: group + fields: + - name: duration_ms + type: long + description: The time it took for the internal DNS to respond. + - name: fallback_strategy + type: keyword + description: The fallback strategy applied over the internal DNS response. + - name: rcode + type: long + description: The return code sent back by the internal DNS service. + - name: view_id + type: keyword + description: The DNS internal view identifier that was sent to the internal DNS service. + - name: zone_id + type: keyword + description: The DNS zone identifier returned by the internal DNS service. - name: is_response_cached type: boolean description: Response comes from cache or not. - name: location type: group fields: - - name: name - type: keyword - description: Name of the location the DNS request is coming from. - name: id type: keyword description: UUID of the location the DNS request is coming from. + - name: name + type: keyword + description: Name of the location the DNS request is coming from. - name: matched type: group fields: @@ -134,24 +154,27 @@ - name: policy type: group fields: - - name: name - type: keyword - description: Name of the policy that was applied (if any) - name: id type: keyword description: ID of the policy/rule that was applied (if any). + - name: name + type: keyword + description: Name of the policy that was applied (if any). - name: protocol type: keyword description: The protocol used for the DNS query by the client. - name: question type: group fields: - - name: id - type: keyword - description: Globally unique identifier of the query. - - name: name - type: keyword - description: The query name. + - name: application + type: group + fields: + - name: ids + type: long + description: ID or IDs of applications the queried domain belongs to. + - name: names + type: keyword + description: Name or names of applications the queried domain belongs to. - name: category type: group fields: @@ -161,6 +184,9 @@ - name: names type: keyword description: Name or names of category that the domain belongs to. + - name: id + type: keyword + description: Globally unique identifier of the query. - name: indicator_feed type: group fields: @@ -170,6 +196,9 @@ - name: names type: keyword description: Name or names of indicator feed(s) that the domain belongs to. + - name: name + type: keyword + description: The query name. - name: reversed type: keyword description: Query name in reverse. @@ -182,6 +211,21 @@ - name: type_id type: long description: ID of the type of DNS query. + - name: redirect_target_uri + type: keyword + description: Custom URI to which the user was redirected, if any. + - name: registration_id + type: keyword + description: The UUID of the device registration from which the HTTP request originated. + - name: request_context_category + type: group + fields: + - name: ids + type: long + description: ID or IDs of the category that was sent to gateway in the EDNS request for filtering. + - name: names + type: keyword + description: Name or names of the category that was sent to gateway in the EDNS request for filtering. - name: resolved_ip_details type: group fields: @@ -209,33 +253,27 @@ - name: policy type: group fields: - - name: id + - name: ids type: keyword description: Resolver policy UUID, if any matched. - name: names type: keyword description: Resolver policy name, if any matched. + - name: resolver_decision + type: keyword + description: Result of the DNS query. - name: resource_records type: group fields: - - name: object - type: flattened - description: The rdata objects. - name: json type: match_only_text description: String that represents the JSON array with the returned resource records. + - name: object + type: flattened + description: The rdata objects. - name: response_code type: keyword description: The return code sent back by the DNS resolver. - - name: answers - type: flattened - description: The response data objects. - - name: resolved_ip - type: ip - description: The resolved IPs in the response, if any. - - name: resolver_decision - type: keyword - description: Result of the DNS query. - name: source type: group fields: @@ -254,6 +292,9 @@ - name: country_code type: keyword description: Country code of the source IP address making the DNS query. + - name: timestamp + type: date + description: The date and time the corresponding DNS request was made. - name: timezone type: keyword description: Time zone used to calculate the current time, if a matched rule was scheduled with it. @@ -263,12 +304,12 @@ - name: user type: group fields: - - name: id - type: keyword - description: User identity where the HTTP request originated from. - name: email type: keyword description: Email used to authenticate the client. + - name: id + type: keyword + description: User identity where the HTTP request originated from. - name: log.source.address type: keyword description: Source address from which the log event was read / sent from. diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml b/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml index 3375063968d..d3b127378f8 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml @@ -1,6 +1,5 @@ - name: cloudflare_logpush.gateway_http type: group - description: Integration for parsing Cloudflare Logpush Gateway HTTP logs. fields: - name: account_id type: keyword @@ -8,6 +7,9 @@ - name: action type: keyword description: Action performed by gateway on the HTTP request. + - name: app_control_info + type: flattened + description: Information about application control operations, APIs, and groups that matched the HTTP request. - name: application type: group fields: @@ -17,6 +19,9 @@ - name: names type: keyword description: Names of the applications that matched the session parameters. + - name: statuses + type: keyword + description: Statuses of the applications that matched the HTTP request parameters. - name: blocked_file type: group fields: @@ -44,9 +49,6 @@ - name: names type: keyword description: Names of the categories that matched the session parameters. - - name: timestamp - type: date - description: The date and time the corresponding HTTP request was made. - name: destination type: group fields: @@ -74,15 +76,6 @@ - name: profiles type: keyword description: List of matched DLP profiles in the HTTP request. - - name: host - type: group - fields: - - name: id - type: keyword - description: UUID of the device where the HTTP request originated from. - - name: name - type: keyword - description: The name of the device where the HTTP request originated from. - name: downloaded_files type: keyword description: List of files downloaded in the HTTP request. @@ -92,39 +85,27 @@ - name: forensic_copy_status type: keyword description: Status of any associated forensic copies that may have been captured during the request. - - name: request + - name: host type: group fields: - - name: host - type: keyword - description: Content of the host header in the HTTP request. - - name: method - type: keyword - description: HTTP request method. - - name: referrer + - name: id type: keyword - description: Contents of the referer header in the HTTP request. - - name: version + description: UUID of the device where the HTTP request originated from. + - name: name type: keyword - description: Version name for the HTTP request. - - name: response - type: group - fields: - - name: status_code - type: long - description: HTTP status code gateway returned to the user. Zero if nothing was returned. + description: The name of the device where the HTTP request originated from. - name: isolated type: boolean description: If the requested was isolated with Cloudflare Browser Isolation or not. - name: policy type: group fields: - - name: name - type: keyword - description: The name of the gateway policy applied to the request, if any. - name: id type: keyword description: The gateway policy UUID applied to the request, if any. + - name: name + type: keyword + description: The name of the gateway policy applied to the request, if any. - name: private_app_aud type: keyword description: The private app AUD, if any. @@ -132,23 +113,50 @@ type: keyword description: The proxy endpoint used on this network session, if any. - name: quarantined - type: keyword + type: boolean description: If the request content was quarantined. + - name: redirect_target_uri + type: keyword + description: Custom URI to which the user was redirected, if any. + - name: registration_id + type: keyword + description: The UUID of the device registration from which the HTTP request originated. + - name: request + type: group + fields: + - name: host + type: keyword + description: Content of the host header in the HTTP request. + - name: method + type: keyword + description: HTTP request method. + - name: referrer + type: keyword + description: Contents of the referer header in the HTTP request. + - name: version + type: keyword + description: Version name for the HTTP request. - name: request_id type: keyword description: Cloudflare request ID. + - name: response + type: group + fields: + - name: status_code + type: long + description: HTTP status code gateway returned to the user. Zero if nothing was returned. - name: session_id type: keyword description: The session identifier of this network session. - name: source type: group fields: - - name: ip - type: ip - description: Source IP of the request. - name: internal_ip type: ip description: Local LAN IP of the device. Only available when connected via a GRE/IPsec tunnel on-ramp. + - name: ip + type: ip + description: Source IP of the request. - name: port type: long description: Source port of the request. @@ -161,9 +169,9 @@ - name: country_code type: keyword description: Country code of the source IP of the network session. - - name: url - type: keyword - description: HTTP request URL. + - name: timestamp + type: date + description: The date and time the corresponding HTTP request was made. - name: untrusted_certificate_action type: keyword description: Action taken when an untrusted origin certificate error occurs. @@ -179,18 +187,21 @@ - name: uploaded_files type: keyword description: List of files uploaded in the HTTP request. - - name: user_agent + - name: url type: keyword - description: Contents of the user agent header in the HTTP request. + description: HTTP request URL. - name: user type: group fields: - - name: id - type: keyword - description: User identity where the HTTP request originated from. - name: email type: keyword description: Email used to authenticate the client. + - name: id + type: keyword + description: User identity where the HTTP request originated from. + - name: user_agent + type: keyword + description: Contents of the user agent header in the HTTP request. - name: virtual_network type: group fields: diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/fields/fields.yml b/packages/cloudflare_logpush/data_stream/gateway_network/fields/fields.yml index c06a47ae932..1255d518443 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_network/fields/fields.yml @@ -1,6 +1,5 @@ - name: cloudflare_logpush.gateway_network type: group - description: Integration for parsing Cloudflare Logpush Gateway Network logs. fields: - name: account_id type: keyword @@ -26,6 +25,15 @@ - name: names type: keyword description: Names of the categories that matched the session parameters. + - name: destination + type: group + fields: + - name: ip + type: ip + description: Destination IP of the network session. + - name: port + type: long + description: Destination port of the network session. - name: destination_ip type: group fields: @@ -38,18 +46,6 @@ - name: detected_protocol type: keyword description: Detected traffic protocol of the network session. - - name: timestamp - type: date - description: The date and time the corresponding network session was made. - - name: destination - type: group - fields: - - name: ip - type: ip - description: Destination IP of the network session. - - name: port - type: long - description: Destination port of the network session. - name: host type: group fields: @@ -64,37 +60,40 @@ fields: - name: ip type: ip - description: Overriden IP of the network session, if any. + description: Overridden IP of the network session, if any. - name: port type: long - description: Overriden port of the network session, if any. + description: Overridden port of the network session, if any. - name: policy type: group fields: - - name: name - type: keyword - description: The name of the gateway policy applied to the session, if any. - name: id type: keyword description: Identifier of the policy/rule that was applied, if any. + - name: name + type: keyword + description: The name of the gateway policy applied to the session, if any. - name: proxy_endpoint type: keyword description: The proxy endpoint used on this network session, if any. - - name: sni + - name: registration_id type: keyword - description: Content of the SNI (Server Name Indication) for the TLS network session, if any. + description: The UUID of the device registration from which the network session originated. - name: session_id type: keyword description: The session identifier of this network session. + - name: sni + type: keyword + description: Content of the SNI (Server Name Indication) for the TLS network session, if any. - name: source type: group fields: - - name: ip - type: ip - description: Source IP of the network session. - name: internal_ip type: ip description: Local LAN IP of the device. Only available when connected via a GRE/IPsec tunnel on-ramp. + - name: ip + type: ip + description: Source IP of the network session. - name: port type: long description: Source port of the network session. @@ -107,18 +106,21 @@ - name: country_code type: keyword description: Country code of the source IP of the network session. + - name: timestamp + type: date + description: The date and time the corresponding network session was made. - name: transport type: keyword description: Transport protocol used for this session. - name: user type: group fields: + - name: email + type: keyword + description: Email associated with the user identity where the network session originated from. - name: id type: keyword description: User identity where the network session originated from. - - name: email - type: keyword - description: Email associated with the user identity where the network sesion originated from. - name: virtual_network type: group fields: diff --git a/packages/cloudflare_logpush/data_stream/http_request/fields/fields.yml b/packages/cloudflare_logpush/data_stream/http_request/fields/fields.yml index 013ea1aab63..747b2faadbe 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/fields/fields.yml @@ -25,6 +25,9 @@ - name: cache type: group fields: + - name: reserve_used + type: boolean + description: Cache Reserve was used to serve this request. - name: response type: group fields: @@ -33,13 +36,10 @@ description: Number of bytes returned by the cache. - name: status type: long - description: Cache status. + description: HTTP status code returned by the cache to the edge. - name: status type: keyword - description: HTTP status code returned by the cache to the edge. - - name: reserve_used - type: boolean - description: Cache Reserve was used to serve this request. + description: Cache status. - name: tiered_fill type: boolean description: Tiered Cache was used to serve this request. @@ -163,6 +163,9 @@ - name: cookies type: flattened description: String key-value pairs for Cookies. + - name: datetime + type: date + description: Timestamp when the request was received. - name: edge type: group fields: @@ -259,6 +262,24 @@ - name: sources type: keyword description: The firewall products that matched the request. + - name: fraud + type: group + fields: + - name: attack + type: keyword + description: The primary attack or use case detected by Fraud detections. + - name: detection_ids + type: long + description: List of IDs correlating to Fraud detections. + - name: detection_tags + type: keyword + description: List of tags correlating to Fraud detections. + - name: email_risk + type: keyword + description: Risk of a specific email address (low/medium/high). + - name: user_id + type: keyword + description: A unique identifier generated by the Fraud Detection system. - name: ja3_hash type: keyword description: The MD5 hash of the JA3 fingerprint used to profile SSL/TLS clients. @@ -268,6 +289,12 @@ - name: ja4_signals type: flattened description: Inter-request statistics computed for this JA4 fingerprint. JA4Signals field is organized in key:value pairs, where values are numbers. Available only for Bot Management customers. + - name: js_detection_passed + type: keyword + description: Whether the request passed background JS Detection (passed/failed/missing). + - name: leaked_credential_check + type: keyword + description: Result of the check for leaked credentials. - name: origin type: group fields: @@ -340,6 +367,9 @@ - name: id type: keyword description: Ray ID of the parent request if this request was made using a Worker script. + - name: pay_per_crawl_status + type: keyword + description: Pay Per Crawl outcome. - name: ray type: group fields: @@ -379,6 +409,9 @@ - name: id type: long description: The “upper tier” datacenter that was checked for a cached copy if Tiered Cache is used. Available in Logpush v2 only. + - name: verified_bot_category + type: keyword + description: The category of verified bot. - name: waf type: group fields: @@ -418,12 +451,24 @@ - name: xss type: long description: WAF score for a Cross-site scripting (XSS) attack. + - name: web_assets + type: group + fields: + - name: labels_managed + type: keyword + description: Cloudflare-defined labels matched for the request. + - name: operation_id + type: keyword + description: UUID of the matched web asset operation. - name: worker type: group fields: - name: cpu_time type: long description: Amount of time in microseconds spent executing a worker, if any. + - name: script_name + type: keyword + description: The Worker script name that made the request. - name: status type: text description: Status returned from worker daemon. @@ -448,9 +493,6 @@ - name: name type: keyword description: The human-readable name of the zone. - - name: datetime - type: date - description: Timestamp when the request was received - name: log.source.address type: keyword description: Source address from which the log event was read / sent from. diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/fields/fields.yml b/packages/cloudflare_logpush/data_stream/magic_ids/fields/fields.yml index b3e6402bbc5..ae2631c9c38 100644 --- a/packages/cloudflare_logpush/data_stream/magic_ids/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/magic_ids/fields/fields.yml @@ -1,6 +1,5 @@ - name: cloudflare_logpush.magic_ids type: group - description: Integration for parsing Cloudflare Logpush Magic IDS logs. fields: - name: action type: keyword diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/fields/fields.yml b/packages/cloudflare_logpush/data_stream/network_analytics/fields/fields.yml index 0bb5eef98fd..4ac77212afc 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/network_analytics/fields/fields.yml @@ -22,12 +22,12 @@ - name: city type: keyword description: The city where the Cloudflare datacenter that received the packet is located. - - name: country - type: keyword - description: The country of colo that received the packet (ISO 3166-1 alpha-2). - name: code type: keyword description: The Cloudflare datacenter that received the packet (nearest IATA airport code). + - name: country + type: keyword + description: The country of colo that received the packet (ISO 3166-1 alpha-2). - name: geo_hash type: keyword description: The Geo Hash where the colo that received the packet is located. @@ -76,6 +76,18 @@ - name: direction type: keyword description: The direction in relation to customer network. + - name: dns + type: group + fields: + - name: query + type: group + fields: + - name: name + type: keyword + description: The DNS query name (domain) if the packet is a DNS query. + - name: type + type: keyword + description: The DNS query type if the packet is a DNS query. - name: gre type: group fields: @@ -111,13 +123,13 @@ fields: - name: checksum type: long - description: Value of the Checksum header field in the ICMP packet + description: Value of the Checksum header field in the ICMP packet. - name: code type: long - description: Value of the Code header field in the ICMP packet + description: Value of the Code header field in the ICMP packet. - name: type type: long - description: Value of the Type header field in the ICMP packet + description: Value of the Type header field in the ICMP packet. - name: ip type: group fields: @@ -235,6 +247,9 @@ - name: outcome type: keyword description: The action that Cloudflare systems took on the packet. + - name: pfp_custom_tag + type: long + description: The custom network analytics tag set by Programmable Flow Protection. - name: protocol_state type: keyword description: State of the packet in the context of the protocol, if any. diff --git a/packages/cloudflare_logpush/data_stream/network_session/fields/fields.yml b/packages/cloudflare_logpush/data_stream/network_session/fields/fields.yml index 7cdffde9dc4..6b597574cdb 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/network_session/fields/fields.yml @@ -1,40 +1,27 @@ - name: cloudflare_logpush.network_session type: group - description: Integration for parsing Cloudflare Zero Trust Network Session logs. fields: - name: account_id type: keyword description: Cloudflare account ID. - - name: timestamp - type: date - description: The network session start timestamp with nanosecond precision. - name: destination type: group fields: + - name: bytes + type: long + description: The number of bytes sent from the origin to the client during the network session. - name: ip type: ip description: The IP of the destination (origin) for the network session. - name: port type: long description: The port of the destination origin for the network session. - - name: bytes - type: long - description: The number of bytes sent from the origin to the client during the network session. - name: tunnel_id type: keyword description: Identifier of the Cloudflare One connector to which the network session was routed to, if any. - name: detected_protocol type: keyword description: Detected traffic protocol of the network session. - - name: host - type: group - fields: - - name: id - type: keyword - description: Identifier of the client device which initiated the network session, if applicable. - - name: name - type: keyword - description: Name of the client device which initiated the network session, if applicable. - name: egress type: group fields: @@ -56,24 +43,69 @@ - name: name type: keyword description: The name of the egress rule that was applied by the Secure Web Gateway, if any. + - name: host + type: group + fields: + - name: id + type: keyword + description: Identifier of the client device which initiated the network session, if applicable. + - name: name + type: keyword + description: Name of the client device which initiated the network session, if applicable. - name: ingress type: group fields: - name: colo_name type: keyword description: The name of the Cloudflare colo to which traffic ingressed. + - name: initial_origin_ip + type: ip + description: The IP used to correlate FQDN matching between Gateway DNS and proxy. - name: offramp type: keyword description: The type of destination to which the network session was routed. - - name: transport + - name: registration_id type: keyword - description: Network protocol used for this network session. + description: Identifier of the client registration. + - name: resolved_fqdn + type: keyword + description: The fully qualified domain name of the destination. - name: rule_evaluation type: group fields: - name: time_ms type: long description: The duration taken by Secure Web Gateway applying applicable Network, HTTP, and Egress rules to the network session in milliseconds. + - name: session + type: group + fields: + - name: end + type: date + description: The network session end timestamp with nanosecond precision. + - name: id + type: keyword + description: The identifier of this network session. + - name: start + type: date + description: The network session start timestamp with nanosecond precision. + - name: sni + type: keyword + description: The server name indication (SNI) value from the TLS handshake. + - name: source + type: group + fields: + - name: bytes + type: long + description: The number of bytes sent from the client to the origin during the network session. + - name: internal_ip + type: ip + description: Local LAN IP of the device. Only available when connected via a GRE/IPsec tunnel on-ramp. + - name: ip + type: ip + description: Source IP of the network session. + - name: port + type: long + description: Source port of the network session. - name: tcp type: group fields: @@ -92,6 +124,9 @@ - name: reuse type: boolean description: Whether the TCP connection was reused for multiple HTTP requests. + - name: timestamp + type: date + description: The network session start timestamp with nanosecond precision. - name: tls type: group fields: @@ -110,15 +145,6 @@ - name: server type: group fields: - - name: cipher - type: keyword - description: TLS cipher suite used in the connection between Cloudflare and the origin. - - name: handshake_time_ms - type: long - description: Duration of handshaking the TLS connection between Cloudflare and the origin in milliseconds. - - name: version - type: keyword - description: TLS protocol version used in the connection between Cloudflare and the origin. - name: certificate type: group fields: @@ -128,42 +154,27 @@ - name: validation_result type: keyword description: The result of validating the TLS certificate of the origin. - - name: session - type: group - fields: - - name: id - type: keyword - description: The identifier of this network session. - - name: start - type: date - description: The network session start timestamp with nanosecond precision. - - name: end - type: date - description: The network session end timestamp with nanosecond precision. - - name: source - type: group - fields: - - name: ip - type: ip - description: Source IP of the network session. - - name: internal_ip - type: ip - description: Local LAN IP of the device. Only available when connected via a GRE/IPsec tunnel on-ramp. - - name: port - type: long - description: Source port of the network session. - - name: bytes - type: long - description: The number of bytes sent from the client to the origin during the network session. + - name: cipher + type: keyword + description: TLS cipher suite used in the connection between Cloudflare and the origin. + - name: handshake_time_ms + type: long + description: Duration of handshaking the TLS connection between Cloudflare and the origin in milliseconds. + - name: version + type: keyword + description: TLS protocol version used in the connection between Cloudflare and the origin. + - name: transport + type: keyword + description: Network protocol used for this network session. - name: user type: group fields: - - name: id - type: keyword - description: User identity where the network session originated from. - name: email type: keyword description: Email address associated with the user identity which initiated the network session. + - name: id + type: keyword + description: User identity where the network session originated from. - name: vlan type: group fields: diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/fields.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/fields.yml index 2a3f4517d91..a422eeb4615 100644 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/fields.yml @@ -1,6 +1,5 @@ - name: cloudflare_logpush.sinkhole_http type: group - description: Integration for parsing Cloudflare Logpush Sinkhole HTTP logs. fields: - name: account_id type: keyword diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/fields/fields.yml b/packages/cloudflare_logpush/data_stream/workers_trace/fields/fields.yml index f643692e89d..7c8a9175b85 100644 --- a/packages/cloudflare_logpush/data_stream/workers_trace/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/workers_trace/fields/fields.yml @@ -1,7 +1,9 @@ - name: cloudflare_logpush.workers_trace type: group - description: Integration for parsing Cloudflare Workers Trace Event logs. fields: + - name: cpu_time_ms + type: long + description: The amount of CPU time used by the Worker script, in milliseconds. - name: dispatch_namespace type: keyword description: The Cloudflare Worker dispatch namespace. @@ -38,6 +40,9 @@ - name: type type: keyword description: The event type that triggered the invocation. + - name: wall_time_ms + type: long + description: The elapsed wall-clock time in milliseconds for the Worker invocation. - name: log.source.address type: keyword description: Source address from which the log event was read / sent from. From 1e0d9024ac7f54645dc78e95182e73c99e209a58 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:19:42 +0530 Subject: [PATCH 03/14] fix email_security_alerts timestamp normalization Correct the Painless script to reference ctx.json.Timestamp (PascalCase) instead of ctx.json.timestamp, matching the actual field name from the Cloudflare API and the guard condition. --- .../elasticsearch/ingest_pipeline/default.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml index f9a0d14c6a5..75c16cee848 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml @@ -40,17 +40,17 @@ processors: source: | try { long t; - if (ctx.json.timestamp instanceof String) { - t = Long.parseLong(ctx.json.timestamp); - } else if (ctx.json.timestamp instanceof Number) { - t = (long)(ctx.json.timestamp); + if (ctx.json.Timestamp instanceof String) { + t = Long.parseLong(ctx.json.Timestamp); + } else if (ctx.json.Timestamp instanceof Number) { + t = (long)(ctx.json.Timestamp); } else { return; } if (t > (long)(1e18)) { - ctx.json.timestamp = t/(long)(1e6) + ctx.json.Timestamp = t/(long)(1e6) } else if (t < (long)(1e10)) { - ctx.json.timestamp = t*(long)(1e3) + ctx.json.Timestamp = t*(long)(1e3) } } catch (Exception e) {} From fc4e9d63110fcc15f6a5ddb2a9ca840f51febc88 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:20:38 +0530 Subject: [PATCH 04/14] fix spectrum_event broken grok guard condition and remove processor Fix the grok guard condition that used an incorrect path (ctx.json?.cloudflare_logpush) instead of (ctx.cloudflare_logpush) and a tautological || operator instead of &&. Also correct the remove processor to reference action instead of event_action. Update test data to use a valid disconnect timestamp. --- .../_dev/test/pipeline/test-pipeline-spectrum-event.log | 2 +- .../spectrum_event/elasticsearch/ingest_pipeline/default.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log index 7c765048f8a..248f66fee6c 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log @@ -1,4 +1,4 @@ -{"Application":"7ef659a2f8ef4810a9bade96fdad7c75","ClientAsn":200391,"ClientBytes":0,"ClientCountry":"bg","ClientIP":"67.43.156.0","ClientMatchedIpFirewall":"UNKNOWN","ClientPort":40456,"ClientProto":"tcp","ClientTcpRtt":0,"ClientTlsCipher":"UNK","ClientTlsClientHelloServerName":"server name","ClientTlsProtocol":"TLSv1.2","ClientTlsStatus":"UNKNOWN","ColoCode":"SOF","ConnectTimestamp":"2022-05-26T09:24:00Z","DisconnectTimestamp":"1970-01-01T00:00:00Z","Event":"connect","IpFirewall":false,"OriginBytes":0,"OriginIP":"175.16.199.0","OriginPort":3389,"OriginProto":"tcp","OriginTcpRtt":0,"OriginTlsCipher":"UNK","OriginTlsFingerprint":"0000000000000000000000000000000000000000000000000000000000000000.","OriginTlsMode":"off","OriginTlsProtocol":"unknown","OriginTlsStatus":"UNKNOWN","ProxyProtocol":"off","Status":0,"Timestamp":"2022-05-26T09:24:00Z"} +{"Application":"7ef659a2f8ef4810a9bade96fdad7c75","ClientAsn":200391,"ClientBytes":0,"ClientCountry":"bg","ClientIP":"67.43.156.0","ClientMatchedIpFirewall":"UNKNOWN","ClientPort":40456,"ClientProto":"tcp","ClientTcpRtt":0,"ClientTlsCipher":"UNK","ClientTlsClientHelloServerName":"server name","ClientTlsProtocol":"TLSv1.2","ClientTlsStatus":"UNKNOWN","ColoCode":"SOF","ConnectTimestamp":"2022-05-26T09:24:00Z","DisconnectTimestamp":"2022-06-01T00:00:00Z","Event":"connect","IpFirewall":false,"OriginBytes":0,"OriginIP":"175.16.199.0","OriginPort":3389,"OriginProto":"tcp","OriginTcpRtt":0,"OriginTlsCipher":"UNK","OriginTlsFingerprint":"0000000000000000000000000000000000000000000000000000000000000000.","OriginTlsMode":"off","OriginTlsProtocol":"unknown","OriginTlsStatus":"UNKNOWN","ProxyProtocol":"off","Status":0,"Timestamp":"2022-05-26T09:24:00Z"} {"Application":"7ef659a2f8ef4810a9bade96fdad7c75","ClientAsn":200391,"ClientBytes":0,"ClientCountry":"bg","ClientIP":"67.43.156.0","ClientMatchedIpFirewall":"UNKNOWN","ClientPort":40456,"ClientProto":"tcp","ClientTcpRtt":0,"ClientTlsCipher":"UNK","ClientTlsClientHelloServerName":"server name","ClientTlsProtocol":"TLSv1.2","ClientTlsStatus":"UNKNOWN","ColoCode":"SOF","ConnectTimestamp":"1653557040","DisconnectTimestamp":"0","Event":"connect","IpFirewall":false,"OriginBytes":0,"OriginIP":"175.16.199.0","OriginPort":3389,"OriginProto":"tcp","OriginTcpRtt":0,"OriginTlsCipher":"UNK","OriginTlsFingerprint":"0000000000000000000000000000000000000000000000000000000000000000.","OriginTlsMode":"off","OriginTlsProtocol":"unknown","OriginTlsStatus":"UNKNOWN","ProxyProtocol":"off","Status":0,"Timestamp":"1653557040"} {"Application":"7ef659a2f8ef4810a9bade96fdad7c75","ClientAsn":200391,"ClientBytes":0,"ClientCountry":"bg","ClientIP":"67.43.156.0","ClientMatchedIpFirewall":"UNKNOWN","ClientPort":40456,"ClientProto":"tcp","ClientTcpRtt":0,"ClientTlsCipher":"UNK","ClientTlsClientHelloServerName":"server name","ClientTlsProtocol":"TLSv1.2","ClientTlsStatus":"UNKNOWN","ColoCode":"SOF","ConnectTimestamp":"1653557040000000000","DisconnectTimestamp":"0","Event":"connect","IpFirewall":false,"OriginBytes":0,"OriginIP":"175.16.199.0","OriginPort":3389,"OriginProto":"tcp","OriginTcpRtt":0,"OriginTlsCipher":"UNK","OriginTlsFingerprint":"0000000000000000000000000000000000000000000000000000000000000000.","OriginTlsMode":"off","OriginTlsProtocol":"unknown","OriginTlsStatus":"UNKNOWN","ProxyProtocol":"off","Status":0,"Timestamp":"1653557040000000000"} {"Application":"7ef659a2f8ef4810a9bade96fdad7c76","ClientAsn":200391,"ClientBytes":0,"ClientCountry":"bg","ClientIP":"67.43.156.0","ClientMatchedIpFirewall":"UNKNOWN","ClientPort":40456,"ClientProto":"tcp","ClientTcpRtt":0,"ClientTlsCipher":"UNK","ClientTlsClientHelloServerName":"server name","ClientTlsProtocol":"TLSv1.2","ClientTlsStatus":"UNKNOWN","ColoCode":"SOF","ConnectTimestamp":"0","DisconnectTimestamp":"0","Event":"connect","IpFirewall":false,"OriginBytes":0,"OriginIP":"175.16.199.0","OriginPort":3389,"OriginProto":"tcp","OriginTcpRtt":0,"OriginT{\"Application\":\"7ef659a2f8ef4810a9bade96fdad7c75\",\"ClientAsn\":200391,\"ClientBytes\":0,\"ClientCountry\":\"bg\",\"ClientIP\":\"67.43.156.0\",\"ClientMatchedIpFirewall\":\"UNKNOWN\",\"ClientPort\":40456,\"ClientProto\":\"tcp\",\"ClientTcpRtt\":0,\"ClientTlsCipher\":\"UNK\",\"ClientTlsClientHelloServerName\":\"server name\",\"ClientTlsProtocol\":\"TLSv1.2\",\"ClientTlsStatus\":\"UNKNOWN\",\"ColoCode\":\"SOF\",\"ConnectTimestamp\":\"0\",\"DisconnectTimestamp\":\"0\",\"Event\":\"connect\",\"IpFirewall\":false,\"OriginBytes\":0,\"OriginIP\":\"175.16.199.0\",\"OriginPort\":3389,\"OriginProto\":\"tcp\",\"OriginTcpRtt\":0,\"OriginTlsCipher\":\"UNK\",\"OriginTlsFingerprint\":\"0000000000000000000000000000000000000000000000000000000000000000.\",\"OriginTlsMode\":\"off\",\"OriginTlsProtocol\":\"unknown\",\"OriginTlsStatus\":\"UNKNOWN\",\"ProxyProtocol\":\"off\",\"Status\":0,\"Timestamp\":\"1653557040000000000\"}\nconnectedlsCipher":"UNK","OriginTlsFingerprint":"0000000000000000000000000000000000000000000000000000000000000000.","OriginTlsMode":"off","OriginTlsProtocol":"unknown","OriginTlsStatus":"UNKNOWN","ProxyProtocol":"off","Status":0,"Timestamp":"1653557040000000000"} diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml index 226d96bb3f1..1839919098f 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml @@ -341,7 +341,7 @@ processors: target_field: cloudflare_logpush.spectrum_event.client.tls.protocol ignore_missing: true - grok: - if: ctx.json?.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != 'none' || ctx.json?.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != 'unknown' + if: ctx.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != 'none' && ctx.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != 'unknown' field: cloudflare_logpush.spectrum_event.client.tls.protocol patterns: - "^%{DATA:tls.version_protocol}v%{DATA:tls.version}$" @@ -439,7 +439,7 @@ processors: - cloudflare_logpush.spectrum_event.origin.ip - cloudflare_logpush.spectrum_event.origin.port - cloudflare_logpush.spectrum_event.application - - cloudflare_logpush.spectrum_event.event_action + - cloudflare_logpush.spectrum_event.action - cloudflare_logpush.spectrum_event.status - cloudflare_logpush.spectrum_event.client.asn - cloudflare_logpush.spectrum_event.client.bytes From 376886c31db67d3bd31d8c5f8e9bdd8301cdec73 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:25:31 +0530 Subject: [PATCH 05/14] fix network_analytics split processor case-sensitivity for TCPSackBlocks Correct the split processor condition to reference ctx.json.TCPSackBlocks consistently instead of mixing TCPSACKBlocks and TCPSackBlocks casing. --- .../network_analytics/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml index e731786f6e1..8afcd42927e 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml @@ -846,7 +846,7 @@ processors: - split: field: json.TCPSackBlocks separator: ', *' - if: ctx.json?.TCPSackBlocks instanceof String && ctx.json.TCPSACKBlocks != '' + if: ctx.json?.TCPSackBlocks instanceof String && ctx.json.TCPSackBlocks != '' - script: tag: script_raise_non_string_tcp_sack_blocks_to_array source: |- From 1c4f6bf2059c549bb95a3ebf9da163e72d4ee9a1 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:26:15 +0530 Subject: [PATCH 06/14] fix audit rename condition to use PascalCase for Interface field Correct the rename condition to use ctx.json?.Interface (PascalCase) matching the actual Cloudflare API field name instead of lowercase. --- .../data_stream/audit/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 304443b7da5..96422195afc 100644 --- a/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -163,7 +163,7 @@ processors: field: json.Interface target_field: cloudflare_logpush.audit.interface ignore_missing: true - if: ctx.json?.interface != '' + if: ctx.json?.Interface != '' - set: tag: set_event_provider field: event.provider From 7ca6dd7e3023518c5779b1afba610194fd0b1389 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:29:16 +0530 Subject: [PATCH 07/14] fix integer-to-keyword type casting by replacing rename with convert Replace rename processors with convert processors (type: string) for fields documented as integers or arrays of integers but mapped as keyword type in fields.yml. Affected fields: gateway_dns (CNAMECategoryIDs, EDEErrors, InitialCategoryIDs, MatchedIndicatorFeedIDs, ResolvedIPCategoryIDs), gateway_http (ApplicationIDs), gateway_network (ApplicationIDs, CategoryIDs). --- .../elasticsearch/ingest_pipeline/default.yml | 15 ++++++++++----- .../elasticsearch/ingest_pipeline/default.yml | 3 ++- .../elasticsearch/ingest_pipeline/default.yml | 6 ++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml index f4b69d1fbbe..cb39c59fab8 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml @@ -272,9 +272,10 @@ processors: - append: field: error.message value: '{{{_ingest.on_failure_message}}}' - - rename: + - convert: field: json.CNAMECategoryIDs target_field: cloudflare_logpush.gateway_dns.cname_category.ids + type: string ignore_missing: true - rename: field: json.CNAMECategoryNames @@ -321,13 +322,15 @@ processors: field: json.DoTSubdomain target_field: cloudflare_logpush.gateway_dns.dot_subdomain ignore_missing: true - - rename: + - convert: field: json.EDEErrors target_field: cloudflare_logpush.gateway_dns.extended_dns_error_codes + type: string ignore_missing: true - - rename: + - convert: field: json.InitialCategoryIDs target_field: cloudflare_logpush.gateway_dns.initial_category.ids + type: string ignore_missing: true - rename: field: json.InitialCategoryNames @@ -358,9 +361,10 @@ processors: field: json.MatchedCategoryNames target_field: cloudflare_logpush.gateway_dns.matched.category.names ignore_missing: true - - rename: + - convert: field: json.MatchedIndicatorFeedIDs target_field: cloudflare_logpush.gateway_dns.matched.indicator_feed.ids + type: string ignore_missing: true - rename: field: json.MatchedIndicatorFeedNames @@ -416,9 +420,10 @@ processors: field: json.QueryType target_field: cloudflare_logpush.gateway_dns.question.type_id ignore_missing: true - - rename: + - convert: field: json.ResolvedIPCategoryIDs target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.category.ids + type: string ignore_missing: true - rename: field: json.ResolvedIPCategoryNames diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml index 0bc4b1904c9..7a1f3e8d028 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml @@ -251,9 +251,10 @@ processors: field: json.AccountID target_field: cloudflare_logpush.gateway_http.account_id ignore_missing: true - - rename: + - convert: field: json.ApplicationIDs target_field: cloudflare_logpush.gateway_http.application.ids + type: string ignore_missing: true - rename: field: json.ApplicationNames diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml index de6a64f0483..c285bd6c6bb 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml @@ -219,17 +219,19 @@ processors: field: json.AccountID target_field: cloudflare_logpush.gateway_network.account_id ignore_missing: true - - rename: + - convert: field: json.ApplicationIDs target_field: cloudflare_logpush.gateway_network.application.ids + type: string ignore_missing: true - rename: field: json.ApplicationNames target_field: cloudflare_logpush.gateway_network.application.names ignore_missing: true - - rename: + - convert: field: json.CategoryIDs target_field: cloudflare_logpush.gateway_network.category.ids + type: string ignore_missing: true - rename: field: json.CategoryNames From b2aaa2b6c45ffdf98b5a7b40e2c62f163d59329c Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:30:10 +0530 Subject: [PATCH 08/14] remove ignore_failure from initial JSON processor Remove ignore_failure: true from the first JSON processor in all data stream pipelines that had it. Parsing failures should surface as errors rather than silently producing partial documents. --- .../elasticsearch/ingest_pipeline/default.yml | 1 - .../data_stream/dns/elasticsearch/ingest_pipeline/default.yml | 1 - .../elasticsearch/ingest_pipeline/default.yml | 1 - .../firewall_event/elasticsearch/ingest_pipeline/default.yml | 1 - .../http_request/elasticsearch/ingest_pipeline/default.yml | 1 - .../nel_report/elasticsearch/ingest_pipeline/default.yml | 1 - .../network_analytics/elasticsearch/ingest_pipeline/default.yml | 1 - .../page_shield_events/elasticsearch/ingest_pipeline/default.yml | 1 - .../spectrum_event/elasticsearch/ingest_pipeline/default.yml | 1 - 9 files changed, 9 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml index 4a19388b999..e94c3b8c17d 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] diff --git a/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml index 45d69312f47..1886ddb6a8b 100644 --- a/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml index 75c16cee848..dd97b18d745 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [email,network] diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml index ce50b4e6e50..b94b6b3bead 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] diff --git a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml index 7f9cf13a68f..3880f3d244c 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] diff --git a/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml index aa032863a84..36efb8c409d 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml index 8afcd42927e..92ae19e136f 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml index 187405b5d33..b77f019ffc4 100644 --- a/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml index 1839919098f..b0688a24929 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml @@ -12,7 +12,6 @@ processors: - json: field: event.original target_field: json - ignore_failure: true - set: field: event.category value: [network] From 0a44ac2e6e8ceaf9278ff03056dfcf42c76d46f2 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:31:36 +0530 Subject: [PATCH 09/14] convert grok processors to dissect for improved performance Replace grok processors with equivalent dissect processors in firewall_event (protocol parsing), http_request (protocol and TLS parsing), and spectrum_event (TLS parsing). Dissect is faster than grok for simple delimiter-based patterns. --- .../elasticsearch/ingest_pipeline/default.yml | 6 +++--- .../elasticsearch/ingest_pipeline/default.yml | 10 ++++------ .../elasticsearch/ingest_pipeline/default.yml | 9 ++++----- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml index b94b6b3bead..7e7644d1e6c 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml @@ -217,10 +217,10 @@ processors: field: json.ClientRequestProtocol target_field: cloudflare_logpush.firewall_event.client.request.protocol ignore_missing: true - - grok: + - dissect: field: cloudflare_logpush.firewall_event.client.request.protocol - patterns: - - "^%{DATA:network.protocol}/%{DATA:http.version}$" + pattern: '%{network.protocol}/%{http.version}' + ignore_missing: true ignore_failure: true - lowercase: field: network.protocol diff --git a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml index 3880f3d244c..0d27208026a 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml @@ -430,10 +430,9 @@ processors: field: json.ClientRequestProtocol target_field: cloudflare_logpush.http_request.client.request.protocol ignore_missing: true - - grok: + - dissect: field: cloudflare_logpush.http_request.client.request.protocol - patterns: - - "^%{DATA:network.protocol}/%{DATA:http.version}$" + pattern: '%{network.protocol}/%{http.version}' ignore_missing: true ignore_failure: true - lowercase: @@ -475,10 +474,9 @@ processors: field: json.ClientSSLProtocol target_field: cloudflare_logpush.http_request.client.ssl.protocol ignore_missing: true - - grok: + - dissect: field: cloudflare_logpush.http_request.client.ssl.protocol - patterns: - - "^%{DATA:tls.version_protocol}v%{DATA:tls.version}$" + pattern: '%{tls.version_protocol}v%{tls.version}' ignore_missing: true ignore_failure: true - lowercase: diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml index b0688a24929..2c8ee1ee7c8 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml @@ -339,12 +339,11 @@ processors: field: json.ClientTlsProtocol target_field: cloudflare_logpush.spectrum_event.client.tls.protocol ignore_missing: true - - grok: - if: ctx.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != 'none' && ctx.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != 'unknown' + - dissect: + if: ctx.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != null && ctx.cloudflare_logpush.spectrum_event.client.tls.protocol != 'none' && ctx.cloudflare_logpush.spectrum_event.client.tls.protocol != 'unknown' field: cloudflare_logpush.spectrum_event.client.tls.protocol - patterns: - - "^%{DATA:tls.version_protocol}v%{DATA:tls.version}$" - ignore_failure: true + pattern: '%{tls.version_protocol}v%{tls.version}' + ignore_missing: true - lowercase: field: tls.version_protocol ignore_missing: true From 5c8d688be717dfb6f95fd47d10e626fddacf7814 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:34:47 +0530 Subject: [PATCH 10/14] consolidate timestamp handling, add convert processors, implement null removal script, standardize error.message format, and add processor tags Consolidate multiple timestamp-to-Unix-millis script processors into a single, efficient script across all data streams. Update test input log files to include varied timestamp format test cases. Add missing convert processors to ensure fields declared as ip, long, boolean, or double in fields.yml are correctly typed at ingest time. Implement the latest null removal script (handleMap/handleList) across all data streams for consistent cleanup of null and empty values. Update error.message values in on_failure blocks to use the full standardized format including processor type, tag, pipeline, and message. Add a tag key to every processor across all data streams for easier debugging and error tracing. Align http_request header naming between pipeline and fields.yml (singular header to plural headers for RequestHeaders/ResponseHeaders). --- .../pipeline/test-pipeline-access-request.log | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 199 ++++-- .../elasticsearch/ingest_pipeline/default.yml | 151 ++-- .../_dev/test/pipeline/test-pipeline-casb.log | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 139 ++-- .../pipeline/test-pipeline-device-posture.log | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 212 ++++-- .../elasticsearch/ingest_pipeline/default.yml | 123 ++-- .../_dev/test/pipeline/test-pipeline-dns.log | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 280 ++++++-- .../elasticsearch/ingest_pipeline/default.yml | 235 +++++-- .../elasticsearch/ingest_pipeline/default.yml | 498 ++++++++++--- .../elasticsearch/ingest_pipeline/default.yml | 384 +++++++++-- .../test-pipeline-gateway-network.log | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 313 ++++++--- .../elasticsearch/ingest_pipeline/default.yml | 652 +++++++++++++----- .../elasticsearch/ingest_pipeline/default.yml | 265 ++++--- .../pipeline/test-pipeline-nel-report.log | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 143 ++-- .../docker/sample_logs/network_analytics.log | 2 +- .../test-pipeline-network-session.log | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 500 +++++++++++--- .../elasticsearch/ingest_pipeline/default.yml | 159 +++-- .../elasticsearch/ingest_pipeline/default.yml | 186 +++-- .../elasticsearch/ingest_pipeline/default.yml | 346 ++++++---- .../elasticsearch/ingest_pipeline/default.yml | 112 ++- 26 files changed, 3583 insertions(+), 1330 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log b/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log index e5dda683cdf..de647c38f5a 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log @@ -1,3 +1,3 @@ {"Action":"login","Allowed":true,"AppDomain":"partner-zt-logs.cloudflareaccess.com/warp","AppUUID":"123e4567-e89b-12d3-a456-426614174000","Connection":"onetimepin","Country":"us","CreatedAt":"2023-05-24T02:48:33+09:30","Email":"user@example.com","IPAddress":"67.43.156.93","PurposeJustificationPrompt":"Please provide your reason for accessing the application.","PurposeJustificationResponse":"I need to access the application for work purposes.","RayID":"00c0ffeeabc12345","TemporaryAccessApprovers":["approver1@example.com","approver2@example.com"],"TemporaryAccessDuration":7200,"UserUID":"166befbb-00e3-5e20-bd6e-27245333949f"} {"Action":"login","Allowed":true,"AppDomain":"partner-zt-logs.cloudflareaccess.com/warp","AppUUID":"123e4567-e89b-12d3-a456-426614174000","Connection":"onetimepin","Country":"us","CreatedAt":1684862313,"Email":"user@example.com","IPAddress":"67.43.156.93","PurposeJustificationPrompt":"Please provide your reason for accessing the application.","PurposeJustificationResponse":"I need to access the application for work purposes.","RayID":"00c0ffeeabc12345","TemporaryAccessApprovers":["approver1@example.com","approver2@example.com"],"TemporaryAccessDuration":7200,"UserUID":"166befbb-00e3-5e20-bd6e-27245333949f"} -{"Action":"login","Allowed":true,"AppDomain":"partner-zt-logs.cloudflareaccess.com/warp","AppUUID":"123e4567-e89b-12d3-a456-426614174000","Connection":"onetimepin","Country":"us","CreatedAt":1684862313000000000,"Email":"user@example.com","IPAddress":"67.43.156.93","PurposeJustificationPrompt":"Please provide your reason for accessing the application.","PurposeJustificationResponse":"I need to access the application for work purposes.","RayID":"00c0ffeeabc12345","TemporaryAccessApprovers":["approver1@example.com","approver2@example.com"],"TemporaryAccessDuration":7200,"UserUID":"166befbb-00e3-5e20-bd6e-27245333949f"} \ No newline at end of file +{"Action":"login","Allowed":true,"AppDomain":"partner-zt-logs.cloudflareaccess.com/warp","AppUUID":"123e4567-e89b-12d3-a456-426614174000","Connection":"onetimepin","Country":"us","CreatedAt":1684862313000000000,"Email":"user@example.com","IPAddress":"67.43.156.93","PurposeJustificationPrompt":"Please provide your reason for accessing the application.","PurposeJustificationResponse":"I need to access the application for work purposes.","RayID":"00c0ffeeabc12345","TemporaryAccessApprovers":["approver1@example.com","approver2@example.com"],"TemporaryAccessDuration":7200,"UserUID":"166befbb-00e3-5e20-bd6e-27245333949f"} diff --git a/packages/cloudflare_logpush/data_stream/access_request/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/access_request/elasticsearch/ingest_pipeline/default.yml index 8b6f31bcfc5..666d1003f25 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/access_request/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Access Request logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_24c7243c field: event.type value: [access] - set: + tag: set_event_kind_de80643c field: event.kind value: event -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,29 +45,31 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# ECS fields + # ECS fields + - convert: + tag: convert_json_CreatedAt_4a713653 + field: json.CreatedAt + type: long + if: ctx.json?.CreatedAt != null && ctx.json.CreatedAt != '' + ignore_failure: true - script: lang: painless tag: painless_created_at_to_milli - if: ctx.json?.CreatedAt != null + if: ctx.json?.CreatedAt != null && ctx.json.CreatedAt instanceof Number source: | - try { - long t; - if (ctx.json.CreatedAt instanceof String) { - t = Long.parseLong(ctx.json.CreatedAt); - } else if (ctx.json.CreatedAt instanceof Number) { - t = (long)(ctx.json.CreatedAt); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.CreatedAt = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.CreatedAt = t*(long)(1e3) - } + long t = (long)(ctx.json.CreatedAt); + if (t > (long)(1e18)) { + ctx.json.CreatedAt = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.CreatedAt = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_5fa1003d + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_CreatedAt_58cc069b field: json.CreatedAt if: ctx.json?.CreatedAt != null && ctx.json.CreatedAt != '' formats: @@ -63,80 +78,114 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_0f15ed2d + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_access_request_timestamp_58233c6b field: cloudflare_logpush.access_request.timestamp copy_from: "@timestamp" + ignore_empty_value: true - rename: + tag: rename_json_Action_to_cloudflare_logpush_access_request_action_cd9b83c4 field: json.Action target_field: cloudflare_logpush.access_request.action ignore_missing: true - set: + tag: set_event_action_3b69f4cf field: event.action copy_from: cloudflare_logpush.access_request.action ignore_empty_value: true - - rename: + - convert: + tag: convert_json_Allowed_to_cloudflare_logpush_access_request_allowed_c88346eb field: json.Allowed target_field: cloudflare_logpush.access_request.allowed + type: boolean ignore_missing: true -# Enrich event.type based on the allowed field + on_failure: + - append: + tag: append_error_message_1b5d5d29 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + # Enrich event.type based on the allowed field - append: + tag: append_event_type_9da9f935 field: event.type value: allowed allow_duplicates: false - if: ctx.cloudflare_logpush?.access_request?.allowed != null && ctx.cloudflare_logpush?.access_request?.allowed == true + if: ctx.cloudflare_logpush?.access_request?.allowed == true - append: + tag: append_event_type_92bf3f71 field: event.type value: denied allow_duplicates: false - if: ctx.cloudflare_logpush?.access_request?.allowed != null && ctx.cloudflare_logpush?.access_request?.allowed == false + if: ctx.cloudflare_logpush?.access_request?.allowed == false - rename: + tag: rename_json_AppDomain_to_cloudflare_logpush_access_request_app_domain_1f56d686 field: json.AppDomain target_field: cloudflare_logpush.access_request.app.domain ignore_missing: true - set: + tag: set_url_domain_7cb52535 field: url.domain copy_from: cloudflare_logpush.access_request.app.domain ignore_empty_value: true - rename: + tag: rename_json_AppUUID_to_cloudflare_logpush_access_request_app_uuid_d39a6a6e field: json.AppUUID target_field: cloudflare_logpush.access_request.app.uuid ignore_missing: true - rename: + tag: rename_json_Connection_to_cloudflare_logpush_access_request_connection_c6197c9c field: json.Connection target_field: cloudflare_logpush.access_request.connection ignore_missing: true - rename: + tag: rename_json_Country_to_cloudflare_logpush_access_request_country_4a1942ee field: json.Country target_field: cloudflare_logpush.access_request.country ignore_missing: true - set: + tag: set_client_geo_country_name_cc252be1 field: client.geo.country_name copy_from: cloudflare_logpush.access_request.country ignore_empty_value: true - rename: + tag: rename_json_Email_to_cloudflare_logpush_access_request_user_email_8368fd05 field: json.Email target_field: cloudflare_logpush.access_request.user.email ignore_missing: true - set: + tag: set_user_email_777a8c8b field: user.email copy_from: cloudflare_logpush.access_request.user.email ignore_empty_value: true - - rename: + - convert: + tag: convert_json_IPAddress_to_cloudflare_logpush_access_request_client_ip_82adb80d field: json.IPAddress target_field: cloudflare_logpush.access_request.client.ip + type: ip + if: ctx.json?.IPAddress != '' ignore_missing: true + on_failure: + - append: + tag: append_error_message_0fdbcdbf + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_client_ip_fea56e55 field: client.ip copy_from: cloudflare_logpush.access_request.client.ip ignore_empty_value: true -# Geo enrichment + # Geo enrichment - geoip: + tag: geoip_client_ip_to_client_geo_90831252 field: client.ip target_field: client.geo + ignore_missing: true - geoip: + tag: geoip_client_ip_to_client_as_f17fb2b3 database_file: GeoLite2-ASN.mmdb field: client.ip target_field: client.as @@ -145,62 +194,82 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_client_as_asn_to_client_as_number_a6e30d01 field: client.as.asn target_field: client.as.number ignore_missing: true - rename: + tag: rename_client_as_organization_name_to_client_as_organization_name_817a526f field: client.as.organization_name target_field: client.as.organization.name ignore_missing: true - rename: + tag: rename_json_PurposeJustificationPrompt_to_cloudflare_logpush_access_request_request_prompt_4c2a8f1f field: json.PurposeJustificationPrompt target_field: cloudflare_logpush.access_request.request.prompt ignore_missing: true - rename: + tag: rename_json_PurposeJustificationResponse_to_cloudflare_logpush_access_request_request_response_634724ab field: json.PurposeJustificationResponse target_field: cloudflare_logpush.access_request.request.response ignore_missing: true - rename: + tag: rename_json_RayID_to_cloudflare_logpush_access_request_ray_id_c5de5bdc field: json.RayID target_field: cloudflare_logpush.access_request.ray.id ignore_missing: true - set: + tag: set_event_id_dcd710c3 field: event.id copy_from: cloudflare_logpush.access_request.ray.id ignore_empty_value: true - rename: + tag: rename_json_TemporaryAccessApprovers_to_cloudflare_logpush_access_request_temp_access_approvers_018ad516 field: json.TemporaryAccessApprovers target_field: cloudflare_logpush.access_request.temp_access.approvers ignore_missing: true - - rename: + - convert: + tag: convert_json_TemporaryAccessDuration_to_cloudflare_logpush_access_request_temp_access_duration_f88c54ed field: json.TemporaryAccessDuration target_field: cloudflare_logpush.access_request.temp_access.duration + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_31ff4f43 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_UserUID_to_cloudflare_logpush_access_request_user_id_dde20323 field: json.UserUID target_field: cloudflare_logpush.access_request.user.id ignore_missing: true - set: + tag: set_user_id_1cba3d65 field: user.id copy_from: cloudflare_logpush.access_request.user.id ignore_empty_value: true -# Create related fields + # Create related fields - append: + tag: append_related_ip_1d5071d9 field: related.ip - value: "{{{client.ip}}}" - if: ctx.client?.ip != null + value: '{{{cloudflare_logpush.access_request.client.ip}}}' + if: ctx.cloudflare_logpush?.access_request?.client?.ip != null allow_duplicates: false - append: + tag: append_related_user_96f7f90b field: related.user - value: "{{{user.id}}}" - if: ctx.user?.id != null + value: '{{{cloudflare_logpush.access_request.user.id}}}' + if: ctx.cloudflare_logpush?.access_request?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_3fa8f231 field: related.user - value: "{{{user.email}}}" - if: ctx.user?.email != null + value: '{{{cloudflare_logpush.access_request.user.email}}}' + if: ctx.cloudflare_logpush?.access_request?.user?.email != null allow_duplicates: false - foreach: + tag: foreach_cloudflare_logpush_access_request_temp_access_approvers_eb3fbf32 field: cloudflare_logpush.access_request.temp_access.approvers processor: append: @@ -215,6 +284,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_3bfb2ba3 field: - cloudflare_logpush.access_request.timestamp - cloudflare_logpush.access_request.action @@ -226,32 +296,51 @@ processors: - cloudflare_logpush.access_request.user.id - cloudflare_logpush.access_request.user.email if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 96422195afc..fe969c85186 100644 --- a/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -2,20 +2,26 @@ description: Pipeline for parsing Cloudflare Audit logs. processors: - set: - tag: set_ecs_version + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: - tag: rename_message field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: - tag: json_event_original + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - ignore_failure: true - set: tag: set_event_type field: event.type @@ -28,9 +34,9 @@ processors: tag: set_event_category field: event.category value: [authentication] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -39,27 +45,28 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_When_fcae7a8e + field: json.When + type: long + if: ctx.json?.When != null && ctx.json.When != '' + ignore_failure: true - script: lang: painless tag: painless_when_to_milli - if: ctx.json?.When != null + if: ctx.json?.When != null && ctx.json.When instanceof Number source: | - try { - long t; - if (ctx.json.When instanceof String) { - t = Long.parseLong(ctx.json.When); - } else if (ctx.json.When instanceof Number) { - t = (long)(ctx.json.When); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.When = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.When = t*(long)(1e3) - } + long t = (long)(ctx.json.When); + if (t > (long)(1e18)) { + ctx.json.When = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.When = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_20151760 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: tag: date_json_when field: json.When @@ -71,8 +78,9 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_864f4d96 field: error.message - value: "{{{_ingest.on_failure_message}}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: tag: set_cloudflare_logpush_audit_timestamp field: cloudflare_logpush.audit.timestamp @@ -131,13 +139,14 @@ processors: tag: convert_json_actorip field: json.ActorIP target_field: cloudflare_logpush.audit.actor.ip - if: ctx.json?.ActorIP != '' type: ip + if: ctx.json?.ActorIP != '' ignore_missing: true on_failure: - append: + tag: append_error_message_e8abdc77 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: tag: set_source_ip field: source.ip @@ -163,7 +172,7 @@ processors: field: json.Interface target_field: cloudflare_logpush.audit.interface ignore_missing: true - if: ctx.json?.Interface != '' + if: ctx.json?.Interface != null && ctx.json.Interface != '' - set: tag: set_event_provider field: event.provider @@ -202,19 +211,29 @@ processors: target_field: cloudflare_logpush.audit.resource.type ignore_missing: true - append: - tag: append_related_user + tag: append_related_user_actor_id field: related.user - value: '{{{user.id}}}' - if: ctx.user?.id != null + value: '{{{cloudflare_logpush.audit.actor.id}}}' + if: ctx.cloudflare_logpush?.audit?.actor?.id != null allow_duplicates: false - ignore_failure: true - append: - tag: append_related_ip + tag: append_related_user_actor_email + field: related.user + value: '{{{cloudflare_logpush.audit.actor.email}}}' + if: ctx.cloudflare_logpush?.audit?.actor?.email != null + allow_duplicates: false + - append: + tag: append_related_ip_actor_ip field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null + value: '{{{cloudflare_logpush.audit.actor.ip}}}' + if: ctx.cloudflare_logpush?.audit?.actor?.ip != null + allow_duplicates: false + - append: + tag: append_related_user_owner_id + field: related.user + value: '{{{cloudflare_logpush.audit.owner.id}}}' + if: ctx.cloudflare_logpush?.audit?.owner?.id != null allow_duplicates: false - ignore_failure: true - remove: tag: remove_json_conf field: @@ -233,41 +252,53 @@ processors: - cloudflare_logpush.audit.actor.email - cloudflare_logpush.audit.actor.id if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - tag: script_drops_null_empty_values_recursively - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: - tag: append_error_message field: error.message - value: > - Processor "{{{ _ingest.on_failure_processor_type }}}" - with tag "{{{ _ingest.on_failure_processor_tag }}}" - in pipeline "{{{ _ingest.on_failure_pipeline }}}" - failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: - tag: set_event_kind_2 field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: - tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log b/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log index e3ef67423f0..58d3303db45 100644 --- a/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log @@ -1,3 +1,3 @@ {"AssetDisplayName":"John Doe","AssetExternalID":"0051N000004mG2LAAA","AssetLink":"https://example.com/resource","AssetMetadata":{"Id":"0051N000004mG2LAAA","Fax":null,"Name":"John Doe","Alias":"JDoe","Email":"user@example.com","Phone":"+3460000000","Title":"Customer Solutions Engineer","Address":{"city":"Singapore","state":null,"street":null,"country":"Singapore","latitude":null,"longitude":null,"stateCode":null,"postalCode":null,"countryCode":"SG","geocodeAccuracy":null},"Division":null,"IsActive":false,"LastName":"Doe","UserType":"Standard","AccountId":null,"BadgeText":"","ContactId":null,"Extension":null,"FirstName":"John","Signature":null,"Department":"521","SenderName":null,"UserRoleId":"00E2G000001E","attributes":{"url":"/services/data/userID","type":"User"},"CompanyName":"MyCompany","MobilePhone":null,"SenderEmail":"sender@example.com","CallCenterId":null,"FullPhotoUrl":"https://photos.com/profilephoto/001","LocaleSidKey":"en_SG","LastLoginDate":"2021-10-06T06:32:09.000+0000","SmallPhotoUrl":"https://photos.com/photo/001","BannerPhotoUrl":"/profilephoto/001","EmployeeNumber":"18124","LastViewedDate":null,"TimeZoneSidKey":"Asia/Singapore","DigestFrequency":"D","ForecastEnabled":false,"EmailEncodingKey":"UTF-8","CommunityNickname":"Doe.John","LanguageLocaleKey":"en_US","LastReferencedDate":null,"ReceivesInfoEmails":true,"SmallBannerPhotoUrl":"/profilephoto/001/D","FederationIdentifier":null,"IsProfilePhotoActive":false,"MediumBannerPhotoUrl":"/profilephoto/001/E","EmailPreferencesAutoBcc":true,"ReceivesAdminInfoEmails":true,"OfflineTrialExpirationDate":null,"UserPermissionsOfflineUser":false,"UserPermissionsSupportUser":false,"UserPermissionsMarketingUser":false,"UserPermissionsInteractionUser":true,"DefaultGroupNotificationFrequency":"N","UserPermissionsCallCenterAutoLogin":false},"DetectedTimestamp":"2023-05-16T10:00:00Z","FindingTypeDisplayName":"Salesforce User Sending Email with Different Email Address","FindingTypeID":"a2790c4f-03f5-449f-b209-5f4447f417aa","FindingTypeSeverity":"Medium","InstanceID":"6b187be4-2dd5-42c5-a37b-111111111111","IntegrationDisplayName":"Salesforce Testing","IntegrationID":"c772678d-5cf1-4c73-bf3f-111111111111","IntegrationPolicyVendor":"Salesforce Connection"} {"AssetDisplayName":"John Doe","AssetExternalID":"0051N000004mG2LAAA","AssetLink":"https://example.com/resource","AssetMetadata":{"Id":"0051N000004mG2LAAA","Fax":null,"Name":"John Doe","Alias":"JDoe","Email":"user@example.com","Phone":"+3460000000","Title":"Customer Solutions Engineer","Address":{"city":"Singapore","state":null,"street":null,"country":"Singapore","latitude":null,"longitude":null,"stateCode":null,"postalCode":null,"countryCode":"SG","geocodeAccuracy":null},"Division":null,"IsActive":false,"LastName":"Doe","UserType":"Standard","AccountId":null,"BadgeText":"","ContactId":null,"Extension":null,"FirstName":"John","Signature":null,"Department":"521","SenderName":null,"UserRoleId":"00E2G000001E","attributes":{"url":"/services/data/userID","type":"User"},"CompanyName":"MyCompany","MobilePhone":null,"SenderEmail":"sender@example.com","CallCenterId":null,"FullPhotoUrl":"https://photos.com/profilephoto/001","LocaleSidKey":"en_SG","LastLoginDate":"2021-10-06T06:32:09.000+0000","SmallPhotoUrl":"https://photos.com/photo/001","BannerPhotoUrl":"/profilephoto/001","EmployeeNumber":"18124","LastViewedDate":null,"TimeZoneSidKey":"Asia/Singapore","DigestFrequency":"D","ForecastEnabled":false,"EmailEncodingKey":"UTF-8","CommunityNickname":"Doe.John","LanguageLocaleKey":"en_US","LastReferencedDate":null,"ReceivesInfoEmails":true,"SmallBannerPhotoUrl":"/profilephoto/001/D","FederationIdentifier":null,"IsProfilePhotoActive":false,"MediumBannerPhotoUrl":"/profilephoto/001/E","EmailPreferencesAutoBcc":true,"ReceivesAdminInfoEmails":true,"OfflineTrialExpirationDate":null,"UserPermissionsOfflineUser":false,"UserPermissionsSupportUser":false,"UserPermissionsMarketingUser":false,"UserPermissionsInteractionUser":true,"DefaultGroupNotificationFrequency":"N","UserPermissionsCallCenterAutoLogin":false},"DetectedTimestamp":"1684231200","FindingTypeDisplayName":"Salesforce User Sending Email with Different Email Address","FindingTypeID":"a2790c4f-03f5-449f-b209-5f4447f417aa","FindingTypeSeverity":"Medium","InstanceID":"6b187be4-2dd5-42c5-a37b-111111111111","IntegrationDisplayName":"Salesforce Testing","IntegrationID":"c772678d-5cf1-4c73-bf3f-111111111111","IntegrationPolicyVendor":"Salesforce Connection"} -{"AssetDisplayName":"John Doe","AssetExternalID":"0051N000004mG2LAAA","AssetLink":"https://example.com/resource","AssetMetadata":{"Id":"0051N000004mG2LAAA","Fax":null,"Name":"John Doe","Alias":"JDoe","Email":"user@example.com","Phone":"+3460000000","Title":"Customer Solutions Engineer","Address":{"city":"Singapore","state":null,"street":null,"country":"Singapore","latitude":null,"longitude":null,"stateCode":null,"postalCode":null,"countryCode":"SG","geocodeAccuracy":null},"Division":null,"IsActive":false,"LastName":"Doe","UserType":"Standard","AccountId":null,"BadgeText":"","ContactId":null,"Extension":null,"FirstName":"John","Signature":null,"Department":"521","SenderName":null,"UserRoleId":"00E2G000001E","attributes":{"url":"/services/data/userID","type":"User"},"CompanyName":"MyCompany","MobilePhone":null,"SenderEmail":"sender@example.com","CallCenterId":null,"FullPhotoUrl":"https://photos.com/profilephoto/001","LocaleSidKey":"en_SG","LastLoginDate":"2021-10-06T06:32:09.000+0000","SmallPhotoUrl":"https://photos.com/photo/001","BannerPhotoUrl":"/profilephoto/001","EmployeeNumber":"18124","LastViewedDate":null,"TimeZoneSidKey":"Asia/Singapore","DigestFrequency":"D","ForecastEnabled":false,"EmailEncodingKey":"UTF-8","CommunityNickname":"Doe.John","LanguageLocaleKey":"en_US","LastReferencedDate":null,"ReceivesInfoEmails":true,"SmallBannerPhotoUrl":"/profilephoto/001/D","FederationIdentifier":null,"IsProfilePhotoActive":false,"MediumBannerPhotoUrl":"/profilephoto/001/E","EmailPreferencesAutoBcc":true,"ReceivesAdminInfoEmails":true,"OfflineTrialExpirationDate":null,"UserPermissionsOfflineUser":false,"UserPermissionsSupportUser":false,"UserPermissionsMarketingUser":false,"UserPermissionsInteractionUser":true,"DefaultGroupNotificationFrequency":"N","UserPermissionsCallCenterAutoLogin":false},"DetectedTimestamp":"1684231200000000000","FindingTypeDisplayName":"Salesforce User Sending Email with Different Email Address","FindingTypeID":"a2790c4f-03f5-449f-b209-5f4447f417aa","FindingTypeSeverity":"Medium","InstanceID":"6b187be4-2dd5-42c5-a37b-111111111111","IntegrationDisplayName":"Salesforce Testing","IntegrationID":"c772678d-5cf1-4c73-bf3f-111111111111","IntegrationPolicyVendor":"Salesforce Connection"} \ No newline at end of file +{"AssetDisplayName":"John Doe","AssetExternalID":"0051N000004mG2LAAA","AssetLink":"https://example.com/resource","AssetMetadata":{"Id":"0051N000004mG2LAAA","Fax":null,"Name":"John Doe","Alias":"JDoe","Email":"user@example.com","Phone":"+3460000000","Title":"Customer Solutions Engineer","Address":{"city":"Singapore","state":null,"street":null,"country":"Singapore","latitude":null,"longitude":null,"stateCode":null,"postalCode":null,"countryCode":"SG","geocodeAccuracy":null},"Division":null,"IsActive":false,"LastName":"Doe","UserType":"Standard","AccountId":null,"BadgeText":"","ContactId":null,"Extension":null,"FirstName":"John","Signature":null,"Department":"521","SenderName":null,"UserRoleId":"00E2G000001E","attributes":{"url":"/services/data/userID","type":"User"},"CompanyName":"MyCompany","MobilePhone":null,"SenderEmail":"sender@example.com","CallCenterId":null,"FullPhotoUrl":"https://photos.com/profilephoto/001","LocaleSidKey":"en_SG","LastLoginDate":"2021-10-06T06:32:09.000+0000","SmallPhotoUrl":"https://photos.com/photo/001","BannerPhotoUrl":"/profilephoto/001","EmployeeNumber":"18124","LastViewedDate":null,"TimeZoneSidKey":"Asia/Singapore","DigestFrequency":"D","ForecastEnabled":false,"EmailEncodingKey":"UTF-8","CommunityNickname":"Doe.John","LanguageLocaleKey":"en_US","LastReferencedDate":null,"ReceivesInfoEmails":true,"SmallBannerPhotoUrl":"/profilephoto/001/D","FederationIdentifier":null,"IsProfilePhotoActive":false,"MediumBannerPhotoUrl":"/profilephoto/001/E","EmailPreferencesAutoBcc":true,"ReceivesAdminInfoEmails":true,"OfflineTrialExpirationDate":null,"UserPermissionsOfflineUser":false,"UserPermissionsSupportUser":false,"UserPermissionsMarketingUser":false,"UserPermissionsInteractionUser":true,"DefaultGroupNotificationFrequency":"N","UserPermissionsCallCenterAutoLogin":false},"DetectedTimestamp":"1684231200000000000","FindingTypeDisplayName":"Salesforce User Sending Email with Different Email Address","FindingTypeID":"a2790c4f-03f5-449f-b209-5f4447f417aa","FindingTypeSeverity":"Medium","InstanceID":"6b187be4-2dd5-42c5-a37b-111111111111","IntegrationDisplayName":"Salesforce Testing","IntegrationID":"c772678d-5cf1-4c73-bf3f-111111111111","IntegrationPolicyVendor":"Salesforce Connection"} diff --git a/packages/cloudflare_logpush/data_stream/casb/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/casb/elasticsearch/ingest_pipeline/default.yml index 5858cf769c5..92a48cfe50c 100644 --- a/packages/cloudflare_logpush/data_stream/casb/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/casb/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare CASB Findings logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_24c7243c field: event.type value: [access] - set: + tag: set_event_kind_de80643c field: event.kind value: event -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,30 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_DetectedTimestamp_13072d0c + field: json.DetectedTimestamp + type: long + if: ctx.json?.DetectedTimestamp != null && ctx.json.DetectedTimestamp != '' + ignore_failure: true - script: lang: painless tag: painless_detected_timestamp_to_milli - if: ctx.json?.DetectedTimestamp != null + if: ctx.json?.DetectedTimestamp != null && ctx.json.DetectedTimestamp instanceof Number source: | - try { - long t; - if (ctx.json.DetectedTimestamp instanceof String) { - t = Long.parseLong(ctx.json.DetectedTimestamp); - } else if (ctx.json.DetectedTimestamp instanceof Number) { - t = (long)(ctx.json.DetectedTimestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.DetectedTimestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.DetectedTimestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.DetectedTimestamp); + if (t > (long)(1e18)) { + ctx.json.DetectedTimestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.DetectedTimestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_4765ab95 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_DetectedTimestamp_db70ac08 field: json.DetectedTimestamp if: ctx.json?.DetectedTimestamp != null && ctx.json.DetectedTimestamp != '' formats: @@ -62,82 +77,102 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_4fa151ba + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_casb_timestamp_e43be374 field: cloudflare_logpush.casb.timestamp copy_from: "@timestamp" + ignore_empty_value: true - rename: + tag: rename_json_AssetDisplayName_to_cloudflare_logpush_casb_asset_name_63f9ef91 field: json.AssetDisplayName target_field: cloudflare_logpush.casb.asset.name ignore_missing: true - rename: + tag: rename_json_AssetExternalID_to_cloudflare_logpush_casb_asset_id_efa6dd3c field: json.AssetExternalID target_field: cloudflare_logpush.casb.asset.id ignore_missing: true - rename: + tag: rename_json_AssetLink_to_cloudflare_logpush_casb_asset_url_cf671294 field: json.AssetLink target_field: cloudflare_logpush.casb.asset.url ignore_missing: true - uri_parts: + tag: uri_parts_cloudflare_logpush_casb_asset_url_to_url_38a3e7b3 field: cloudflare_logpush.casb.asset.url target_field: url if: ctx.cloudflare_logpush?.casb?.asset?.url != null - rename: + tag: rename_json_AssetMetadata_to_cloudflare_logpush_casb_asset_metadata_b65bc3fd field: json.AssetMetadata target_field: cloudflare_logpush.casb.asset.metadata ignore_missing: true - rename: + tag: rename_json_FindingTypeDisplayName_to_cloudflare_logpush_casb_finding_type_name_5434146f field: json.FindingTypeDisplayName target_field: cloudflare_logpush.casb.finding.type.name ignore_missing: true - rename: + tag: rename_json_FindingTypeID_to_cloudflare_logpush_casb_finding_type_id_1de37d4d field: json.FindingTypeID target_field: cloudflare_logpush.casb.finding.type.id ignore_missing: true - rename: + tag: rename_json_FindingTypeSeverity_to_cloudflare_logpush_casb_finding_type_severity_94779089 field: json.FindingTypeSeverity target_field: cloudflare_logpush.casb.finding.type.severity ignore_missing: true -# Set event.severity based on the incoming severity value ("low", "medium", "high", "critical") + # Set event.severity based on the incoming severity value ("low", "medium", "high", "critical") - set: + tag: set_event_severity_114ed469 field: event.severity value: 1 if: ctx.cloudflare_logpush?.casb?.finding?.type?.severity == "Low" - set: + tag: set_event_severity_a8bcf8b1 field: event.severity value: 2 if: ctx.cloudflare_logpush?.casb?.finding?.type?.severity == "Medium" - set: + tag: set_event_severity_49e96e4b field: event.severity value: 3 if: ctx.cloudflare_logpush?.casb?.finding?.type?.severity == "High" - set: + tag: set_event_severity_8f2d0035 field: event.severity value: 4 if: ctx.cloudflare_logpush?.casb?.finding?.type?.severity == "Critical" - rename: + tag: rename_json_InstanceID_to_cloudflare_logpush_casb_finding_id_ecd3a70b field: json.InstanceID target_field: cloudflare_logpush.casb.finding.id ignore_missing: true - set: + tag: set_event_id_e9188c13 field: event.id copy_from: cloudflare_logpush.casb.finding.id ignore_empty_value: true - rename: + tag: rename_json_IntegrationDisplayName_to_cloudflare_logpush_casb_integration_name_ad60962d field: json.IntegrationDisplayName target_field: cloudflare_logpush.casb.integration.name ignore_missing: true - rename: + tag: rename_json_IntegrationID_to_cloudflare_logpush_casb_integration_id_4913e059 field: json.IntegrationID target_field: cloudflare_logpush.casb.integration.id ignore_missing: true - rename: + tag: rename_json_IntegrationPolicyVendor_to_cloudflare_logpush_casb_integration_policy_vendor_48bcfb2c field: json.IntegrationPolicyVendor target_field: cloudflare_logpush.casb.integration.policy_vendor ignore_missing: true -# Clean event + # Clean event - remove: tag: remove_json_conf field: @@ -145,38 +180,58 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_0a57d463 field: - cloudflare_logpush.casb.timestamp - cloudflare_logpush.casb.asset.url - cloudflare_logpush.casb.finding.id - cloudflare_logpush.casb.finding.type.severity if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log b/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log index eed2d59c061..f7e4e214f37 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log @@ -1,3 +1,3 @@ {"ClientVersion":"2023.3.258","DeviceID":"083a8354-d56c-11ed-9771-111111111","DeviceManufacturer":"Google Compute Engine","DeviceModel":"Google Compute Engine","DeviceName":"zt-test-vm1","DeviceSerialNumber":"GoogleCloud-ABCD1234567890","DeviceType":"linux","Email":"user@example.com","OSVersion":"5.15.0","PolicyID":"policy-abcdefgh","PostureCheckName":"Ubuntu","PostureCheckType":"os_version","PostureEvaluatedResult":true,"PostureExpectedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"PostureReceivedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"Timestamp":"2023-05-17T12:00:00Z","UserUID":"user-abcdefgh"} {"ClientVersion":"2023.3.258","DeviceID":"083a8354-d56c-11ed-9771-111111111","DeviceManufacturer":"Google Compute Engine","DeviceModel":"Google Compute Engine","DeviceName":"zt-test-vm1","DeviceSerialNumber":"GoogleCloud-ABCD1234567890","DeviceType":"linux","Email":"user@example.com","OSVersion":"5.15.0","PolicyID":"policy-abcdefgh","PostureCheckName":"Ubuntu","PostureCheckType":"os_version","PostureEvaluatedResult":true,"PostureExpectedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"PostureReceivedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"Timestamp":"1684324800","UserUID":"user-abcdefgh"} -{"ClientVersion":"2023.3.258","DeviceID":"083a8354-d56c-11ed-9771-111111111","DeviceManufacturer":"Google Compute Engine","DeviceModel":"Google Compute Engine","DeviceName":"zt-test-vm1","DeviceSerialNumber":"GoogleCloud-ABCD1234567890","DeviceType":"linux","Email":"user@example.com","OSVersion":"5.15.0","PolicyID":"policy-abcdefgh","PostureCheckName":"Ubuntu","PostureCheckType":"os_version","PostureEvaluatedResult":true,"PostureExpectedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"PostureReceivedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"Timestamp":"1684324800000000000","UserUID":"user-abcdefgh"} \ No newline at end of file +{"ClientVersion":"2023.3.258","DeviceID":"083a8354-d56c-11ed-9771-111111111","DeviceManufacturer":"Google Compute Engine","DeviceModel":"Google Compute Engine","DeviceName":"zt-test-vm1","DeviceSerialNumber":"GoogleCloud-ABCD1234567890","DeviceType":"linux","Email":"user@example.com","OSVersion":"5.15.0","PolicyID":"policy-abcdefgh","PostureCheckName":"Ubuntu","PostureCheckType":"os_version","PostureEvaluatedResult":true,"PostureExpectedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"PostureReceivedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"Timestamp":"1684324800000000000","UserUID":"user-abcdefgh"} diff --git a/packages/cloudflare_logpush/data_stream/device_posture/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/device_posture/elasticsearch/ingest_pipeline/default.yml index 1a5a5b918af..2a4161f839c 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/device_posture/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Device Posture Results logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_ba9b1d84 field: event.category value: [host] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - set: + tag: set_event_kind_de80643c field: event.kind value: event -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,30 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_df40dd8a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Timestamp_70be028a field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' formats: @@ -62,145 +77,216 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_d6556db8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_device_posture_timestamp_8d6a05f8 field: cloudflare_logpush.device_posture.timestamp copy_from: "@timestamp" + ignore_empty_value: true - rename: + tag: rename_json_ClientVersion_to_cloudflare_logpush_device_posture_version_cd931ae0 field: json.ClientVersion target_field: cloudflare_logpush.device_posture.version ignore_missing: true - set: + tag: set_user_agent_version_0039ca63 field: user_agent.version copy_from: cloudflare_logpush.device_posture.version ignore_empty_value: true - rename: + tag: rename_json_DeviceID_to_cloudflare_logpush_device_posture_host_id_3ce2738f field: json.DeviceID target_field: cloudflare_logpush.device_posture.host.id ignore_missing: true - set: + tag: set_host_id_e7544a66 field: host.id copy_from: cloudflare_logpush.device_posture.host.id ignore_empty_value: true + - set: + field: device.id + tag: set_device_id_from_host_id + copy_from: cloudflare_logpush.device_posture.host.id + ignore_empty_value: true - rename: + tag: rename_json_DeviceManufacturer_to_cloudflare_logpush_device_posture_host_manufacturer_043abe93 field: json.DeviceManufacturer target_field: cloudflare_logpush.device_posture.host.manufacturer ignore_missing: true + - set: + field: device.manufacturer + tag: set_device_manufacturer_from_host_manufacturer + copy_from: cloudflare_logpush.device_posture.host.manufacturer + ignore_empty_value: true - rename: + tag: rename_json_DeviceModel_to_cloudflare_logpush_device_posture_host_model_265ecd11 field: json.DeviceModel target_field: cloudflare_logpush.device_posture.host.model ignore_missing: true + - set: + field: device.model.name + tag: set_device_model_name_from_host_model + copy_from: cloudflare_logpush.device_posture.host.model + ignore_empty_value: true - rename: + tag: rename_json_DeviceName_to_cloudflare_logpush_device_posture_host_name_707832af field: json.DeviceName target_field: cloudflare_logpush.device_posture.host.name ignore_missing: true - set: + tag: set_host_name_6d640aee field: host.name copy_from: cloudflare_logpush.device_posture.host.name ignore_empty_value: true - rename: + tag: rename_json_DeviceSerialNumber_to_cloudflare_logpush_device_posture_host_serial_b4965794 field: json.DeviceSerialNumber target_field: cloudflare_logpush.device_posture.host.serial ignore_missing: true + - set: + field: device.serial_number + tag: set_device_serial_number_from_host_serial + copy_from: cloudflare_logpush.device_posture.host.serial + ignore_empty_value: true - rename: + tag: rename_json_DeviceType_to_cloudflare_logpush_device_posture_host_os_family_44e10223 field: json.DeviceType target_field: cloudflare_logpush.device_posture.host.os.family ignore_missing: true - set: + tag: set_host_os_family_d17288fa field: host.os.family copy_from: cloudflare_logpush.device_posture.host.os.family ignore_empty_value: true + - set: + field: device.type + tag: set_device_type_from_host_os_family + copy_from: cloudflare_logpush.device_posture.host.os.family + ignore_empty_value: true - rename: + tag: rename_json_OSVersion_to_cloudflare_logpush_device_posture_host_os_version_07952967 field: json.OSVersion target_field: cloudflare_logpush.device_posture.host.os.version ignore_missing: true - set: + tag: set_host_os_version_0207737e field: host.os.version copy_from: cloudflare_logpush.device_posture.host.os.version ignore_empty_value: true - rename: + tag: rename_json_PolicyID_to_cloudflare_logpush_device_posture_rule_id_49b576e7 field: json.PolicyID target_field: cloudflare_logpush.device_posture.rule.id ignore_missing: true - set: + tag: set_rule_id_c7dba46e field: rule.id copy_from: cloudflare_logpush.device_posture.rule.id ignore_empty_value: true - rename: + tag: rename_json_PostureCheckName_to_cloudflare_logpush_device_posture_rule_name_7da2cd15 field: json.PostureCheckName target_field: cloudflare_logpush.device_posture.rule.name ignore_missing: true - set: + tag: set_rule_name_74bd3f32 field: rule.name copy_from: cloudflare_logpush.device_posture.rule.name ignore_empty_value: true - rename: + tag: rename_json_PostureCheckType_to_cloudflare_logpush_device_posture_rule_category_cee57ee7 field: json.PostureCheckType target_field: cloudflare_logpush.device_posture.rule.category ignore_missing: true - set: + tag: set_rule_category_96fa1c5a field: rule.category copy_from: cloudflare_logpush.device_posture.rule.category ignore_empty_value: true - - rename: + - convert: + tag: convert_json_PostureEvaluatedResult_to_cloudflare_logpush_device_posture_eval_result_08f711ef field: json.PostureEvaluatedResult target_field: cloudflare_logpush.device_posture.eval.result + type: boolean ignore_missing: true -# Set event.outcome from cloudflare_logpush.device_posture.eval.result + on_failure: + - append: + tag: append_error_message_6ebc95b5 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + # Set event.outcome from cloudflare_logpush.device_posture.eval.result - set: + tag: set_event_outcome_1d222747 field: event.outcome value: success if: ctx.cloudflare_logpush?.device_posture?.eval?.result != null && ctx.cloudflare_logpush?.device_posture?.eval?.result == true - set: + tag: set_event_outcome_47177ebb field: event.outcome value: failure if: ctx.cloudflare_logpush?.device_posture?.eval?.result != null && ctx.cloudflare_logpush?.device_posture?.eval?.result == false - rename: + tag: rename_json_PostureExpectedJSON_to_cloudflare_logpush_device_posture_eval_expected_624f00fb field: json.PostureExpectedJSON target_field: cloudflare_logpush.device_posture.eval.expected ignore_missing: true - rename: + tag: rename_json_PostureReceivedJSON_to_cloudflare_logpush_device_posture_eval_received_c94d95bd field: json.PostureReceivedJSON target_field: cloudflare_logpush.device_posture.eval.received ignore_missing: true - rename: + tag: rename_json_UserUID_to_cloudflare_logpush_device_posture_user_id_b055e3c0 field: json.UserUID target_field: cloudflare_logpush.device_posture.user.id ignore_missing: true - set: + tag: set_user_id_9ab83028 field: user.id copy_from: cloudflare_logpush.device_posture.user.id if: ctx.cloudflare_logpush?.device_posture?.user?.id != null && ctx.cloudflare_logpush?.device_posture?.user?.id != "" - rename: + tag: rename_json_Email_to_cloudflare_logpush_device_posture_user_email_70ea23e4 field: json.Email target_field: cloudflare_logpush.device_posture.user.email ignore_missing: true - set: + tag: set_user_email_839e5d58 field: user.email copy_from: cloudflare_logpush.device_posture.user.email if: ctx.cloudflare_logpush?.device_posture?.user?.email != null && ctx.cloudflare_logpush?.device_posture?.user?.email != "" -# Create related fields + - rename: + tag: rename_json_RegistrationID_to_cloudflare_logpush_device_posture_registration_id_aecfd64a + field: json.RegistrationID + target_field: cloudflare_logpush.device_posture.registration_id + ignore_missing: true + # Create related fields - append: + tag: append_related_hosts_4acd695f field: related.hosts - value: "{{{host.id}}}" - if: ctx.host?.id != null + value: '{{{cloudflare_logpush.device_posture.host.id}}}' + if: ctx.cloudflare_logpush?.device_posture?.host?.id != null allow_duplicates: false - append: + tag: append_related_hosts_27d4beef field: related.hosts - value: "{{{host.name}}}" - if: ctx.host?.name != null + value: '{{{cloudflare_logpush.device_posture.host.name}}}' + if: ctx.cloudflare_logpush?.device_posture?.host?.name != null allow_duplicates: false - append: + tag: append_related_user_867a7edd field: related.user - value: "{{{user.id}}}" - if: ctx.user?.id != null + value: '{{{cloudflare_logpush.device_posture.user.id}}}' + if: ctx.cloudflare_logpush?.device_posture?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_32655efd field: related.user - value: "{{{user.email}}}" - if: ctx.user?.email != null + value: '{{{cloudflare_logpush.device_posture.user.email}}}' + if: ctx.cloudflare_logpush?.device_posture?.user?.email != null allow_duplicates: false - remove: tag: remove_json_conf @@ -209,46 +295,68 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_00fb8f37 field: - cloudflare_logpush.device_posture.timestamp - cloudflare_logpush.device_posture.version - cloudflare_logpush.device_posture.host.id - cloudflare_logpush.device_posture.host.name + - cloudflare_logpush.device_posture.host.manufacturer + - cloudflare_logpush.device_posture.host.model + - cloudflare_logpush.device_posture.host.serial - cloudflare_logpush.device_posture.host.os.family - cloudflare_logpush.device_posture.host.os.version - cloudflare_logpush.device_posture.rule.id - cloudflare_logpush.device_posture.rule.name - cloudflare_logpush.device_posture.rule.category - - cloudflare_logpush.device_posture.result - cloudflare_logpush.device_posture.user.id - cloudflare_logpush.device_posture.user.email if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml index e94c3b8c17d..6e10f6991d4 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare DLP Forensic Copies logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,30 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Datetime_712ea5a9 + field: json.Datetime + type: long + if: ctx.json?.Datetime != null && ctx.json.Datetime != '' + ignore_failure: true - script: lang: painless tag: painless_datetime_to_milli - if: ctx.json?.Datetime != null + if: ctx.json?.Datetime != null && ctx.json.Datetime instanceof Number source: | - try { - long t; - if (ctx.json.Datetime instanceof String) { - t = Long.parseLong(ctx.json.Datetime); - } else if (ctx.json.Datetime instanceof Number) { - t = (long)(ctx.json.Datetime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Datetime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Datetime = t*(long)(1e3) - } + long t = (long)(ctx.json.Datetime); + if (t > (long)(1e18)) { + ctx.json.Datetime = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Datetime = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_6318b156 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Datetime_138a0e6b field: json.Datetime if: ctx.json?.Datetime != null && ctx.json.Datetime != '' formats: @@ -62,45 +77,52 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - append: + tag: append_error_message_ef827e11 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_dlp_forensic_copies_datetime_94563ee8 field: cloudflare_logpush.dlp_forensic_copies.datetime copy_from: '@timestamp' ignore_empty_value: true - rename: + tag: rename_json_AccountID_to_cloudflare_logpush_dlp_forensic_copies_account_id_7a9b9bc3 field: json.AccountID target_field: cloudflare_logpush.dlp_forensic_copies.account_id ignore_missing: true - rename: + tag: rename_json_ForensicCopyID_to_cloudflare_logpush_dlp_forensic_copies_forensic_copy_id_837e04d6 field: json.ForensicCopyID target_field: cloudflare_logpush.dlp_forensic_copies.forensic_copy_id ignore_missing: true - rename: + tag: rename_json_GatewayRequestID_to_cloudflare_logpush_dlp_forensic_copies_gateway_request_id_6dd30812 field: json.GatewayRequestID target_field: cloudflare_logpush.dlp_forensic_copies.gateway_request_id ignore_missing: true - rename: + tag: rename_json_Headers_to_cloudflare_logpush_dlp_forensic_copies_headers_6c33bc32 field: json.Headers target_field: cloudflare_logpush.dlp_forensic_copies.headers ignore_missing: true - rename: + tag: rename_json_Payload_to_cloudflare_logpush_dlp_forensic_copies_payload_78daacc6 field: json.Payload target_field: cloudflare_logpush.dlp_forensic_copies.payload ignore_missing: true - rename: + tag: rename_json_Phase_to_cloudflare_logpush_dlp_forensic_copies_phase_153ee19a field: json.Phase target_field: cloudflare_logpush.dlp_forensic_copies.phase ignore_missing: true - rename: + tag: rename_json_TriggeredRuleID_to_cloudflare_logpush_dlp_forensic_copies_triggered_rule_id_3e1335d4 field: json.TriggeredRuleID target_field: cloudflare_logpush.dlp_forensic_copies.triggered_rule_id ignore_missing: true - set: + tag: set_rule_id_f2c7e8a4 field: rule.id copy_from: cloudflare_logpush.dlp_forensic_copies.triggered_rule_id ignore_empty_value: true @@ -111,6 +133,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_bd294ab1 field: - cloudflare_logpush.dlp_forensic_copies.action - cloudflare_logpush.dlp_forensic_copies.host @@ -120,28 +143,38 @@ processors: ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); - set: field: event.kind - value: pipeline_error tag: set_pipeline_error_into_event_kind + value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -149,12 +182,10 @@ processors: on_failure: - append: field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log b/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log index 51120651623..c317600651a 100644 --- a/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log @@ -1,3 +1,3 @@ {"ColoCode":"MRS","EDNSSubnet":"1.128.0.0","EDNSSubnetLength":0,"QueryName":"example.com","QueryType":65535,"ResponseCached":false,"ResponseCode":0,"SourceIP":"175.16.199.0","Timestamp":"2022-05-26T09:23:54Z"} {"ColoCode":"MRS","EDNSSubnet":"1.128.0.0","EDNSSubnetLength":0,"QueryName":"example.com","QueryType":65535,"ResponseCached":false,"ResponseCode":0,"SourceIP":"175.16.199.0","Timestamp":"1653557034"} -{"ColoCode":"MRS","EDNSSubnet":"1.128.0.0","EDNSSubnetLength":0,"QueryName":"example.com","QueryType":65535,"ResponseCached":false,"ResponseCode":0,"SourceIP":"175.16.199.0","Timestamp":"1653557034000000000"} \ No newline at end of file +{"ColoCode":"MRS","EDNSSubnet":"1.128.0.0","EDNSSubnetLength":0,"QueryName":"example.com","QueryType":65535,"ResponseCached":false,"ResponseCode":0,"SourceIP":"175.16.199.0","Timestamp":"1653557034000000000"} diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml index dd97b18d745..4adc3982da3 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Email Security Alerts logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_52cf4ba2 field: event.category - value: [email,network] + value: [email, network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,30 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_Timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_76b0386a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Timestamp_70be028a field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' formats: @@ -62,81 +77,129 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - append: + tag: append_error_message_d6556db8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_email_security_alerts_timestamp_c2f86b60 field: cloudflare_logpush.email_security_alerts.timestamp copy_from: '@timestamp' ignore_empty_value: true - rename: + tag: rename_json_AlertID_to_cloudflare_logpush_email_security_alerts_alert_id_97d97274 field: json.AlertID target_field: cloudflare_logpush.email_security_alerts.alert_id ignore_missing: true - rename: + tag: rename_json_AlertReasons_to_cloudflare_logpush_email_security_alerts_alert_reasons_2fb8abf0 field: json.AlertReasons target_field: cloudflare_logpush.email_security_alerts.alert_reasons ignore_missing: true - rename: + tag: rename_json_Attachments_to_cloudflare_logpush_email_security_alerts_attachments_8e776cc3 field: json.Attachments target_field: cloudflare_logpush.email_security_alerts.attachments ignore_missing: true - rename: + tag: rename_json_CC_to_cloudflare_logpush_email_security_alerts_cc_57428a37 field: json.CC target_field: cloudflare_logpush.email_security_alerts.cc ignore_missing: true + - foreach: + tag: foreach_cloudflare_logpush_email_security_alerts_cc_368836c1 + field: cloudflare_logpush.email_security_alerts.cc + if: ctx.cloudflare_logpush?.email_security_alerts?.cc instanceof List + processor: + append: + field: email.cc.address + tag: append_cc_value_into_email_cc_address + value: '{{{_ingest._value}}}' + allow_duplicates: false - rename: + tag: rename_json_CCName_to_cloudflare_logpush_email_security_alerts_cc_name_4557bc82 field: json.CCName target_field: cloudflare_logpush.email_security_alerts.cc_name ignore_missing: true - rename: + tag: rename_json_FinalDisposition_to_cloudflare_logpush_email_security_alerts_final_disposition_395b5850 field: json.FinalDisposition target_field: cloudflare_logpush.email_security_alerts.final_disposition ignore_missing: true - rename: + tag: rename_json_From_to_cloudflare_logpush_email_security_alerts_from_fb9cca97 field: json.From target_field: cloudflare_logpush.email_security_alerts.from ignore_missing: true + - append: + field: email.from.address + tag: append_from_into_email_from_address + value: '{{{cloudflare_logpush.email_security_alerts.from}}}' + if: ctx.cloudflare_logpush?.email_security_alerts?.from != null + allow_duplicates: false - rename: + tag: rename_json_FromName_to_cloudflare_logpush_email_security_alerts_from_name_ec6c39de field: json.FromName target_field: cloudflare_logpush.email_security_alerts.from_name ignore_missing: true - rename: + tag: rename_json_Links_to_cloudflare_logpush_email_security_alerts_links_c0d5311f field: json.Links target_field: cloudflare_logpush.email_security_alerts.links ignore_missing: true - rename: + tag: rename_json_MessageDeliveryMode_to_cloudflare_logpush_email_security_alerts_message_delivery_mode_bdce12b1 field: json.MessageDeliveryMode target_field: cloudflare_logpush.email_security_alerts.message_delivery_mode ignore_missing: true - rename: + tag: rename_json_MessageID_to_cloudflare_logpush_email_security_alerts_message_id_db845350 field: json.MessageID target_field: cloudflare_logpush.email_security_alerts.message_id ignore_missing: true + - set: + field: email.message_id + tag: set_email_message_id + copy_from: cloudflare_logpush.email_security_alerts.message_id + ignore_empty_value: true - rename: + tag: rename_json_Origin_to_cloudflare_logpush_email_security_alerts_origin_0d91ddaf field: json.Origin target_field: cloudflare_logpush.email_security_alerts.origin ignore_missing: true - rename: + tag: rename_json_OriginalSender_to_cloudflare_logpush_email_security_alerts_original_sender_c6da8814 field: json.OriginalSender target_field: cloudflare_logpush.email_security_alerts.original_sender ignore_missing: true - rename: + tag: rename_json_ReplyTo_to_cloudflare_logpush_email_security_alerts_reply_to_0912c860 field: json.ReplyTo target_field: cloudflare_logpush.email_security_alerts.reply_to ignore_missing: true + - append: + field: email.reply_to.address + tag: append_reply_to_into_email_reply_to_address + value: '{{{cloudflare_logpush.email_security_alerts.reply_to}}}' + if: ctx.cloudflare_logpush?.email_security_alerts?.reply_to != null + allow_duplicates: false - rename: + tag: rename_json_ReplyToName_to_cloudflare_logpush_email_security_alerts_reply_to_name_67e356fb field: json.ReplyToName target_field: cloudflare_logpush.email_security_alerts.reply_to_name ignore_missing: true - rename: + tag: rename_json_SMTPEnvelopeFrom_to_cloudflare_logpush_email_security_alerts_smtp_envelope_from_3d667d21 field: json.SMTPEnvelopeFrom target_field: cloudflare_logpush.email_security_alerts.smtp_envelope_from ignore_missing: true + - set: + field: email.sender.address + tag: set_email_sender_address_from_smtp_envelope_from + copy_from: cloudflare_logpush.email_security_alerts.smtp_envelope_from + ignore_empty_value: true - rename: + tag: rename_json_SMTPEnvelopeTo_to_cloudflare_logpush_email_security_alerts_smtp_envelope_to_78d640cb field: json.SMTPEnvelopeTo target_field: cloudflare_logpush.email_security_alerts.smtp_envelope_to ignore_missing: true @@ -144,67 +207,112 @@ processors: field: json.SMTPHeloServerIP target_field: cloudflare_logpush.email_security_alerts.smtp_helo_server_ip tag: convert_smtp_helo_server_ip + if: ctx.json?.SMTPHeloServerIP != '' type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_98cee656 field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_SMTPHeloServerIPAsName_to_cloudflare_logpush_email_security_alerts_smtp_helo_server_ip_as_name_b961a980 field: json.SMTPHeloServerIPAsName target_field: cloudflare_logpush.email_security_alerts.smtp_helo_server_ip_as_name ignore_missing: true - rename: + tag: rename_json_SMTPHeloServerIPAsNumber_to_cloudflare_logpush_email_security_alerts_smtp_helo_server_ip_as_number_65e39794 field: json.SMTPHeloServerIPAsNumber target_field: cloudflare_logpush.email_security_alerts.smtp_helo_server_ip_as_number ignore_missing: true - rename: + tag: rename_json_SMTPHeloServerIPGeo_to_cloudflare_logpush_email_security_alerts_smtp_helo_server_ip_geo_ec006311 field: json.SMTPHeloServerIPGeo target_field: cloudflare_logpush.email_security_alerts.smtp_helo_server_ip_geo ignore_missing: true - rename: + tag: rename_json_SMTPHeloServerName_to_cloudflare_logpush_email_security_alerts_smtp_helo_server_name_9f512ad6 field: json.SMTPHeloServerName target_field: cloudflare_logpush.email_security_alerts.smtp_helo_server_name ignore_missing: true - rename: + tag: rename_json_Subject_to_cloudflare_logpush_email_security_alerts_subject_5da0dbf7 field: json.Subject target_field: cloudflare_logpush.email_security_alerts.subject ignore_missing: true + - set: + field: email.subject + tag: set_email_subject + copy_from: cloudflare_logpush.email_security_alerts.subject + ignore_empty_value: true - rename: + tag: rename_json_ThreatCategories_to_cloudflare_logpush_email_security_alerts_threat_categories_c839555e field: json.ThreatCategories target_field: cloudflare_logpush.email_security_alerts.threat_categories ignore_missing: true - rename: + tag: rename_json_To_to_cloudflare_logpush_email_security_alerts_to_4cac6e15 field: json.To target_field: cloudflare_logpush.email_security_alerts.to ignore_missing: true + - foreach: + tag: foreach_cloudflare_logpush_email_security_alerts_to_207f4a7c + field: cloudflare_logpush.email_security_alerts.to + if: ctx.cloudflare_logpush?.email_security_alerts?.to instanceof List + processor: + append: + field: email.to.address + tag: append_to_value_into_email_to_address + value: '{{{_ingest._value}}}' + allow_duplicates: false - rename: + tag: rename_json_ToName_to_cloudflare_logpush_email_security_alerts_to_name_4d463964 field: json.ToName target_field: cloudflare_logpush.email_security_alerts.to_name ignore_missing: true - set: + tag: set_event_kind_ca27f3f7 field: event.kind value: alert if: ctx.cloudflare_logpush?.email_security_alerts?.final_disposition != null && ctx.cloudflare_logpush.email_security_alerts.final_disposition != 'unset' - append: + tag: append_related_user_f58b7b82 field: related.user value: '{{{cloudflare_logpush.email_security_alerts.from}}}' - if: ctx.cloudflare_logpush?.email_security_alerts?.from != null && ctx.cloudflare_logpush.email_security_alerts.from != '' + if: ctx.cloudflare_logpush?.email_security_alerts?.from != null allow_duplicates: false - append: + tag: append_related_user_a2c20d32 field: related.user value: '{{{cloudflare_logpush.email_security_alerts.from_name}}}' - if: ctx.cloudflare_logpush?.email_security_alerts?.from_name != null && ctx.cloudflare_logpush.email_security_alerts.from_name != '' + if: ctx.cloudflare_logpush?.email_security_alerts?.from_name != null allow_duplicates: false - append: + tag: append_related_user_46dd7312 field: related.user value: '{{{cloudflare_logpush.email_security_alerts.smtp_envelope_from}}}' - if: ctx.cloudflare_logpush?.email_security_alerts?.smtp_envelope_from != null && ctx.cloudflare_logpush.email_security_alerts.smtp_envelope_from != '' + if: ctx.cloudflare_logpush?.email_security_alerts?.smtp_envelope_from != null + allow_duplicates: false + - append: + tag: append_related_user_e6d94b26 + field: related.user + value: '{{{cloudflare_logpush.email_security_alerts.original_sender}}}' + if: ctx.cloudflare_logpush?.email_security_alerts?.original_sender != null + allow_duplicates: false + - append: + tag: append_related_user_8ff5f9e2 + field: related.user + value: '{{{cloudflare_logpush.email_security_alerts.reply_to}}}' + if: ctx.cloudflare_logpush?.email_security_alerts?.reply_to != null + allow_duplicates: false + - append: + tag: append_related_user_046348da + field: related.user + value: '{{{cloudflare_logpush.email_security_alerts.reply_to_name}}}' + if: ctx.cloudflare_logpush?.email_security_alerts?.reply_to_name != null allow_duplicates: false - foreach: + tag: foreach_cloudflare_logpush_email_security_alerts_to_968c1ea6 field: cloudflare_logpush.email_security_alerts.to if: ctx.cloudflare_logpush?.email_security_alerts?.to instanceof List processor: @@ -213,6 +321,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_cloudflare_logpush_email_security_alerts_smtp_envelope_to_47da8216 field: cloudflare_logpush.email_security_alerts.smtp_envelope_to if: ctx.cloudflare_logpush?.email_security_alerts?.smtp_envelope_to instanceof List processor: @@ -221,6 +330,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_cloudflare_logpush_email_security_alerts_to_name_e4b51f52 field: cloudflare_logpush.email_security_alerts.to_name if: ctx.cloudflare_logpush?.email_security_alerts?.to_name instanceof List processor: @@ -229,6 +339,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_cloudflare_logpush_email_security_alerts_cc_4f4644fc field: cloudflare_logpush.email_security_alerts.cc if: ctx.cloudflare_logpush?.email_security_alerts?.cc instanceof List processor: @@ -237,6 +348,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_cloudflare_logpush_email_security_alerts_cc_name_dcbdbc3e field: cloudflare_logpush.email_security_alerts.cc_name if: ctx.cloudflare_logpush?.email_security_alerts?.cc_name instanceof List processor: @@ -245,6 +357,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - set: + tag: set_server_address_b86252fa field: server.address copy_from: cloudflare_logpush.email_security_alerts.smtp_helo_server_name ignore_empty_value: true @@ -252,18 +365,22 @@ processors: field: server.address target_field: _tmp.server_ip tag: convert_smtp_helo_server_name_ip + if: ctx.server?.address != '' type: ip ignore_missing: true on_failure: - set: + tag: set_server_domain_02f7f467 field: server.domain copy_from: server.address ignore_empty_value: true - set: + tag: set__tmp_links_65cfc504 field: _tmp.links copy_from: cloudflare_logpush.email_security_alerts.links ignore_empty_value: true - foreach: + tag: foreach__tmp_links_a23458c9 field: _tmp.links if: ctx._tmp?.links instanceof List processor: @@ -275,11 +392,9 @@ processors: on_failure: - append: field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach__tmp_links_94875667 field: _tmp.links if: ctx._tmp?.links instanceof List processor: @@ -288,6 +403,7 @@ processors: value: '{{{_ingest._value.domain}}}' allow_duplicates: false - script: + tag: script_5526f769 lang: painless if: ctx.related?.user instanceof List || (ctx.server?.domain != null) source: |- @@ -313,19 +429,29 @@ processors: } } ctx.related.hosts = domains; + on_failure: + - append: + tag: append_error_message_ec62ca07 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_server_ip_2b83be7f field: server.ip copy_from: cloudflare_logpush.email_security_alerts.smtp_helo_server_ip ignore_empty_value: true - append: + tag: append_related_ip_e49c0078 field: related.ip - value: '{{{server.ip}}}' - if: ctx.server?.ip != null + value: '{{{cloudflare_logpush.email_security_alerts.smtp_helo_server_ip}}}' + if: ctx.cloudflare_logpush?.email_security_alerts?.smtp_helo_server_ip != null allow_duplicates: false - geoip: + tag: geoip_server_ip_to_server_geo_6cc613c2 field: server.ip target_field: server.geo + ignore_missing: true - geoip: + tag: geoip_server_ip_to_server_as_ed2798db database_file: GeoLite2-ASN.mmdb field: server.ip target_field: server.as @@ -334,21 +460,26 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_server_as_asn_to_server_as_number_f46ba339 field: server.as.asn target_field: server.as.number ignore_missing: true - rename: + tag: rename_server_as_organization_name_to_server_as_organization_name_a7e512d7 field: server.as.organization_name target_field: server.as.organization.name ignore_missing: true - script: lang: painless + tag: script_build_file_and_email_attachments if: ctx.cloudflare_logpush?.email_security_alerts?.attachments instanceof List source: |- def files = []; + def emailAttachments = []; def hashes = new HashSet(); for (def a: ctx.cloudflare_logpush.email_security_alerts.attachments) { def f = [:]; + def efa = [:]; if (a.Name != null) { f.name = a.Name; } @@ -378,13 +509,26 @@ processors: f.hash = fh; } files.add(f); + efa.put('file', f); + emailAttachments.add(efa); for (def h: fh.entrySet()) { hashes.add(h.getValue()); } } ctx.file = files; + if (emailAttachments.size() != 0) { + if (ctx.email == null) { + ctx.email = [:]; + } + ctx.email.attachments = emailAttachments; + } ctx.related = ctx.related ?: [:]; ctx.related.hash = hashes; + on_failure: + - append: + tag: append_error_message_c4bee641 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: tag: remove_json_conf_tmp @@ -394,35 +538,53 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_duplicate_custom_cloudflare_email_fields field: - cloudflare_logpush.email_security_alerts.timestamp - cloudflare_logpush.email_security_alerts.smtp_helo_server_ip + - cloudflare_logpush.email_security_alerts.from + - cloudflare_logpush.email_security_alerts.to + - cloudflare_logpush.email_security_alerts.cc + - cloudflare_logpush.email_security_alerts.subject + - cloudflare_logpush.email_security_alerts.message_id + - cloudflare_logpush.email_security_alerts.reply_to + - cloudflare_logpush.email_security_alerts.smtp_envelope_from if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); - set: field: event.kind - value: pipeline_error tag: set_pipeline_error_into_event_kind + value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -430,12 +592,10 @@ processors: on_failure: - append: field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml index 7e7644d1e6c..a0bdbbd5ce6 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Firewall Event logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,30 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Datetime_712ea5a9 + field: json.Datetime + type: long + if: ctx.json?.Datetime != null && ctx.json.Datetime != '' + ignore_failure: true - script: lang: painless tag: painless_datetime_to_milli - if: ctx.json?.Datetime != null + if: ctx.json?.Datetime != null && ctx.json.Datetime instanceof Number source: | - try { - long t; - if (ctx.json.Datetime instanceof String) { - t = Long.parseLong(ctx.json.Datetime); - } else if (ctx.json.Datetime instanceof Number) { - t = (long)(ctx.json.Datetime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Datetime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Datetime = t*(long)(1e3) - } + long t = (long)(ctx.json.Datetime); + if (t > (long)(1e18)) { + ctx.json.Datetime = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Datetime = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_6318b156 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Datetime_138a0e6b field: json.Datetime if: ctx.json?.Datetime != null && ctx.json.Datetime != '' formats: @@ -62,102 +77,124 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_ef827e11 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_firewall_event_timestamp_98eb2c1e field: cloudflare_logpush.firewall_event.timestamp copy_from: '@timestamp' ignore_empty_value: true - rename: + tag: rename_json_Action_to_cloudflare_logpush_firewall_event_action_143ea91b field: json.Action target_field: cloudflare_logpush.firewall_event.action ignore_missing: true - set: + tag: set_event_action_e4c1757e field: event.action copy_from: cloudflare_logpush.firewall_event.action ignore_empty_value: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - rename: + tag: rename_json_ClientRequestMethod_to_cloudflare_logpush_firewall_event_client_request_method_5f681fbf field: json.ClientRequestMethod target_field: cloudflare_logpush.firewall_event.client.request.method ignore_missing: true - rename: + tag: rename_json_ContentScanObjResults_to_cloudflare_logpush_firewall_event_content_scan_results_da351a8b field: json.ContentScanObjResults target_field: cloudflare_logpush.firewall_event.content_scan.results ignore_missing: true - convert: + tag: convert_json_ContentScanObjSizes_to_cloudflare_logpush_firewall_event_content_scan_sizes_8280ca40 field: json.ContentScanObjSizes target_field: cloudflare_logpush.firewall_event.content_scan.sizes type: long ignore_missing: true on_failure: - append: + tag: append_error_message_951aac2a field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ContentScanObjTypes_to_cloudflare_logpush_firewall_event_content_scan_types_55800203 field: json.ContentScanObjTypes target_field: cloudflare_logpush.firewall_event.content_scan.types ignore_missing: true - set: + tag: set_http_request_method_9615ebe7 field: http.request.method copy_from: cloudflare_logpush.firewall_event.client.request.method ignore_empty_value: true - convert: + tag: convert_json_EdgeResponseStatus_to_cloudflare_logpush_firewall_event_edge_response_status_8ff95d40 field: json.EdgeResponseStatus target_field: cloudflare_logpush.firewall_event.edge.response.status - if: ctx.json?.EdgeResponseStatus != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_291d5fa2 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_http_response_status_code_80e2f001 field: http.response.status_code copy_from: cloudflare_logpush.firewall_event.edge.response.status ignore_empty_value: true - rename: + tag: rename_json_RuleID_to_cloudflare_logpush_firewall_event_rule_id_c313f3c5 field: json.RuleID target_field: cloudflare_logpush.firewall_event.rule.id ignore_missing: true - set: + tag: set_rule_id_df8c5850 field: rule.id copy_from: cloudflare_logpush.firewall_event.rule.id ignore_empty_value: true - rename: + tag: rename_json_Description_to_cloudflare_logpush_firewall_event_rule_description_0ce0fc91 field: json.Description target_field: cloudflare_logpush.firewall_event.rule.description ignore_missing: true - set: + tag: set_rule_description_f88f8340 field: rule.description copy_from: cloudflare_logpush.firewall_event.rule.description ignore_empty_value: true - convert: + tag: convert_json_ClientASN_to_cloudflare_logpush_firewall_event_client_asn_value_6f2186db field: json.ClientASN target_field: cloudflare_logpush.firewall_event.client.asn.value - if: ctx.json?.ClientASN != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2fec2505 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_as_number_3e029828 field: source.as.number copy_from: cloudflare_logpush.firewall_event.client.asn.value ignore_empty_value: true - rename: + tag: rename_json_ClientCountry_to_cloudflare_logpush_firewall_event_client_country_c56768f5 field: json.ClientCountry target_field: cloudflare_logpush.firewall_event.client.country ignore_missing: true - set: + tag: set_source_geo_country_iso_code_e3426c91 field: source.geo.country_iso_code copy_from: cloudflare_logpush.firewall_event.client.country ignore_empty_value: true - convert: + tag: convert_json_ClientIP_to_cloudflare_logpush_firewall_event_client_ip_4fa09aa0 field: json.ClientIP target_field: cloudflare_logpush.firewall_event.client.ip if: ctx.json?.ClientIP != '' @@ -165,175 +202,225 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_574a271e field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_134d8e5a field: source.ip copy_from: cloudflare_logpush.firewall_event.client.ip ignore_empty_value: true - rename: + tag: rename_json_ClientASNDescription_to_cloudflare_logpush_firewall_event_client_asn_description_55ba93ab field: json.ClientASNDescription target_field: cloudflare_logpush.firewall_event.client.asn.description ignore_missing: true - rename: + tag: rename_json_ClientIPClass_to_cloudflare_logpush_firewall_event_client_ip_class_f46c33fa field: json.ClientIPClass target_field: cloudflare_logpush.firewall_event.client.ip_class ignore_missing: true - rename: + tag: rename_json_ClientRefererHost_to_cloudflare_logpush_firewall_event_client_referer_host_bf194225 field: json.ClientRefererHost target_field: cloudflare_logpush.firewall_event.client.referer.host ignore_missing: true - rename: + tag: rename_json_ClientRefererPath_to_cloudflare_logpush_firewall_event_client_referer_path_63db6fdb field: json.ClientRefererPath target_field: cloudflare_logpush.firewall_event.client.referer.path ignore_missing: true - rename: + tag: rename_json_ClientRefererQuery_to_cloudflare_logpush_firewall_event_client_referer_query_1dcfdcdd field: json.ClientRefererQuery target_field: cloudflare_logpush.firewall_event.client.referer.query ignore_missing: true - rename: + tag: rename_json_ClientRefererScheme_to_cloudflare_logpush_firewall_event_client_referer_scheme_704ecab7 field: json.ClientRefererScheme target_field: cloudflare_logpush.firewall_event.client.referer.scheme ignore_missing: true - set: + tag: set_url_domain_72a6f745 field: url.domain copy_from: json.ClientRequestHost ignore_empty_value: true if: ctx.url?.domain == null - rename: + tag: rename_json_ClientRequestHost_to_cloudflare_logpush_firewall_event_client_request_host_8cf01305 field: json.ClientRequestHost target_field: cloudflare_logpush.firewall_event.client.request.host ignore_missing: true - set: + tag: set_url_path_b062c440 field: url.path copy_from: json.ClientRequestPath ignore_empty_value: true if: ctx.url?.path == null - rename: + tag: rename_json_ClientRequestPath_to_cloudflare_logpush_firewall_event_client_request_path_87996df7 field: json.ClientRequestPath target_field: cloudflare_logpush.firewall_event.client.request.path ignore_missing: true - rename: + tag: rename_json_ClientRequestProtocol_to_cloudflare_logpush_firewall_event_client_request_protocol_99f3ee29 field: json.ClientRequestProtocol target_field: cloudflare_logpush.firewall_event.client.request.protocol ignore_missing: true - dissect: field: cloudflare_logpush.firewall_event.client.request.protocol + tag: dissect_client_request_protocol pattern: '%{network.protocol}/%{http.version}' ignore_missing: true ignore_failure: true - lowercase: + tag: lowercase_network_protocol_49872259 field: network.protocol ignore_missing: true - set: + tag: set_url_query_a659fb29 field: url.query copy_from: json.ClientRequestQuery ignore_empty_value: true if: ctx.url?.query == null - script: - description: Trim leading '?' in query if it exists. - lang: painless - source: | - ctx.url.query = ctx.url.query.substring(1); - if: ctx.url?.query instanceof String && ctx.url.query.startsWith('?') + tag: script_d0ff083b + description: Trim leading '?' in query if it exists. + lang: painless + source: | + ctx.url.query = ctx.url.query.substring(1); + if: ctx.url?.query instanceof String && ctx.url.query.startsWith('?') + on_failure: + - append: + tag: append_error_message_81170cd9 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ClientRequestQuery_to_cloudflare_logpush_firewall_event_client_request_query_c26d538d field: json.ClientRequestQuery target_field: cloudflare_logpush.firewall_event.client.request.query ignore_missing: true - rename: + tag: rename_json_ClientRequestScheme_to_cloudflare_logpush_firewall_event_client_request_scheme_0adf8f33 field: json.ClientRequestScheme target_field: cloudflare_logpush.firewall_event.client.request.scheme ignore_missing: true - set: + tag: set_url_scheme_4a92982f field: url.scheme copy_from: cloudflare_logpush.firewall_event.client.request.scheme ignore_empty_value: true - user_agent: + tag: user_agent_json_ClientRequestUserAgent_a0e18c70 field: json.ClientRequestUserAgent if: ctx.json?.ClientRequestUserAgent != '' ignore_failure: true - rename: + tag: rename_json_ClientRequestUserAgent_to_cloudflare_logpush_firewall_event_client_request_user_agent_f438ab23 field: json.ClientRequestUserAgent target_field: cloudflare_logpush.firewall_event.client.request.user.agent ignore_missing: true - rename: + tag: rename_json_EdgeColoCode_to_cloudflare_logpush_firewall_event_edge_colo_code_8d5606b1 field: json.EdgeColoCode target_field: cloudflare_logpush.firewall_event.edge.colo.code ignore_missing: true - rename: + tag: rename_json_Kind_to_cloudflare_logpush_firewall_event_kind_7b726d4b field: json.Kind target_field: cloudflare_logpush.firewall_event.kind ignore_missing: true - rename: + tag: rename_json_LeakedCredentialCheckResult_to_cloudflare_logpush_firewall_event_leaked_credential_check_1e3c7db0 field: json.LeakedCredentialCheckResult target_field: cloudflare_logpush.firewall_event.leaked_credential_check ignore_missing: true - convert: + tag: convert_json_MatchIndex_to_cloudflare_logpush_firewall_event_match_index_5e39c8a7 field: json.MatchIndex target_field: cloudflare_logpush.firewall_event.match_index - if: ctx.json?.MatchIndex != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_72c3a219 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_Metadata_to_cloudflare_logpush_firewall_event_meta_data_2ca55072 field: json.Metadata target_field: cloudflare_logpush.firewall_event.meta_data ignore_missing: true - convert: + tag: convert_json_OriginResponseStatus_to_cloudflare_logpush_firewall_event_origin_response_status_8b00a29c field: json.OriginResponseStatus target_field: cloudflare_logpush.firewall_event.origin.response.status - if: ctx.json?.OriginResponseStatus != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_eeddaf16 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_OriginatorRayID_to_cloudflare_logpush_firewall_event_origin_ray_id_5be4f8a5 field: json.OriginatorRayID target_field: cloudflare_logpush.firewall_event.origin.ray.id ignore_missing: true - rename: + tag: rename_json_Ref_to_cloudflare_logpush_firewall_event_ref_59cbb419 field: json.Ref target_field: cloudflare_logpush.firewall_event.ref ignore_missing: true - rename: + tag: rename_json_RayID_to_cloudflare_logpush_firewall_event_ray_id_ce479357 field: json.RayID target_field: cloudflare_logpush.firewall_event.ray.id ignore_missing: true - set: + tag: set_event_id_24bc1dca field: event.id copy_from: cloudflare_logpush.firewall_event.ray.id ignore_empty_value: true - rename: + tag: rename_json_Source_to_cloudflare_logpush_firewall_event_source_24749817 field: json.Source target_field: cloudflare_logpush.firewall_event.source ignore_missing: true - rename: + tag: rename_json_FraudUserID_to_cloudflare_logpush_firewall_event_fraud_user_id_20fee20c + field: json.FraudUserID + target_field: cloudflare_logpush.firewall_event.fraud.user_id + ignore_missing: true + - rename: + tag: rename_json_ZoneName_to_cloudflare_logpush_firewall_event_zone_name_641ba6e1 field: json.ZoneName target_field: cloudflare_logpush.firewall_event.zone.name ignore_missing: true - append: - field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null - allow_duplicates: false - ignore_failure: true - - append: + tag: append_related_hosts_57f6b6d2 field: related.hosts value: '{{{cloudflare_logpush.firewall_event.client.referer.host}}}' if: ctx.cloudflare_logpush?.firewall_event?.client?.referer?.host != null allow_duplicates: false - ignore_failure: true - append: + tag: append_related_hosts_cbc2ad5e field: related.hosts value: '{{{cloudflare_logpush.firewall_event.client.request.host}}}' if: ctx.cloudflare_logpush?.firewall_event?.client?.request?.host != null allow_duplicates: false - ignore_failure: true + - append: + tag: append_related_user_4ab7e6c3 + field: related.user + value: '{{{cloudflare_logpush.firewall_event.fraud.user_id}}}' + if: ctx.cloudflare_logpush?.firewall_event?.fraud?.user_id != null + allow_duplicates: false + - append: + tag: append_related_ip_1ba796df + field: related.ip + value: '{{{cloudflare_logpush.firewall_event.client.ip}}}' + if: ctx.cloudflare_logpush?.firewall_event?.client?.ip != null + allow_duplicates: false - remove: tag: remove_json_conf field: @@ -341,6 +428,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_001998de field: - cloudflare_logpush.firewall_event.timestamp - cloudflare_logpush.firewall_event.action @@ -352,32 +440,51 @@ processors: - cloudflare_logpush.firewall_event.client.country - cloudflare_logpush.firewall_event.client.ip if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml index cb39c59fab8..17a3a73f9b0 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Gateway DNS logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,29 +45,31 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# ECS fields + # ECS fields + - convert: + tag: convert_json_Datetime_712ea5a9 + field: json.Datetime + type: long + if: ctx.json?.Datetime != null && ctx.json.Datetime != '' + ignore_failure: true - script: lang: painless tag: painless_datetime_to_milli - if: ctx.json?.Datetime != null + if: ctx.json?.Datetime != null && ctx.json.Datetime instanceof Number source: | - try { - long t; - if (ctx.json.Datetime instanceof String) { - t = Long.parseLong(ctx.json.Datetime); - } else if (ctx.json.Datetime instanceof Number) { - t = (long)(ctx.json.Datetime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Datetime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Datetime = t*(long)(1e3) - } + long t = (long)(ctx.json.Datetime); + if (t > (long)(1e18)) { + ctx.json.Datetime = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Datetime = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_6318b156 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Datetime_138a0e6b field: json.Datetime if: ctx.json?.Datetime != null && ctx.json.Datetime != '' formats: @@ -63,50 +78,75 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_ef827e11 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_gateway_dns_timestamp_dbb62f77 field: cloudflare_logpush.gateway_dns.timestamp copy_from: "@timestamp" + ignore_empty_value: true - rename: + tag: rename_json_DeviceID_to_cloudflare_logpush_gateway_dns_host_id_0913a274 field: json.DeviceID target_field: cloudflare_logpush.gateway_dns.host.id ignore_missing: true - set: + tag: set_host_id_a0ae6001 field: host.id copy_from: cloudflare_logpush.gateway_dns.host.id ignore_empty_value: true + - set: + field: device.id + tag: set_device_id_from_host_id + copy_from: cloudflare_logpush.gateway_dns.host.id + ignore_empty_value: true - rename: + tag: rename_json_DeviceName_to_cloudflare_logpush_gateway_dns_host_name_f807b974 field: json.DeviceName target_field: cloudflare_logpush.gateway_dns.host.name ignore_missing: true - set: + tag: set_host_name_569f7319 field: host.name copy_from: cloudflare_logpush.gateway_dns.host.name ignore_empty_value: true - rename: + tag: rename_json_Email_to_cloudflare_logpush_gateway_dns_user_email_4b03f587 field: json.Email target_field: cloudflare_logpush.gateway_dns.user.email ignore_missing: true - set: + tag: set_user_email_1716d357 field: user.email copy_from: cloudflare_logpush.gateway_dns.user.email ignore_empty_value: true - - rename: + - convert: + tag: convert_json_DstIP_to_cloudflare_logpush_gateway_dns_destination_ip_ecdcc802 field: json.DstIP target_field: cloudflare_logpush.gateway_dns.destination.ip + if: ctx.json?.DstIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_f5c17844 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_ip_7bf16b8b field: destination.ip copy_from: cloudflare_logpush.gateway_dns.destination.ip ignore_empty_value: true -# Geo enrichment (destination IP) + # Geo enrichment (destination IP) - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -115,97 +155,123 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_DstPort_to_cloudflare_logpush_gateway_dns_destination_port_989b11a2 field: json.DstPort target_field: cloudflare_logpush.gateway_dns.destination.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_f27ea7b0 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port_31b4e3b7 field: destination.port copy_from: cloudflare_logpush.gateway_dns.destination.port ignore_empty_value: true - rename: + tag: rename_json_Protocol_to_cloudflare_logpush_gateway_dns_protocol_57d684da field: json.Protocol target_field: cloudflare_logpush.gateway_dns.protocol ignore_missing: true - set: + tag: set_network_protocol_f955c405 field: network.protocol copy_from: cloudflare_logpush.gateway_dns.protocol ignore_empty_value: true - rename: + tag: rename_json_QueryName_to_cloudflare_logpush_gateway_dns_question_name_34684116 field: json.QueryName target_field: cloudflare_logpush.gateway_dns.question.name ignore_missing: true - set: + tag: set_dns_question_name_172294a2 field: dns.question.name copy_from: cloudflare_logpush.gateway_dns.question.name ignore_empty_value: true - rename: + tag: rename_json_QueryTypeName_to_cloudflare_logpush_gateway_dns_question_type_016cf433 field: json.QueryTypeName target_field: cloudflare_logpush.gateway_dns.question.type ignore_missing: true - set: + tag: set_dns_question_type_68d1ca0e field: dns.question.type copy_from: cloudflare_logpush.gateway_dns.question.type ignore_empty_value: true -# Set event.outcome based on the response code + # Set event.outcome based on the response code - set: + tag: set_event_outcome_73b064e5 field: event.outcome value: success if: ctx.json?.RCode != null && ctx.json?.RCode == 0 - set: + tag: set_event_outcome_a3475405 field: event.outcome value: failure if: ctx.json?.RCode != null && ctx.json?.RCode > 0 - convert: + tag: convert_json_RCode_to_cloudflare_logpush_gateway_dns_response_code_7b17891c field: json.RCode target_field: cloudflare_logpush.gateway_dns.response_code - if: ctx.json?.RCode != '' type: string ignore_missing: true on_failure: - append: + tag: append_error_message_6e0077a6 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_dns_response_code_d271d506 field: dns.response_code copy_from: cloudflare_logpush.gateway_dns.response_code + ignore_empty_value: true - rename: + tag: rename_json_RData_to_cloudflare_logpush_gateway_dns_answers_9bdd6d3f field: json.RData target_field: cloudflare_logpush.gateway_dns.answers ignore_missing: true - set: + tag: set_dns_answers_ab8b7c56 field: dns.answers copy_from: cloudflare_logpush.gateway_dns.answers ignore_empty_value: true - - rename: - field: json.ResolvedIPs - target_field: cloudflare_logpush.gateway_dns.resolved_ip - ignore_missing: true - - set: - field: dns.resolved_ip - copy_from: cloudflare_logpush.gateway_dns.resolved_ip - ignore_empty_value: true - - rename: + - convert: + tag: convert_json_SrcIP_to_cloudflare_logpush_gateway_dns_source_ip_7ccd4ee9 field: json.SrcIP target_field: cloudflare_logpush.gateway_dns.source.ip + if: ctx.json?.SrcIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_f8d0dcdf + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_bb6bfb91 field: source.ip copy_from: cloudflare_logpush.gateway_dns.source.ip ignore_empty_value: true -# Geo enrichment (source IP) + # Geo enrichment (source IP) - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -214,302 +280,532 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_SrcPort_to_cloudflare_logpush_gateway_dns_source_port_66e84e2a field: json.SrcPort target_field: cloudflare_logpush.gateway_dns.source.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_4811dc74 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_port_57acad71 field: source.port copy_from: cloudflare_logpush.gateway_dns.source.port + ignore_empty_value: true - rename: + tag: rename_json_TimeZone_to_cloudflare_logpush_gateway_dns_timezone_665792ac field: json.TimeZone target_field: cloudflare_logpush.gateway_dns.timezone ignore_missing: true - set: + tag: set_event_timezone_417ae119 field: event.timezone copy_from: cloudflare_logpush.gateway_dns.timezone + ignore_empty_value: true - rename: + tag: rename_json_UserID_to_cloudflare_logpush_gateway_dns_user_id_c4e5da4a field: json.UserID target_field: cloudflare_logpush.gateway_dns.user.id ignore_missing: true - set: + tag: set_user_id_f156da41 field: user.id copy_from: cloudflare_logpush.gateway_dns.user.id -# Custom fields + ignore_empty_value: true + # Custom fields - rename: + tag: rename_json_ColoCode_to_cloudflare_logpush_gateway_dns_colo_code_9e8b055e field: json.ColoCode target_field: cloudflare_logpush.gateway_dns.colo.code ignore_missing: true - - rename: + - convert: + tag: convert_json_ColoID_to_cloudflare_logpush_gateway_dns_colo_id_39837983 field: json.ColoID target_field: cloudflare_logpush.gateway_dns.colo.id + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_b1d1e385 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_AccountID_to_cloudflare_logpush_gateway_dns_account_id_6d1ff20d field: json.AccountID target_field: cloudflare_logpush.gateway_dns.account_id ignore_missing: true - - rename: + - convert: + tag: convert_json_ApplicationID_to_cloudflare_logpush_gateway_dns_application_id_fa29f01c field: json.ApplicationID target_field: cloudflare_logpush.gateway_dns.application_id + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_d3d6d276 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ApplicationName_to_cloudflare_logpush_gateway_dns_application_name_ba9cfc41 field: json.ApplicationName target_field: cloudflare_logpush.gateway_dns.application_name ignore_missing: true - convert: + tag: convert_json_AuthoritativeNameServerIPs_to_cloudflare_logpush_gateway_dns_authoritative_name_server_ip_bbb3d55e field: json.AuthoritativeNameServerIPs target_field: cloudflare_logpush.gateway_dns.authoritative_name_server_ip + if: ctx.json?.AuthoritativeNameServerIPs != '' type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_bfa7f478 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_CNAMECategoryIDs_to_cloudflare_logpush_gateway_dns_cname_category_ids_0bde2b01 field: json.CNAMECategoryIDs target_field: cloudflare_logpush.gateway_dns.cname_category.ids type: string ignore_missing: true - rename: + tag: rename_json_CNAMECategoryNames_to_cloudflare_logpush_gateway_dns_cname_category_names_7a0914db field: json.CNAMECategoryNames target_field: cloudflare_logpush.gateway_dns.cname_category.names ignore_missing: true - rename: + tag: rename_json_CNAMEs_to_cloudflare_logpush_gateway_dns_cname_68bb9d35 field: json.CNAMEs target_field: cloudflare_logpush.gateway_dns.cname ignore_missing: true - rename: + tag: rename_json_CNAMEsReversed_to_cloudflare_logpush_gateway_dns_cname_reversed_a3bb7c08 field: json.CNAMEsReversed target_field: cloudflare_logpush.gateway_dns.cname_reversed ignore_missing: true - convert: + tag: convert_json_CustomResolveDurationMs_to_cloudflare_logpush_gateway_dns_custom_resolver_duration_milli_b7d3c3e6 field: json.CustomResolveDurationMs target_field: cloudflare_logpush.gateway_dns.custom_resolver.duration_milli type: long ignore_missing: true on_failure: - append: + tag: append_error_message_13fe9df8 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_CustomResolverAddress_to_cloudflare_logpush_gateway_dns_custom_resolver_address_25998f09 field: json.CustomResolverAddress target_field: cloudflare_logpush.gateway_dns.custom_resolver.address ignore_missing: true - rename: + tag: rename_json_CustomResolverPolicyID_to_cloudflare_logpush_gateway_dns_custom_resolver_policy_ids_51fbe982 field: json.CustomResolverPolicyID target_field: cloudflare_logpush.gateway_dns.custom_resolver.policy.ids ignore_missing: true - rename: + tag: rename_json_CustomResolverPolicyName_to_cloudflare_logpush_gateway_dns_custom_resolver_policy_names_fd7d6c8a field: json.CustomResolverPolicyName target_field: cloudflare_logpush.gateway_dns.custom_resolver.policy.names ignore_missing: true - rename: + tag: rename_json_CustomResolverResponse_to_cloudflare_logpush_gateway_dns_custom_resolver_response_6c1c4d8d field: json.CustomResolverResponse target_field: cloudflare_logpush.gateway_dns.custom_resolver.response ignore_missing: true - rename: + tag: rename_json_DoHSubdomain_to_cloudflare_logpush_gateway_dns_doh_subdomain_0ad0b205 field: json.DoHSubdomain target_field: cloudflare_logpush.gateway_dns.doh_subdomain ignore_missing: true - rename: + tag: rename_json_DoTSubdomain_to_cloudflare_logpush_gateway_dns_dot_subdomain_e962eec5 field: json.DoTSubdomain target_field: cloudflare_logpush.gateway_dns.dot_subdomain ignore_missing: true - convert: + tag: convert_json_EDEErrors_to_cloudflare_logpush_gateway_dns_extended_dns_error_codes_fdab34d0 field: json.EDEErrors target_field: cloudflare_logpush.gateway_dns.extended_dns_error_codes type: string ignore_missing: true - convert: + tag: convert_json_InitialCategoryIDs_to_cloudflare_logpush_gateway_dns_initial_category_ids_461fbfd1 field: json.InitialCategoryIDs target_field: cloudflare_logpush.gateway_dns.initial_category.ids type: string ignore_missing: true - rename: + tag: rename_json_InitialCategoryNames_to_cloudflare_logpush_gateway_dns_initial_category_names_cfbbf7ff field: json.InitialCategoryNames target_field: cloudflare_logpush.gateway_dns.initial_category.names ignore_missing: true - convert: + tag: convert_json_IsResponseCached_to_cloudflare_logpush_gateway_dns_is_response_cached_9e2213e7 field: json.IsResponseCached target_field: cloudflare_logpush.gateway_dns.is_response_cached type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_38a8e939 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_Location_to_cloudflare_logpush_gateway_dns_location_name_eccbdcc3 field: json.Location target_field: cloudflare_logpush.gateway_dns.location.name ignore_missing: true - rename: + tag: rename_json_LocationID_to_cloudflare_logpush_gateway_dns_location_id_2bba31fa field: json.LocationID target_field: cloudflare_logpush.gateway_dns.location.id ignore_missing: true - - rename: + - convert: + tag: convert_json_MatchedCategoryIDs_to_cloudflare_logpush_gateway_dns_matched_category_ids_6b8f0369 field: json.MatchedCategoryIDs target_field: cloudflare_logpush.gateway_dns.matched.category.ids + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_29853823 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_MatchedCategoryNames_to_cloudflare_logpush_gateway_dns_matched_category_names_6ebe505e field: json.MatchedCategoryNames target_field: cloudflare_logpush.gateway_dns.matched.category.names ignore_missing: true - convert: + tag: convert_json_MatchedIndicatorFeedIDs_to_cloudflare_logpush_gateway_dns_matched_indicator_feed_ids_d9c12ab1 field: json.MatchedIndicatorFeedIDs target_field: cloudflare_logpush.gateway_dns.matched.indicator_feed.ids type: string ignore_missing: true - rename: + tag: rename_json_MatchedIndicatorFeedNames_to_cloudflare_logpush_gateway_dns_matched_indicator_feed_names_deac207d field: json.MatchedIndicatorFeedNames target_field: cloudflare_logpush.gateway_dns.matched.indicator_feed.names ignore_missing: true - rename: + tag: rename_json_PolicyID_to_cloudflare_logpush_gateway_dns_policy_id_d10336d0 field: json.PolicyID target_field: cloudflare_logpush.gateway_dns.policy.id ignore_missing: true - rename: + tag: rename_json_PolicyName_to_cloudflare_logpush_gateway_dns_policy_name_5c572998 field: json.PolicyName target_field: cloudflare_logpush.gateway_dns.policy.name ignore_missing: true - append: + tag: append_cloudflare_logpush_gateway_dns_policy_name_2036581b field: cloudflare_logpush.gateway_dns.policy.name value: '{{{json.Policy}}}' if: ctx.cloudflare_logpush?.gateway_dns?.policy?.name != null && ctx.json?.Policy != null && ctx.json.Policy != '' && ctx.json.Policy != ctx.cloudflare_logpush?.gateway_dns?.policy?.name allow_duplicates: false - rename: + tag: rename_json_Policy_to_cloudflare_logpush_gateway_dns_policy_name_de9b6002 field: json.Policy target_field: cloudflare_logpush.gateway_dns.policy.name if: ctx.cloudflare_logpush?.gateway_dns?.policy?.name == null ignore_missing: true - - rename: + - convert: + tag: convert_json_QueryCategoryIDs_to_cloudflare_logpush_gateway_dns_question_category_ids_f0e3e443 field: json.QueryCategoryIDs target_field: cloudflare_logpush.gateway_dns.question.category.ids + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_3bd801b1 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_QueryID_to_cloudflare_logpush_gateway_dns_question_id_2f8c08b6 field: json.QueryID target_field: cloudflare_logpush.gateway_dns.question.id ignore_missing: true - - rename: + - convert: + tag: convert_json_QueryIndicatorFeedIDs_to_cloudflare_logpush_gateway_dns_question_indicator_feed_ids_3cb6caac field: json.QueryIndicatorFeedIDs target_field: cloudflare_logpush.gateway_dns.question.indicator_feed.ids + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_2062839a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_QueryIndicatorFeedNames_to_cloudflare_logpush_gateway_dns_question_indicator_feed_names_8336b1f3 field: json.QueryIndicatorFeedNames target_field: cloudflare_logpush.gateway_dns.question.indicator_feed.names ignore_missing: true - rename: + tag: rename_json_QueryCategoryNames_to_cloudflare_logpush_gateway_dns_question_category_names_e7ffd8d2 field: json.QueryCategoryNames target_field: cloudflare_logpush.gateway_dns.question.category.names ignore_missing: true - rename: + tag: rename_json_QueryNameReversed_to_cloudflare_logpush_gateway_dns_question_reversed_4f968253 field: json.QueryNameReversed target_field: cloudflare_logpush.gateway_dns.question.reversed ignore_missing: true - - rename: + - convert: + tag: convert_json_QuerySize_to_cloudflare_logpush_gateway_dns_question_size_21021d69 field: json.QuerySize target_field: cloudflare_logpush.gateway_dns.question.size + type: long ignore_missing: true - - rename: + on_failure: + - append: + tag: append_error_message_b4fbb57f + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_QueryType_to_cloudflare_logpush_gateway_dns_question_type_id_ace9ad8b field: json.QueryType target_field: cloudflare_logpush.gateway_dns.question.type_id + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_26728071 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_ResolvedIPCategoryIDs_to_cloudflare_logpush_gateway_dns_resolved_ip_details_category_ids_03060dca field: json.ResolvedIPCategoryIDs target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.category.ids type: string ignore_missing: true - rename: + tag: rename_json_ResolvedIPCategoryNames_to_cloudflare_logpush_gateway_dns_resolved_ip_details_category_names_158080c6 field: json.ResolvedIPCategoryNames target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.category.names ignore_missing: true - rename: + tag: rename_json_ResolvedIPContinentCodes_to_cloudflare_logpush_gateway_dns_resolved_ip_details_continent_codes_c77efd69 field: json.ResolvedIPContinentCodes target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.continent_codes ignore_missing: true - rename: + tag: rename_json_ResolvedIPCountryCodes_to_cloudflare_logpush_gateway_dns_resolved_ip_details_country_codes_5019bd29 field: json.ResolvedIPCountryCodes target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.country_codes ignore_missing: true - convert: + tag: convert_json_ResolvedIPs_to_cloudflare_logpush_gateway_dns_resolved_ip_details_ips_55b80890 field: json.ResolvedIPs target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.ips + if: ctx.json?.ResolvedIPs != '' type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_60fa7d8e field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + tag: set_dns_resolved_ip_e8621df1 + field: dns.resolved_ip + copy_from: cloudflare_logpush.gateway_dns.resolved_ip_details.ips + ignore_empty_value: true - rename: + tag: rename_json_ResolverPolicyID_to_cloudflare_logpush_gateway_dns_resolver_policy_ids_f7d8c48b field: json.ResolverPolicyID target_field: cloudflare_logpush.gateway_dns.resolver.policy.ids ignore_missing: true - rename: + tag: rename_json_ResolverPolicyName_to_cloudflare_logpush_gateway_dns_resolver_policy_names_033b4dc3 field: json.ResolverPolicyName target_field: cloudflare_logpush.gateway_dns.resolver.policy.names ignore_missing: true - rename: + tag: rename_json_ResolverDecision_to_cloudflare_logpush_gateway_dns_resolver_decision_911c662d field: json.ResolverDecision target_field: cloudflare_logpush.gateway_dns.resolver_decision ignore_missing: true - rename: + tag: rename_json_ResourceRecords_to_cloudflare_logpush_gateway_dns_resource_records_object_43f55d40 field: json.ResourceRecords target_field: cloudflare_logpush.gateway_dns.resource_records.object ignore_missing: true - rename: + tag: rename_json_ResourceRecordsJSON_to_cloudflare_logpush_gateway_dns_resource_records_json_47fc6b61 field: json.ResourceRecordsJSON target_field: cloudflare_logpush.gateway_dns.resource_records.json ignore_missing: true - rename: + tag: rename_json_SrcIPContinentCode_to_cloudflare_logpush_gateway_dns_source_id_continent_code_c0059dd3 field: json.SrcIPContinentCode target_field: cloudflare_logpush.gateway_dns.source_id.continent_code ignore_missing: true - rename: + tag: rename_json_SrcIPCountryCode_to_cloudflare_logpush_gateway_dns_source_id_country_code_8a74743f field: json.SrcIPCountryCode target_field: cloudflare_logpush.gateway_dns.source_id.country_code ignore_missing: true - rename: + tag: rename_json_TimeZoneInferredMethod_to_cloudflare_logpush_gateway_dns_timezone_inferred_method_7bdd7c84 field: json.TimeZoneInferredMethod target_field: cloudflare_logpush.gateway_dns.timezone_inferred_method ignore_missing: true -# Create related fields - - append: - field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null - allow_duplicates: false - - append: - field: related.ip - value: '{{{destination.ip}}}' - if: ctx.destination?.ip != null - allow_duplicates: false + - rename: + tag: rename_json_InitialResolvedIPs_to_cloudflare_logpush_gateway_dns_initial_resolved_ips_466876a4 + field: json.InitialResolvedIPs + target_field: cloudflare_logpush.gateway_dns.initial_resolved_ips + ignore_missing: true + - convert: + tag: convert_json_InternalDNSDurationMs_to_cloudflare_logpush_gateway_dns_internal_dns_duration_ms_5cf064a7 + field: json.InternalDNSDurationMs + target_field: cloudflare_logpush.gateway_dns.internal_dns.duration_ms + type: long + ignore_missing: true + on_failure: + - append: + tag: append_error_message_5ebffd81 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + tag: rename_json_InternalDNSFallbackStrategy_to_cloudflare_logpush_gateway_dns_internal_dns_fallback_strategy_520cff78 + field: json.InternalDNSFallbackStrategy + target_field: cloudflare_logpush.gateway_dns.internal_dns.fallback_strategy + ignore_missing: true + - convert: + tag: convert_json_InternalDNSRCode_to_cloudflare_logpush_gateway_dns_internal_dns_rcode_e0a7366e + field: json.InternalDNSRCode + target_field: cloudflare_logpush.gateway_dns.internal_dns.rcode + type: long + ignore_missing: true + on_failure: + - append: + tag: append_error_message_e22099b8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + tag: rename_json_InternalDNSViewID_to_cloudflare_logpush_gateway_dns_internal_dns_view_id_fa155a94 + field: json.InternalDNSViewID + target_field: cloudflare_logpush.gateway_dns.internal_dns.view_id + ignore_missing: true + - rename: + tag: rename_json_InternalDNSZoneID_to_cloudflare_logpush_gateway_dns_internal_dns_zone_id_f43b780c + field: json.InternalDNSZoneID + target_field: cloudflare_logpush.gateway_dns.internal_dns.zone_id + ignore_missing: true + - convert: + tag: convert_json_QueryApplicationIDs_to_cloudflare_logpush_gateway_dns_question_application_ids_526fb0ad + field: json.QueryApplicationIDs + target_field: cloudflare_logpush.gateway_dns.question.application.ids + type: long + ignore_missing: true + on_failure: + - append: + tag: append_error_message_8c46f9cb + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + tag: rename_json_QueryApplicationNames_to_cloudflare_logpush_gateway_dns_question_application_names_11808228 + field: json.QueryApplicationNames + target_field: cloudflare_logpush.gateway_dns.question.application.names + ignore_missing: true + - rename: + tag: rename_json_RedirectTargetURI_to_cloudflare_logpush_gateway_dns_redirect_target_uri_8da36c04 + field: json.RedirectTargetURI + target_field: cloudflare_logpush.gateway_dns.redirect_target_uri + ignore_missing: true + - rename: + tag: rename_json_RegistrationID_to_cloudflare_logpush_gateway_dns_registration_id_c2a72a3b + field: json.RegistrationID + target_field: cloudflare_logpush.gateway_dns.registration_id + ignore_missing: true + - convert: + tag: convert_json_RequestContextCategoryIDs_to_cloudflare_logpush_gateway_dns_request_context_category_ids_14d330e1 + field: json.RequestContextCategoryIDs + target_field: cloudflare_logpush.gateway_dns.request_context_category.ids + type: long + ignore_missing: true + on_failure: + - append: + tag: append_error_message_a7e24a1b + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + tag: rename_json_RequestContextCategoryNames_to_cloudflare_logpush_gateway_dns_request_context_category_names_e9b6b904 + field: json.RequestContextCategoryNames + target_field: cloudflare_logpush.gateway_dns.request_context_category.names + ignore_missing: true + # Create related fields - append: + tag: append_related_hosts_eb427c61 field: related.hosts - value: "{{{host.id}}}" - if: ctx.host?.id != null + value: '{{{cloudflare_logpush.gateway_dns.host.id}}}' + if: ctx.cloudflare_logpush?.gateway_dns?.host?.id != null allow_duplicates: false - append: + tag: append_related_hosts_221cc33d field: related.hosts - value: "{{{host.name}}}" - if: ctx.host?.name != null + value: '{{{cloudflare_logpush.gateway_dns.host.name}}}' + if: ctx.cloudflare_logpush?.gateway_dns?.host?.name != null allow_duplicates: false - append: + tag: append_related_user_869af0d5 field: related.user - value: '{{{user.id}}}' - if: ctx.user?.id != null + value: '{{{cloudflare_logpush.gateway_dns.user.email}}}' + if: ctx.cloudflare_logpush?.gateway_dns?.user?.email != null allow_duplicates: false - append: + tag: append_related_user_2d15a575 field: related.user - value: '{{{user.email}}}' - if: ctx.user?.email != null + value: '{{{cloudflare_logpush.gateway_dns.user.id}}}' + if: ctx.cloudflare_logpush?.gateway_dns?.user?.id != null + allow_duplicates: false + - append: + tag: append_related_ip_78b56e23 + field: related.ip + value: '{{{cloudflare_logpush.gateway_dns.destination.ip}}}' + if: ctx.cloudflare_logpush?.gateway_dns?.destination?.ip != null allow_duplicates: false -# Clean resulting event + - append: + tag: append_related_ip_3a09431f + field: related.ip + value: '{{{cloudflare_logpush.gateway_dns.source.ip}}}' + if: ctx.cloudflare_logpush?.gateway_dns?.source?.ip != null + allow_duplicates: false + - foreach: + field: cloudflare_logpush.gateway_dns.resolved_ip_details.ips + if: ctx.cloudflare_logpush?.gateway_dns?.resolved_ip_details?.ips instanceof List + tag: foreach_gateway_dns_resolved_ips_to_related_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false + - foreach: + field: cloudflare_logpush.gateway_dns.authoritative_name_server_ip + if: ctx.cloudflare_logpush?.gateway_dns?.authoritative_name_server_ip instanceof List + tag: foreach_gateway_dns_authoritative_ns_ip_to_related_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false + # Clean resulting event - remove: tag: remove_json_conf field: @@ -517,6 +813,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_4771d562 field: - cloudflare_logpush.gateway_dns.timestamp - cloudflare_logpush.gateway_dns.host.id @@ -528,39 +825,58 @@ processors: - cloudflare_logpush.gateway_dns.question.type - cloudflare_logpush.gateway_dns.response_code - cloudflare_logpush.gateway_dns.answers - - cloudflare_logpush.gateway_dns.resolved_ip + - cloudflare_logpush.gateway_dns.resolved_ip_details.ips - cloudflare_logpush.gateway_dns.source.ip - cloudflare_logpush.gateway_dns.source.port - cloudflare_logpush.gateway_dns.timezone - cloudflare_logpush.gateway_dns.user.id - cloudflare_logpush.gateway_dns.user.email if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml index 7a1f3e8d028..2de47bfc043 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml @@ -2,43 +2,59 @@ description: Pipeline for parsing Cloudflare Gateway HTTP logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - append: + tag: append_event_type_f178ce9b field: event.type value: allowed allow_duplicates: false if: ctx.json?.Action != '' && ctx.json?.Action == 'allow' - append: + tag: append_event_type_0e4bad26 field: event.type value: allowed allow_duplicates: false if: ctx.json?.Action != '' && ctx.json?.Action == 'bypass' - append: + tag: append_event_type_267507e8 field: event.type value: denied allow_duplicates: false if: ctx.json?.Action != '' && ctx.json?.Action == 'block' - set: + tag: set_event_kind_de80643c field: event.kind value: event -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -47,29 +63,31 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# ECS fields + # ECS fields + - convert: + tag: convert_json_Datetime_712ea5a9 + field: json.Datetime + type: long + if: ctx.json?.Datetime != null && ctx.json.Datetime != '' + ignore_failure: true - script: lang: painless tag: painless_datetime_to_milli - if: ctx.json?.Datetime != null + if: ctx.json?.Datetime != null && ctx.json.Datetime instanceof Number source: | - try { - long t; - if (ctx.json.Datetime instanceof String) { - t = Long.parseLong(ctx.json.Datetime); - } else if (ctx.json.Datetime instanceof Number) { - t = (long)(ctx.json.Datetime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Datetime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Datetime = t*(long)(1e3) - } + long t = (long)(ctx.json.Datetime); + if (t > (long)(1e18)) { + ctx.json.Datetime = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Datetime = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_6318b156 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Datetime_138a0e6b field: json.Datetime if: ctx.json?.Datetime != null && ctx.json.Datetime != '' formats: @@ -78,50 +96,75 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_ef827e11 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_gateway_http_timestamp_4a452d2e field: cloudflare_logpush.gateway_http.timestamp copy_from: "@timestamp" + ignore_empty_value: true - rename: + tag: rename_json_Action_to_cloudflare_logpush_gateway_http_action_e2c0d6b7 field: json.Action target_field: cloudflare_logpush.gateway_http.action ignore_missing: true - set: + tag: set_event_action_d51e882e field: event.action copy_from: cloudflare_logpush.gateway_http.action ignore_empty_value: true - rename: + tag: rename_json_DeviceID_to_cloudflare_logpush_gateway_http_host_id_65ac8bc9 field: json.DeviceID target_field: cloudflare_logpush.gateway_http.host.id ignore_missing: true - set: + tag: set_host_id_99855bb0 field: host.id copy_from: cloudflare_logpush.gateway_http.host.id ignore_empty_value: true + - set: + field: device.id + tag: set_device_id_from_host_id + copy_from: cloudflare_logpush.gateway_http.host.id + ignore_empty_value: true - rename: + tag: rename_json_DeviceName_to_cloudflare_logpush_gateway_http_host_name_84c5dcb9 field: json.DeviceName target_field: cloudflare_logpush.gateway_http.host.name ignore_missing: true - set: + tag: set_host_name_dd86bf58 field: host.name copy_from: cloudflare_logpush.gateway_http.host.name ignore_empty_value: true - - rename: + - convert: + tag: convert_json_DestinationIP_to_cloudflare_logpush_gateway_http_destination_ip_23ceccc5 field: json.DestinationIP target_field: cloudflare_logpush.gateway_http.destination.ip + if: ctx.json?.DestinationIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_e894888b + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_ip_837c9892 field: destination.ip copy_from: cloudflare_logpush.gateway_http.destination.ip ignore_empty_value: true -# Geo enrichment (destination IP) + # Geo enrichment (destination IP) - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -130,67 +173,102 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_DestinationPort_to_cloudflare_logpush_gateway_http_destination_port_5fe6aa08 field: json.DestinationPort target_field: cloudflare_logpush.gateway_http.destination.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_bd7120ce + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port_f3bb2b4e field: destination.port copy_from: cloudflare_logpush.gateway_http.destination.port ignore_empty_value: true - rename: + tag: rename_json_HTTPMethod_to_cloudflare_logpush_gateway_http_request_method_6172f61c field: json.HTTPMethod target_field: cloudflare_logpush.gateway_http.request.method ignore_missing: true - set: + tag: set_http_request_method_69168ea6 field: http.request.method copy_from: cloudflare_logpush.gateway_http.request.method ignore_empty_value: true - - rename: + - convert: + tag: convert_json_HTTPStatusCode_to_cloudflare_logpush_gateway_http_response_status_code_fbea1526 field: json.HTTPStatusCode target_field: cloudflare_logpush.gateway_http.response.status_code + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_ffa89038 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_http_response_status_code_63af1e22 field: http.response.status_code copy_from: cloudflare_logpush.gateway_http.response.status_code ignore_empty_value: true - rename: + tag: rename_json_HTTPVersion_to_cloudflare_logpush_gateway_http_request_version_c7ca5f2e field: json.HTTPVersion target_field: cloudflare_logpush.gateway_http.request.version ignore_missing: true - set: + tag: set_http_version_2a76bccd field: http.version copy_from: cloudflare_logpush.gateway_http.request.version ignore_empty_value: true - rename: + tag: rename_json_Referer_to_cloudflare_logpush_gateway_http_request_referrer_9148694c field: json.Referer target_field: cloudflare_logpush.gateway_http.request.referrer ignore_missing: true - set: + tag: set_http_request_referrer_e0995d6e field: http.request.referrer copy_from: cloudflare_logpush.gateway_http.request.referrer ignore_empty_value: true - - rename: + - convert: + tag: convert_json_SourceIP_to_cloudflare_logpush_gateway_http_source_ip_d990d89a field: json.SourceIP target_field: cloudflare_logpush.gateway_http.source.ip + if: ctx.json?.SourceIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_fb6be4ac + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_2da8b268 field: source.ip copy_from: cloudflare_logpush.gateway_http.source.ip ignore_empty_value: true -# Geo enrichment (source IP) + # Geo enrichment (source IP) - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -199,235 +277,367 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_SourcePort_to_cloudflare_logpush_gateway_http_source_port_af671f3a field: json.SourcePort target_field: cloudflare_logpush.gateway_http.source.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_02e6a2b8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_port_0321a1a0 field: source.port copy_from: cloudflare_logpush.gateway_http.source.port ignore_empty_value: true - rename: + tag: rename_json_URL_to_cloudflare_logpush_gateway_http_url_6b9de1e1 field: json.URL target_field: cloudflare_logpush.gateway_http.url ignore_missing: true - uri_parts: + tag: uri_parts_cloudflare_logpush_gateway_http_url_to_url_a1cdd978 field: cloudflare_logpush.gateway_http.url target_field: url if: ctx.cloudflare_logpush?.gateway_http?.url != null - rename: + tag: rename_json_UserAgent_to_cloudflare_logpush_gateway_http_user_agent_a8bd2af6 field: json.UserAgent target_field: cloudflare_logpush.gateway_http.user_agent ignore_missing: true - set: + tag: set_user_agent_original_55a5449d field: user_agent.original copy_from: cloudflare_logpush.gateway_http.user_agent ignore_empty_value: true - rename: + tag: rename_json_UserID_to_cloudflare_logpush_gateway_http_user_id_651716a9 field: json.UserID target_field: cloudflare_logpush.gateway_http.user.id ignore_missing: true - set: + tag: set_user_id_b987eb50 field: user.id copy_from: cloudflare_logpush.gateway_http.user.id ignore_empty_value: true - rename: + tag: rename_json_Email_to_cloudflare_logpush_gateway_http_user_email_75bd2046 field: json.Email target_field: cloudflare_logpush.gateway_http.user.email ignore_missing: true - set: + tag: set_user_email_515524fe field: user.email copy_from: cloudflare_logpush.gateway_http.user.email ignore_empty_value: true -# Custom fields + # Custom fields - rename: + tag: rename_json_AccountID_to_cloudflare_logpush_gateway_http_account_id_96473be8 field: json.AccountID target_field: cloudflare_logpush.gateway_http.account_id ignore_missing: true - convert: + tag: convert_json_ApplicationIDs_to_cloudflare_logpush_gateway_http_application_ids_e5c4d709 field: json.ApplicationIDs target_field: cloudflare_logpush.gateway_http.application.ids type: string ignore_missing: true - rename: + tag: rename_json_ApplicationNames_to_cloudflare_logpush_gateway_http_application_names_f73b8713 field: json.ApplicationNames target_field: cloudflare_logpush.gateway_http.application.names ignore_missing: true - rename: + tag: rename_json_BlockedFileHash_to_cloudflare_logpush_gateway_http_blocked_file_hash_10149780 field: json.BlockedFileHash target_field: cloudflare_logpush.gateway_http.blocked_file.hash ignore_missing: true + - set: + field: file.hash.sha256 + tag: set_file_hash_sha256_from_blocked_file_hash + copy_from: cloudflare_logpush.gateway_http.blocked_file.hash + ignore_empty_value: true - rename: + tag: rename_json_BlockedFileName_to_cloudflare_logpush_gateway_http_blocked_file_name_92a883de field: json.BlockedFileName target_field: cloudflare_logpush.gateway_http.blocked_file.name ignore_missing: true + - set: + field: file.name + tag: set_file_name_from_blocked_file_name + copy_from: cloudflare_logpush.gateway_http.blocked_file.name + ignore_empty_value: true - rename: + tag: rename_json_BlockedFileReason_to_cloudflare_logpush_gateway_http_blocked_file_reason_d2532038 field: json.BlockedFileReason target_field: cloudflare_logpush.gateway_http.blocked_file.reason ignore_missing: true - - rename: + - convert: + tag: convert_json_BlockedFileSize_to_cloudflare_logpush_gateway_http_blocked_file_size_fce5c4fb field: json.BlockedFileSize target_field: cloudflare_logpush.gateway_http.blocked_file.size + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_7cdb1a61 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: file.size + tag: set_file_size_from_blocked_file_size + copy_from: cloudflare_logpush.gateway_http.blocked_file.size + ignore_empty_value: true - rename: + tag: rename_json_BlockedFileType_to_cloudflare_logpush_gateway_http_blocked_file_type_813e4450 field: json.BlockedFileType target_field: cloudflare_logpush.gateway_http.blocked_file.type ignore_missing: true + - set: + field: file.extension + tag: set_file_extension_from_blocked_file_type + copy_from: cloudflare_logpush.gateway_http.blocked_file.type + ignore_empty_value: true - rename: + tag: rename_json_CategoryIDs_to_cloudflare_logpush_gateway_http_category_ids_3d6fd593 field: json.CategoryIDs target_field: cloudflare_logpush.gateway_http.category.ids ignore_missing: true - rename: + tag: rename_json_CategoryNames_to_cloudflare_logpush_gateway_http_category_names_40f1817b field: json.CategoryNames target_field: cloudflare_logpush.gateway_http.category.names ignore_missing: true - rename: + tag: rename_json_DestinationIPContinentCode_to_cloudflare_logpush_gateway_http_destination_ip_continent_code_9fdde7b1 field: json.DestinationIPContinentCode target_field: cloudflare_logpush.gateway_http.destination_ip.continent_code ignore_missing: true - rename: + tag: rename_json_DestinationIPCountryCode_to_cloudflare_logpush_gateway_http_destination_ip_country_code_4de8d4c5 field: json.DestinationIPCountryCode target_field: cloudflare_logpush.gateway_http.destination_ip.country_code ignore_missing: true - rename: + tag: rename_json_DownloadedFileNames_to_cloudflare_logpush_gateway_http_downloaded_files_0a00963d field: json.DownloadedFileNames target_field: cloudflare_logpush.gateway_http.downloaded_files ignore_missing: true - rename: + tag: rename_json_DownloadMatchedDlpProfileEntries_to_cloudflare_logpush_gateway_http_download_matched_dlp_profile_entries_849ef096 field: json.DownloadMatchedDlpProfileEntries target_field: cloudflare_logpush.gateway_http.download_matched_dlp.profile_entries ignore_missing: true - rename: + tag: rename_json_DownloadMatchedDlpProfiles_to_cloudflare_logpush_gateway_http_download_matched_dlp_profiles_634a15d7 field: json.DownloadMatchedDlpProfiles target_field: cloudflare_logpush.gateway_http.download_matched_dlp.profiles ignore_missing: true - rename: + tag: rename_json_FileInfo_to_cloudflare_logpush_gateway_http_file_info_0862bd42 field: json.FileInfo target_field: cloudflare_logpush.gateway_http.file_info ignore_missing: true - rename: + tag: rename_json_ForensicCopyStatus_to_cloudflare_logpush_gateway_http_forensic_copy_status_346706cb field: json.ForensicCopyStatus target_field: cloudflare_logpush.gateway_http.forensic_copy_status ignore_missing: true - rename: + tag: rename_json_HTTPHost_to_cloudflare_logpush_gateway_http_request_host_bc8a245c field: json.HTTPHost target_field: cloudflare_logpush.gateway_http.request.host ignore_missing: true - - rename: + - convert: + tag: convert_json_IsIsolated_to_cloudflare_logpush_gateway_http_isolated_27dc5d60 field: json.IsIsolated target_field: cloudflare_logpush.gateway_http.isolated + type: boolean ignore_missing: true + on_failure: + - append: + tag: append_error_message_f1743fba + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_PolicyID_to_cloudflare_logpush_gateway_http_policy_id_24ccffdd field: json.PolicyID target_field: cloudflare_logpush.gateway_http.policy.id ignore_missing: true - rename: + tag: rename_json_PolicyName_to_cloudflare_logpush_gateway_http_policy_name_16ffb46d field: json.PolicyName target_field: cloudflare_logpush.gateway_http.policy.name ignore_missing: true - rename: + tag: rename_json_PrivateAppAUD_to_cloudflare_logpush_gateway_http_private_app_aud_4b5ea161 field: json.PrivateAppAUD target_field: cloudflare_logpush.gateway_http.private_app_aud ignore_missing: true - rename: + tag: rename_json_ProxyEndpoint_to_cloudflare_logpush_gateway_http_proxy_endpoint_5f4e91b6 field: json.ProxyEndpoint target_field: cloudflare_logpush.gateway_http.proxy_endpoint ignore_missing: true - convert: + tag: convert_json_Quarantined_to_cloudflare_logpush_gateway_http_quarantined_01064750 field: json.Quarantined target_field: cloudflare_logpush.gateway_http.quarantined type: boolean ignore_missing: true on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_714bb33e + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_RequestID_to_cloudflare_logpush_gateway_http_request_id_ad9f8c34 field: json.RequestID target_field: cloudflare_logpush.gateway_http.request_id ignore_missing: true - rename: + tag: rename_json_SessionID_to_cloudflare_logpush_gateway_http_session_id_ed7f705a field: json.SessionID target_field: cloudflare_logpush.gateway_http.session_id ignore_missing: true - - rename: + - convert: + tag: convert_json_SourceInternalIP_to_cloudflare_logpush_gateway_http_source_internal_ip_b69ce478 field: json.SourceInternalIP target_field: cloudflare_logpush.gateway_http.source.internal_ip + if: ctx.json?.SourceInternalIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_4be2c88e + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_SourceIPContinentCode_to_cloudflare_logpush_gateway_http_source_ip_continent_code_28450e67 field: json.SourceIPContinentCode target_field: cloudflare_logpush.gateway_http.source_ip.continent_code ignore_missing: true - rename: + tag: rename_json_SourceIPCountryCode_to_cloudflare_logpush_gateway_http_source_ip_country_code_548be907 field: json.SourceIPCountryCode target_field: cloudflare_logpush.gateway_http.source_ip.country_code ignore_missing: true - rename: + tag: rename_json_UntrustedCertificateAction_to_cloudflare_logpush_gateway_http_untrusted_certificate_action_717d6185 field: json.UntrustedCertificateAction target_field: cloudflare_logpush.gateway_http.untrusted_certificate_action ignore_missing: true - rename: + tag: rename_json_UploadMatchedDlpProfileEntries_to_cloudflare_logpush_gateway_http_upload_matched_dlp_profile_entries_24d0b066 field: json.UploadMatchedDlpProfileEntries target_field: cloudflare_logpush.gateway_http.upload_matched_dlp.profile_entries ignore_missing: true - rename: + tag: rename_json_UploadMatchedDlpProfiles_to_cloudflare_logpush_gateway_http_upload_matched_dlp_profiles_eefa4a53 field: json.UploadMatchedDlpProfiles target_field: cloudflare_logpush.gateway_http.upload_matched_dlp.profiles ignore_missing: true - rename: + tag: rename_json_UploadedFileNames_to_cloudflare_logpush_gateway_http_uploaded_files_7e0bf6c3 field: json.UploadedFileNames target_field: cloudflare_logpush.gateway_http.uploaded_files ignore_missing: true - rename: + tag: rename_json_VirtualNetworkID_to_cloudflare_logpush_gateway_http_virtual_network_id_3d61e73a field: json.VirtualNetworkID target_field: cloudflare_logpush.gateway_http.virtual_network.id ignore_missing: true - rename: + tag: rename_json_VirtualNetworkName_to_cloudflare_logpush_gateway_http_virtual_network_name_b9ac8c6e field: json.VirtualNetworkName target_field: cloudflare_logpush.gateway_http.virtual_network.name ignore_missing: true -# Create related fields - - append: - field: related.ip - value: "{{{source.ip}}}" - if: ctx.source?.ip != null - allow_duplicates: false + - rename: + tag: rename_json_AppControlInfo_to_cloudflare_logpush_gateway_http_app_control_info_63f61099 + field: json.AppControlInfo + target_field: cloudflare_logpush.gateway_http.app_control_info + ignore_missing: true + - rename: + tag: rename_json_ApplicationStatuses_to_cloudflare_logpush_gateway_http_application_statuses_336d5293 + field: json.ApplicationStatuses + target_field: cloudflare_logpush.gateway_http.application.statuses + ignore_missing: true + - rename: + tag: rename_json_RedirectTargetURI_to_cloudflare_logpush_gateway_http_redirect_target_uri_0db7ee2d + field: json.RedirectTargetURI + target_field: cloudflare_logpush.gateway_http.redirect_target_uri + ignore_missing: true + - rename: + tag: rename_json_RegistrationID_to_cloudflare_logpush_gateway_http_registration_id_ec6607ec + field: json.RegistrationID + target_field: cloudflare_logpush.gateway_http.registration_id + ignore_missing: true + # Create related fields - append: - field: related.ip - value: "{{{destination.ip}}}" - if: ctx.destination?.ip != null + tag: append_related_hash_abefda82 + field: related.hash + value: '{{{cloudflare_logpush.gateway_http.blocked_file.hash}}}' + if: ctx.cloudflare_logpush?.gateway_http?.blocked_file?.hash != null allow_duplicates: false - append: - field: related.ip - value: "{{{cloudflare_logpush.gateway_http.source.internal_ip}}}" - if: ctx.cloudflare_logpush?.gateway_http?.source?.internal_ip != null && ctx.cloudflare_logpush.gateway_http.source.internal_ip != '' + tag: append_related_hosts_673b760f + field: related.hosts + value: '{{{cloudflare_logpush.gateway_http.host.id}}}' + if: ctx.cloudflare_logpush?.gateway_http?.host?.id != null allow_duplicates: false - append: + tag: append_related_hosts_6496a807 field: related.hosts - value: "{{{host.id}}}" - if: ctx.host?.id != null + value: '{{{cloudflare_logpush.gateway_http.host.name}}}' + if: ctx.cloudflare_logpush?.gateway_http?.host?.name != null allow_duplicates: false - append: + tag: append_related_hosts_28f0f35d field: related.hosts - value: "{{{host.name}}}" - if: ctx.host?.name != null + value: '{{{cloudflare_logpush.gateway_http.request.host}}}' + if: ctx.cloudflare_logpush?.gateway_http?.request?.host != null allow_duplicates: false - append: + tag: append_related_user_801fdf05 field: related.user - value: "{{{user.id}}}" - if: ctx.user?.id != null + value: '{{{cloudflare_logpush.gateway_http.user.email}}}' + if: ctx.cloudflare_logpush?.gateway_http?.user?.email != null allow_duplicates: false - append: + tag: append_related_user_405a0451 field: related.user - value: "{{{user.email}}}" - if: ctx.user?.email != null + value: '{{{cloudflare_logpush.gateway_http.user.id}}}' + if: ctx.cloudflare_logpush?.gateway_http?.user?.id != null + allow_duplicates: false + - append: + tag: append_related_ip_866283bb + field: related.ip + value: '{{{cloudflare_logpush.gateway_http.destination.ip}}}' + if: ctx.cloudflare_logpush?.gateway_http?.destination?.ip != null + allow_duplicates: false + - append: + tag: append_related_ip_4cf120e3 + field: related.ip + value: '{{{cloudflare_logpush.gateway_http.source.ip}}}' + if: ctx.cloudflare_logpush?.gateway_http?.source?.ip != null + allow_duplicates: false + - append: + tag: append_related_ip_9173fdeb + field: related.ip + value: '{{{cloudflare_logpush.gateway_http.source.internal_ip}}}' + if: ctx.cloudflare_logpush?.gateway_http?.source?.internal_ip != null allow_duplicates: false - remove: tag: remove_json_conf @@ -436,6 +646,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_7bd52ebd field: - cloudflare_logpush.gateway_http.timestamp - cloudflare_logpush.gateway_http.action @@ -453,33 +664,56 @@ processors: - cloudflare_logpush.gateway_http.user_agent - cloudflare_logpush.gateway_http.user.id - cloudflare_logpush.gateway_http.user.email + - cloudflare_logpush.gateway_http.blocked_file.hash + - cloudflare_logpush.gateway_http.blocked_file.name + - cloudflare_logpush.gateway_http.blocked_file.size + - cloudflare_logpush.gateway_http.blocked_file.type if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log index c1ea4231460..9df6f8f295b 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log @@ -1,3 +1,3 @@ {"AccountID":"e1836771179f98aabb828da5ea69a111","Action":"allowedOnNoRuleMatch","Datetime":"2023-05-19T06:42:57+09:30","DestinationIP":"89.160.20.129","DestinationPort":443,"DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","Email":"user@test.com","OverrideIP":"175.16.199.4","OverridePort":8080,"PolicyID":"85063bec-74cb-4546-85a3-e0cde2cdfda2","PolicyName":"My policy","SNI":"www.elastic.co","SessionID":"5f2d04be-3512-11e8-b467-0ed5f89f718b","SourceIP":"67.43.156.2","SourceInternalIP":"192.168.1.3","SourcePort":47924,"Transport":"tcp","UserID":"166befbb-00e3-5e20-bd6e-27245723949f"} {"AccountID":"e1836771179f98aabb828da5ea69a111","Action":"allowedOnNoRuleMatch","Datetime":1684444377,"DestinationIP":"89.160.20.129","DestinationPort":443,"DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","Email":"user@test.com","OverrideIP":"175.16.199.4","OverridePort":8080,"PolicyID":"85063bec-74cb-4546-85a3-e0cde2cdfda2","PolicyName":"My policy","SNI":"www.elastic.co","SessionID":"5f2d04be-3512-11e8-b467-0ed5f89f718b","SourceIP":"67.43.156.2","SourceInternalIP":"192.168.1.3","SourcePort":47924,"Transport":"tcp","UserID":"166befbb-00e3-5e20-bd6e-27245723949f"} -{"AccountID":"e1836771179f98aabb828da5ea69a111","Action":"allowedOnNoRuleMatch","Datetime":1684444377058000000,"DestinationIP":"89.160.20.129","DestinationPort":443,"DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","Email":"user@test.com","OverrideIP":"175.16.199.4","OverridePort":8080,"PolicyID":"85063bec-74cb-4546-85a3-e0cde2cdfda2","PolicyName":"My policy","SNI":"www.elastic.co","SessionID":"5f2d04be-3512-11e8-b467-0ed5f89f718b","SourceIP":"67.43.156.2","SourceInternalIP":"192.168.1.3","SourcePort":47924,"Transport":"tcp","UserID":"166befbb-00e3-5e20-bd6e-27245723949f"} \ No newline at end of file +{"AccountID":"e1836771179f98aabb828da5ea69a111","Action":"allowedOnNoRuleMatch","Datetime":1684444377058000000,"DestinationIP":"89.160.20.129","DestinationPort":443,"DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","Email":"user@test.com","OverrideIP":"175.16.199.4","OverridePort":8080,"PolicyID":"85063bec-74cb-4546-85a3-e0cde2cdfda2","PolicyName":"My policy","SNI":"www.elastic.co","SessionID":"5f2d04be-3512-11e8-b467-0ed5f89f718b","SourceIP":"67.43.156.2","SourceInternalIP":"192.168.1.3","SourcePort":47924,"Transport":"tcp","UserID":"166befbb-00e3-5e20-bd6e-27245723949f"} diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml index c285bd6c6bb..e15b500f6e9 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Gateway Network logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - set: + tag: set_event_kind_de80643c field: event.kind value: event -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,29 +45,31 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# ECS fields + # ECS fields + - convert: + tag: convert_json_Datetime_712ea5a9 + field: json.Datetime + type: long + if: ctx.json?.Datetime != null && ctx.json.Datetime != '' + ignore_failure: true - script: lang: painless tag: painless_datetime_to_milli - if: ctx.json?.Datetime != null + if: ctx.json?.Datetime != null && ctx.json.Datetime instanceof Number source: | - try { - long t; - if (ctx.json.Datetime instanceof String) { - t = Long.parseLong(ctx.json.Datetime); - } else if (ctx.json.Datetime instanceof Number) { - t = (long)(ctx.json.Datetime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Datetime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Datetime = t*(long)(1e3) - } + long t = (long)(ctx.json.Datetime); + if (t > (long)(1e18)) { + ctx.json.Datetime = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Datetime = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_6318b156 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Datetime_138a0e6b field: json.Datetime if: ctx.json?.Datetime != null && ctx.json.Datetime != '' formats: @@ -63,33 +78,50 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_ef827e11 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_gateway_network_timestamp_9bc80b60 field: cloudflare_logpush.gateway_network.timestamp copy_from: "@timestamp" + ignore_empty_value: true - rename: + tag: rename_json_Action_to_cloudflare_logpush_gateway_network_action_f9131c6f field: json.Action target_field: cloudflare_logpush.gateway_network.action ignore_missing: true - set: + tag: set_event_action_1c1356b0 field: event.action copy_from: cloudflare_logpush.gateway_network.action ignore_empty_value: true - - rename: + - convert: + tag: convert_json_DestinationIP_to_cloudflare_logpush_gateway_network_destination_ip_0aad6e1d field: json.DestinationIP target_field: cloudflare_logpush.gateway_network.destination.ip + if: ctx.json?.DestinationIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_de8a1e7b + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_ip_fc548a68 field: destination.ip copy_from: cloudflare_logpush.gateway_network.destination.ip ignore_empty_value: true -# Geo enrichment (destination IP) + # Geo enrichment (destination IP) - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo + ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -98,70 +130,106 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_DestinationPort_to_cloudflare_logpush_gateway_network_destination_port_4aa33198 field: json.DestinationPort target_field: cloudflare_logpush.gateway_network.destination.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_35aa76b6 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port_b39fa4bc field: destination.port copy_from: cloudflare_logpush.gateway_network.destination.port ignore_empty_value: true - rename: + tag: rename_json_DeviceID_to_cloudflare_logpush_gateway_network_host_id_23bb4a67 field: json.DeviceID target_field: cloudflare_logpush.gateway_network.host.id ignore_missing: true - set: + tag: set_host_id_0fad592e field: host.id copy_from: cloudflare_logpush.gateway_network.host.id ignore_empty_value: true + - set: + field: device.id + tag: set_device_id_from_host_id + copy_from: cloudflare_logpush.gateway_network.host.id + ignore_empty_value: true - rename: + tag: rename_json_DeviceName_to_cloudflare_logpush_gateway_network_host_name_bfa47a8b field: json.DeviceName target_field: cloudflare_logpush.gateway_network.host.name ignore_missing: true - set: + tag: set_host_name_a1136376 field: host.name copy_from: cloudflare_logpush.gateway_network.host.name ignore_empty_value: true - rename: + tag: rename_json_SNI_to_cloudflare_logpush_gateway_network_sni_5dfb7cb7 field: json.SNI target_field: cloudflare_logpush.gateway_network.sni ignore_missing: true - set: + tag: set_tls_client_server_name_43b005ff field: tls.client.server_name copy_from: cloudflare_logpush.gateway_network.sni ignore_empty_value: true - set: + tag: set_destination_domain_7150388e field: destination.domain copy_from: tls.client.server_name ignore_empty_value: true - rename: + tag: rename_json_SessionID_to_cloudflare_logpush_gateway_network_session_id_0f621aa8 field: json.SessionID target_field: cloudflare_logpush.gateway_network.session_id ignore_missing: true - set: + tag: set_event_id_eab92dbf field: event.id copy_from: cloudflare_logpush.gateway_network.session_id ignore_empty_value: true - - rename: + - convert: + tag: convert_json_SourceIP_to_cloudflare_logpush_gateway_network_source_ip_d1c9e2c2 field: json.SourceIP target_field: cloudflare_logpush.gateway_network.source.ip + if: ctx.json?.SourceIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_520fde1c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_a7eaf136 field: source.ip copy_from: cloudflare_logpush.gateway_network.source.ip ignore_empty_value: true -# Geo enrichment (source IP) + # Geo enrichment (source IP) - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo + ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -170,170 +238,233 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_SourcePort_to_cloudflare_logpush_gateway_network_source_port_54df52a0 field: json.SourcePort target_field: cloudflare_logpush.gateway_network.source.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_67801e76 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_port_f2cb7e4e field: source.port copy_from: cloudflare_logpush.gateway_network.source.port ignore_empty_value: true - rename: + tag: rename_json_TransportProtocol_to_cloudflare_logpush_gateway_network_transport_9ad64d3b field: json.TransportProtocol target_field: cloudflare_logpush.gateway_network.transport ignore_missing: true - rename: + tag: rename_json_Transport_to_cloudflare_logpush_gateway_network_transport_b090eb6c field: json.Transport target_field: cloudflare_logpush.gateway_network.transport if: ctx.cloudflare_logpush?.gateway_network?.transport == null ignore_missing: true - set: + tag: set_network_transport_6112981a field: network.transport copy_from: cloudflare_logpush.gateway_network.transport ignore_empty_value: true - rename: + tag: rename_json_UserID_to_cloudflare_logpush_gateway_network_user_id_50554b81 field: json.UserID target_field: cloudflare_logpush.gateway_network.user.id ignore_missing: true - set: + tag: set_user_id_3ad0ce7e field: user.id copy_from: cloudflare_logpush.gateway_network.user.id ignore_empty_value: true - rename: + tag: rename_json_Email_to_cloudflare_logpush_gateway_network_user_email_960d254e field: json.Email target_field: cloudflare_logpush.gateway_network.user.email ignore_missing: true - set: + tag: set_user_email_a33e851c field: user.email copy_from: cloudflare_logpush.gateway_network.user.email ignore_empty_value: true -# Custom fields + # Custom fields - rename: + tag: rename_json_AccountID_to_cloudflare_logpush_gateway_network_account_id_c6306708 field: json.AccountID target_field: cloudflare_logpush.gateway_network.account_id ignore_missing: true - convert: + tag: convert_json_ApplicationIDs_to_cloudflare_logpush_gateway_network_application_ids_541dd06b field: json.ApplicationIDs target_field: cloudflare_logpush.gateway_network.application.ids type: string ignore_missing: true - rename: + tag: rename_json_ApplicationNames_to_cloudflare_logpush_gateway_network_application_names_5b43358b field: json.ApplicationNames target_field: cloudflare_logpush.gateway_network.application.names ignore_missing: true - convert: + tag: convert_json_CategoryIDs_to_cloudflare_logpush_gateway_network_category_ids_bd855f5b field: json.CategoryIDs target_field: cloudflare_logpush.gateway_network.category.ids type: string ignore_missing: true - rename: + tag: rename_json_CategoryNames_to_cloudflare_logpush_gateway_network_category_names_ca527583 field: json.CategoryNames target_field: cloudflare_logpush.gateway_network.category.names ignore_missing: true - rename: + tag: rename_json_DestinationIPContinentCode_to_cloudflare_logpush_gateway_network_destination_ip_continent_code_1cec9049 field: json.DestinationIPContinentCode target_field: cloudflare_logpush.gateway_network.destination_ip.continent_code ignore_missing: true - rename: + tag: rename_json_DestinationIPCountryCode_to_cloudflare_logpush_gateway_network_destination_ip_country_code_c7588d55 field: json.DestinationIPCountryCode target_field: cloudflare_logpush.gateway_network.destination_ip.country_code ignore_missing: true - rename: + tag: rename_json_DetectedProtocol_to_cloudflare_logpush_gateway_network_detected_protocol_53051a82 field: json.DetectedProtocol target_field: cloudflare_logpush.gateway_network.detected_protocol ignore_missing: true - - rename: + - convert: + tag: convert_json_OverrideIP_to_cloudflare_logpush_gateway_network_override_ip_92332523 field: json.OverrideIP target_field: cloudflare_logpush.gateway_network.override.ip + if: ctx.json?.OverrideIP != '' + type: ip ignore_missing: true - - rename: + on_failure: + - append: + tag: append_error_message_fe8bf085 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_OverridePort_to_cloudflare_logpush_gateway_network_override_port_dbb699c6 field: json.OverridePort target_field: cloudflare_logpush.gateway_network.override.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_a2a58a4c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_PolicyID_to_cloudflare_logpush_gateway_network_policy_id_70acd37b field: json.PolicyID target_field: cloudflare_logpush.gateway_network.policy.id ignore_missing: true - rename: + tag: rename_json_PolicyName_to_cloudflare_logpush_gateway_network_policy_name_2532643f field: json.PolicyName target_field: cloudflare_logpush.gateway_network.policy.name ignore_missing: true - rename: + tag: rename_json_ProxyEndpoint_to_cloudflare_logpush_gateway_network_proxy_endpoint_5182661c field: json.ProxyEndpoint target_field: cloudflare_logpush.gateway_network.proxy_endpoint ignore_missing: true - - rename: + - convert: + tag: convert_json_SourceInternalIP_to_cloudflare_logpush_gateway_network_source_internal_ip_21599bc0 field: json.SourceInternalIP target_field: cloudflare_logpush.gateway_network.source.internal_ip + if: ctx.json?.SourceInternalIP != '' + type: ip ignore_missing: true + on_failure: + - append: + tag: append_error_message_4d9e8bbe + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_SourceIPContinentCode_to_cloudflare_logpush_gateway_network_source_ip_continent_code_fddc275d field: json.SourceIPContinentCode target_field: cloudflare_logpush.gateway_network.source_ip.continent_code ignore_missing: true - rename: + tag: rename_json_SourceIPCountryCode_to_cloudflare_logpush_gateway_network_source_ip_country_code_a48881b1 field: json.SourceIPCountryCode target_field: cloudflare_logpush.gateway_network.source_ip.country_code ignore_missing: true - rename: + tag: rename_json_VirtualNetworkID_to_cloudflare_logpush_gateway_network_virtual_network_id_1e540812 field: json.VirtualNetworkID target_field: cloudflare_logpush.gateway_network.virtual_network.id ignore_missing: true - rename: + tag: rename_json_VirtualNetworkName_to_cloudflare_logpush_gateway_network_virtual_network_name_f490f5ce field: json.VirtualNetworkName target_field: cloudflare_logpush.gateway_network.virtual_network.name ignore_missing: true -# Create related fields - - append: - field: related.ip - value: "{{{source.ip}}}" - if: ctx.source?.ip != null - allow_duplicates: false + - rename: + tag: rename_json_RegistrationID_to_cloudflare_logpush_gateway_network_registration_id_f5c57cfc + field: json.RegistrationID + target_field: cloudflare_logpush.gateway_network.registration_id + ignore_missing: true + # Create related fields - append: - field: related.ip - value: "{{{destination.ip}}}" - if: ctx.destination?.ip != null + tag: append_related_hosts_62013c2d + field: related.hosts + value: '{{{cloudflare_logpush.gateway_network.host.id}}}' + if: ctx.cloudflare_logpush?.gateway_network?.host?.id != null allow_duplicates: false - append: - field: related.ip - value: "{{{cloudflare_logpush.gateway_network.override.ip}}}" - if: ctx.cloudflare_logpush?.gateway_network?.override?.ip != null + tag: append_related_hosts_574bb47d + field: related.hosts + value: '{{{cloudflare_logpush.gateway_network.host.name}}}' + if: ctx.cloudflare_logpush?.gateway_network?.host?.name != null allow_duplicates: false - append: - field: related.ip - value: "{{{cloudflare_logpush.gateway_network.source.internal_ip}}}" - if: ctx.cloudflare_logpush?.gateway_network?.source?.internal_ip != null && ctx.cloudflare_logpush.gateway_network.source.internal_ip != '' + tag: append_related_user_a8be630b + field: related.user + value: '{{{cloudflare_logpush.gateway_network.user.email}}}' + if: ctx.cloudflare_logpush?.gateway_network?.user?.email != null allow_duplicates: false - append: - field: related.hosts - value: "{{{destination.domain}}}" - if: ctx.destination?.domain != null + tag: append_related_user_7d59e9b1 + field: related.user + value: '{{{cloudflare_logpush.gateway_network.user.id}}}' + if: ctx.cloudflare_logpush?.gateway_network?.user?.id != null allow_duplicates: false - append: - field: related.hosts - value: "{{{host.id}}}" - if: ctx.host?.id != null + tag: append_related_ip_b61fffa9 + field: related.ip + value: '{{{cloudflare_logpush.gateway_network.destination.ip}}}' + if: ctx.cloudflare_logpush?.gateway_network?.destination?.ip != null allow_duplicates: false - append: - field: related.hosts - value: "{{{host.name}}}" - if: ctx.host?.name != null + tag: append_related_ip_ed97380d + field: related.ip + value: '{{{cloudflare_logpush.gateway_network.override.ip}}}' + if: ctx.cloudflare_logpush?.gateway_network?.override?.ip != null allow_duplicates: false - append: - field: related.user - value: "{{{user.id}}}" - if: ctx.user?.id != null + tag: append_related_ip_dbf0fe73 + field: related.ip + value: '{{{cloudflare_logpush.gateway_network.source.ip}}}' + if: ctx.cloudflare_logpush?.gateway_network?.source?.ip != null allow_duplicates: false - append: - field: related.user - value: "{{{user.email}}}" - if: ctx.user?.email != null + tag: append_related_ip_86a48687 + field: related.ip + value: '{{{cloudflare_logpush.gateway_network.source.internal_ip}}}' + if: ctx.cloudflare_logpush?.gateway_network?.source?.internal_ip != null allow_duplicates: false - remove: tag: remove_json_conf @@ -342,6 +473,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_4cbff841 field: - cloudflare_logpush.gateway_network.timestamp - cloudflare_logpush.gateway_network.action @@ -357,32 +489,51 @@ processors: - cloudflare_logpush.gateway_network.user.id - cloudflare_logpush.gateway_network.user.email if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml index 0d27208026a..9801f7d7cfa 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare HTTP Request logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,52 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_EdgeStartTimestamp_3a05abf9 + field: json.EdgeStartTimestamp + type: long + if: ctx.json?.EdgeStartTimestamp != null && ctx.json.EdgeStartTimestamp != '' + ignore_failure: true + - convert: + tag: convert_json_EdgeEndTimestamp_56e7554c + field: json.EdgeEndTimestamp + type: long + if: ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp != '' + ignore_failure: true + - convert: + tag: convert_json_Datetime_712ea5a9 + field: json.Datetime + type: long + if: ctx.json?.Datetime != null && ctx.json.Datetime != '' + ignore_failure: true - script: lang: painless tag: painless_edge_start_timestamp_to_milli - if: ctx.json?.EdgeStartTimestamp != null source: | - try { - long t; - if (ctx.json.EdgeStartTimestamp instanceof String) { - t = Long.parseLong(ctx.json.EdgeStartTimestamp); - } else if (ctx.json.EdgeStartTimestamp instanceof Number) { - t = (long)(ctx.json.EdgeStartTimestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.EdgeStartTimestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.EdgeStartTimestamp = t*(long)(1e3) + def convertToMillis(long timestamp) { + if (timestamp > (long)(1e18)) { + return timestamp/(long)(1e6) + } else if (timestamp < (long)(1e10)) { + return timestamp*(long)(1e3) } + return timestamp } - catch (Exception e) {} + if (ctx.json?.EdgeStartTimestamp != null && ctx.json.EdgeStartTimestamp instanceof Number) { + ctx.json.EdgeStartTimestamp = convertToMillis(ctx.json.EdgeStartTimestamp); + } + if (ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp instanceof Number) { + ctx.json.EdgeEndTimestamp = convertToMillis(ctx.json.EdgeEndTimestamp); + } + if (ctx.json?.Datetime != null && ctx.json.Datetime instanceof Number) { + ctx.json.Datetime = convertToMillis(ctx.json.Datetime); + } + on_failure: + - append: + tag: append_error_message_edcb660c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_EdgeStartTimestamp_to_cloudflare_logpush_http_request_edge_start_time_6e9e517a field: json.EdgeStartTimestamp if: ctx.json?.EdgeStartTimestamp != null && ctx.json.EdgeStartTimestamp != '' formats: @@ -63,10 +100,27 @@ processors: timezone: UTC target_field: cloudflare_logpush.http_request.edge.start_time on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_0749b228 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_EdgeEndTimestamp_to_cloudflare_logpush_http_request_edge_end_time_b977f372 + field: json.EdgeEndTimestamp + if: ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp != '' + formats: + - UNIX_MS + - ISO8601 + - yyyy-MM-dd'T'HH:mm:ssZ + timezone: UTC + target_field: cloudflare_logpush.http_request.edge.end_time + on_failure: + - append: + tag: append_error_message_d27501b8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + tag: date_json_Datetime_to_cloudflare_logpush_http_request_datetime_9a7be6f2 field: json.Datetime if: ctx.json?.Datetime != null && ctx.json.Datetime != '' formats: @@ -76,52 +130,22 @@ processors: timezone: UTC target_field: cloudflare_logpush.http_request.datetime on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_bb7462c8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_b2955a8a field: '@timestamp' copy_from: cloudflare_logpush.http_request.datetime ignore_empty_value: true - set: + tag: set_timestamp_f04385f0 field: '@timestamp' copy_from: cloudflare_logpush.http_request.edge.start_time ignore_empty_value: true - - script: - lang: painless - tag: painless_edge_end_timestamp_to_milli - if: ctx.json?.EdgeEndTimestamp != null - source: | - try { - long t; - if (ctx.json.EdgeEndTimestamp instanceof String) { - t = Long.parseLong(ctx.json.EdgeEndTimestamp); - } else if (ctx.json.EdgeEndTimestamp instanceof Number) { - t = (long)(ctx.json.EdgeEndTimestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.EdgeEndTimestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.EdgeEndTimestamp = t*(long)(1e3) - } - } - catch (Exception e) {} - - date: - field: json.EdgeEndTimestamp - if: ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp != '' - formats: - - UNIX_MS - - ISO8601 - - yyyy-MM-dd'T'HH:mm:ssZ - timezone: UTC - target_field: cloudflare_logpush.http_request.edge.end_time - on_failure: - - append: - field: error.message - value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_OriginResponseHTTPExpires_to_cloudflare_logpush_http_request_origin_response_http_expires_1c6e26fc field: json.OriginResponseHTTPExpires if: ctx.json?.OriginResponseHTTPExpires != null && ctx.json.OriginResponseHTTPExpires != '' formats: @@ -136,9 +160,11 @@ processors: target_field: cloudflare_logpush.http_request.origin.response.http.expires on_failure: - append: + tag: append_error_message_7907381e field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_OriginResponseHTTPLastModified_to_cloudflare_logpush_http_request_origin_response_http_last_modified_3ed0cc17 field: json.OriginResponseHTTPLastModified if: ctx.json?.OriginResponseHTTPLastModified != null && ctx.json.OriginResponseHTTPLastModified != '' timezone: UTC @@ -153,9 +179,11 @@ processors: target_field: cloudflare_logpush.http_request.origin.response.http.last_modified on_failure: - append: + tag: append_error_message_3701b701 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginIP_to_cloudflare_logpush_http_request_origin_ip_4c213612 field: json.OriginIP target_field: cloudflare_logpush.http_request.origin.ip if: ctx.json?.OriginIP != '' @@ -163,65 +191,78 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7461deb4 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_ip_76755d6d field: destination.ip copy_from: cloudflare_logpush.http_request.origin.ip ignore_empty_value: true - rename: + tag: rename_json_ClientRequestMethod_to_cloudflare_logpush_http_request_client_request_method_f40864a4 field: json.ClientRequestMethod target_field: cloudflare_logpush.http_request.client.request.method ignore_missing: true - set: + tag: set_http_request_method_08f4da24 field: http.request.method copy_from: cloudflare_logpush.http_request.client.request.method ignore_empty_value: true - rename: + tag: rename_json_EdgeResponseContentType_to_cloudflare_logpush_http_request_edge_response_content_type_5fd766bf field: json.EdgeResponseContentType target_field: cloudflare_logpush.http_request.edge.response.content_type ignore_missing: true - set: + tag: set_http_response_mime_type_1f0cd015 field: http.response.mime_type copy_from: cloudflare_logpush.http_request.edge.response.content_type ignore_empty_value: true - convert: + tag: convert_json_EdgeResponseStatus_to_cloudflare_logpush_http_request_edge_response_status_52bdf495 field: json.EdgeResponseStatus target_field: cloudflare_logpush.http_request.edge.response.status - if: ctx.json?.EdgeResponseStatus != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_44de6703 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_http_response_status_code_ae85d3c4 field: http.response.status_code copy_from: cloudflare_logpush.http_request.edge.response.status ignore_empty_value: true - convert: + tag: convert_json_ClientASN_to_cloudflare_logpush_http_request_client_asn_1d5bbc87 field: json.ClientASN target_field: cloudflare_logpush.http_request.client.asn - if: ctx.json?.ClientASN != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0d91873d field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_as_number_bff92218 field: source.as.number copy_from: cloudflare_logpush.http_request.client.asn ignore_empty_value: true - rename: + tag: rename_json_ClientCountry_to_cloudflare_logpush_http_request_client_country_f44be20c field: json.ClientCountry target_field: cloudflare_logpush.http_request.client.country ignore_missing: true - set: + tag: set_source_geo_country_iso_code_1764ada0 field: source.geo.country_iso_code copy_from: cloudflare_logpush.http_request.client.country ignore_empty_value: true - convert: + tag: convert_json_ClientIP_to_cloudflare_logpush_http_request_client_ip_0150280f field: json.ClientIP target_field: cloudflare_logpush.http_request.client.ip if: ctx.json?.ClientIP != '' @@ -229,17 +270,25 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c126a271 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_a2ad6d49 field: source.ip copy_from: cloudflare_logpush.http_request.client.ip ignore_empty_value: true - - rename: + - convert: field: json.BotDetectionIDs target_field: cloudflare_logpush.http_request.bot.detection_ids - tag: rename_bot_detections_ids + tag: convert_bot_detection_ids + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_3a583ff4 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.BotDetectionTags target_field: cloudflare_logpush.http_request.bot.detection_tags @@ -248,14 +297,14 @@ processors: - convert: field: json.BotScore target_field: cloudflare_logpush.http_request.bot.score.value - if: ctx.json?.BotScore != '' tag: convert_bot_score_value type: long ignore_missing: true on_failure: - append: + tag: append_error_message_1c29ba40 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.BotScoreSrc target_field: cloudflare_logpush.http_request.bot.score.src @@ -267,132 +316,173 @@ processors: tag: rename_bot_tags ignore_missing: true - rename: + tag: rename_json_CacheCacheStatus_to_cloudflare_logpush_http_request_cache_status_1744173a field: json.CacheCacheStatus target_field: cloudflare_logpush.http_request.cache.status ignore_missing: true - convert: + tag: convert_json_CacheReserveUsed_to_cloudflare_logpush_http_request_cache_reserve_used_6cf7bbd8 field: json.CacheReserveUsed target_field: cloudflare_logpush.http_request.cache.reserve_used - if: ctx.json?.CacheReserveUsed != null && ctx.json.CacheReserveUsed != '' type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_331e1306 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_CacheResponseBytes_to_cloudflare_logpush_http_request_cache_response_bytes_af475a81 field: json.CacheResponseBytes target_field: cloudflare_logpush.http_request.cache.response.bytes - if: ctx.json?.CacheResponseBytes != null && ctx.json.CacheResponseBytes != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_416e43ef field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_CacheResponseStatus_to_cloudflare_logpush_http_request_cache_response_status_02873249 field: json.CacheResponseStatus target_field: cloudflare_logpush.http_request.cache.response.status - if: ctx.json?.CacheResponseStatus != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2c5f6807 field: error.message - value: '{{{_ingest.on_failure_message}}}' - - rename: + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_CacheTieredFill_to_cloudflare_logpush_http_request_cache_tiered_fill_c15adcf8 field: json.CacheTieredFill target_field: cloudflare_logpush.http_request.cache.tiered_fill + type: boolean ignore_missing: true + on_failure: + - append: + tag: append_error_message_6af6a9d6 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ClientCity_to_cloudflare_logpush_http_request_client_city_c3f90d8e field: json.ClientCity target_field: cloudflare_logpush.http_request.client.city ignore_missing: true - rename: + tag: rename_json_ClientDeviceType_to_cloudflare_logpush_http_request_client_device_type_1a6d51fa field: json.ClientDeviceType target_field: cloudflare_logpush.http_request.client.device.type ignore_missing: true + - set: + field: device.type + tag: set_device_type_from_client_device_type + copy_from: cloudflare_logpush.http_request.client.device.type + ignore_empty_value: true - rename: + tag: rename_json_ClientIPClass_to_cloudflare_logpush_http_request_client_ip_class_5adb6839 field: json.ClientIPClass target_field: cloudflare_logpush.http_request.client.ip_class ignore_missing: true - rename: + tag: rename_json_ClientLatitude_to_cloudflare_logpush_http_request_client_latitude_06f27c28 field: json.ClientLatitude target_field: cloudflare_logpush.http_request.client.latitude ignore_missing: true - rename: + tag: rename_json_ClientLongitude_to_cloudflare_logpush_http_request_client_longitude_b4f7387c field: json.ClientLongitude target_field: cloudflare_logpush.http_request.client.longitude ignore_missing: true - rename: + tag: rename_json_ClientMTLSAuthCertFingerprint_to_cloudflare_logpush_http_request_client_mtls_auth_fingerprint_735ba6be field: json.ClientMTLSAuthCertFingerprint target_field: cloudflare_logpush.http_request.client.mtls.auth.fingerprint ignore_missing: true - rename: + tag: rename_json_ClientMTLSAuthStatus_to_cloudflare_logpush_http_request_client_mtls_auth_status_b0179374 field: json.ClientMTLSAuthStatus target_field: cloudflare_logpush.http_request.client.mtls.auth.status ignore_missing: true - rename: + tag: rename_json_ClientRegionCode_to_cloudflare_logpush_http_request_client_region_code_01e279bb field: json.ClientRegionCode target_field: cloudflare_logpush.http_request.client.region_code ignore_missing: true - convert: + tag: convert_json_ClientRequestBytes_to_cloudflare_logpush_http_request_client_request_bytes_4f8136e9 field: json.ClientRequestBytes target_field: cloudflare_logpush.http_request.client.request.bytes - if: ctx.json?.ClientRequestBytes != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_a30f5c3f field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: http.request.bytes + tag: set_http_request_bytes_from_client_request_bytes + copy_from: cloudflare_logpush.http_request.client.request.bytes + ignore_empty_value: true - rename: + tag: rename_json_ContentScanObjResults_to_cloudflare_logpush_http_request_content_scan_results_9bec8ee2 field: json.ContentScanObjResults target_field: cloudflare_logpush.http_request.content_scan.results ignore_missing: true - convert: + tag: convert_json_ContentScanObjSizes_to_cloudflare_logpush_http_request_content_scan_sizes_a8b04a6d field: json.ContentScanObjSizes target_field: cloudflare_logpush.http_request.content_scan.sizes type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c1f2642b field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ContentScanObjTypes_to_cloudflare_logpush_http_request_content_scan_types_a06b8342 field: json.ContentScanObjTypes target_field: cloudflare_logpush.http_request.content_scan.types ignore_missing: true - rename: + tag: rename_json_LeakedCredentialCheckResult_to_cloudflare_logpush_http_request_leaked_credential_check_d4d0d473 field: json.LeakedCredentialCheckResult target_field: cloudflare_logpush.http_request.leaked_credential_check ignore_missing: true - set: + tag: set_url_scheme_212f97c8 field: url.scheme copy_from: json.ClientRequestScheme ignore_empty_value: true if: ctx.url?.scheme == null - set: + tag: set_url_scheme_26af34b2 field: url.scheme value: https ignore_empty_value: true if: ctx.url?.scheme == null && ctx.cloudflare?.client?.ssl?.protocol != null - set: + tag: set_url_scheme_cb50bce2 field: url.scheme value: http ignore_empty_value: true if: ctx.url?.scheme == null - set: + tag: set_url_domain_72a6f745 field: url.domain copy_from: json.ClientRequestHost ignore_empty_value: true if: ctx.url?.domain == null - set: + tag: set_url_path_b062c440 field: url.path copy_from: json.ClientRequestPath ignore_empty_value: true if: ctx.url?.path == null - set: + tag: set_url_query_a659fb29 field: url.query copy_from: json.ClientRequestQuery ignore_empty_value: true @@ -401,10 +491,7 @@ processors: field: tmp.constructed_uri tag: set_tmp_constructed_uri value: '{{{url.scheme}}}://{{{url.domain}}}{{{json.ClientRequestURI}}}' - if: ctx.json?.ClientRequestURI != null - && ctx.url?.scheme != null - && ctx.url?.domain != null - && ctx.json.ClientRequestURI.startsWith('/') + if: ctx.json?.ClientRequestURI != null && ctx.url?.scheme != null && ctx.url?.domain != null && ctx.json.ClientRequestURI.startsWith('/') - set: field: tmp.constructed_uri tag: set_tmp_constructed_uri_from_ClientRequestURI @@ -416,179 +503,229 @@ processors: ignore_failure: true if: ctx.tmp?.constructed_uri != null - remove: + tag: remove_tmp_constructed_uri_9f65bf9e field: tmp.constructed_uri ignore_missing: true - rename: + tag: rename_json_ClientRequestHost_to_cloudflare_logpush_http_request_client_request_host_d4498d5e field: json.ClientRequestHost target_field: cloudflare_logpush.http_request.client.request.host ignore_missing: true - rename: + tag: rename_json_ClientRequestPath_to_cloudflare_logpush_http_request_client_request_path_f5f75878 field: json.ClientRequestPath target_field: cloudflare_logpush.http_request.client.request.path ignore_missing: true - rename: + tag: rename_json_ClientRequestProtocol_to_cloudflare_logpush_http_request_client_request_protocol_07c944a2 field: json.ClientRequestProtocol target_field: cloudflare_logpush.http_request.client.request.protocol ignore_missing: true - dissect: field: cloudflare_logpush.http_request.client.request.protocol + tag: dissect_client_request_protocol pattern: '%{network.protocol}/%{http.version}' ignore_missing: true ignore_failure: true - lowercase: + tag: lowercase_network_protocol_49872259 field: network.protocol ignore_missing: true - rename: + tag: rename_json_ClientRequestReferer_to_cloudflare_logpush_http_request_client_request_referer_e413e594 field: json.ClientRequestReferer target_field: cloudflare_logpush.http_request.client.request.referer ignore_missing: true - rename: + tag: rename_json_ClientRequestScheme_to_cloudflare_logpush_http_request_client_request_scheme_349d1500 field: json.ClientRequestScheme target_field: cloudflare_logpush.http_request.client.request.scheme ignore_missing: true - rename: + tag: rename_json_ClientRequestSource_to_cloudflare_logpush_http_request_client_request_source_1b28f824 field: json.ClientRequestSource target_field: cloudflare_logpush.http_request.client.request.source ignore_missing: true - rename: + tag: rename_json_ClientRequestURI_to_cloudflare_logpush_http_request_client_request_uri_9f914974 field: json.ClientRequestURI target_field: cloudflare_logpush.http_request.client.request.uri ignore_missing: true - user_agent: + tag: user_agent_json_ClientRequestUserAgent_a0e18c70 field: json.ClientRequestUserAgent if: ctx.json?.ClientRequestUserAgent != '' ignore_failure: true - rename: + tag: rename_json_ClientRequestUserAgent_to_cloudflare_logpush_http_request_client_request_user_agent_712bf2d0 field: json.ClientRequestUserAgent target_field: cloudflare_logpush.http_request.client.request.user.agent ignore_missing: true - rename: + tag: rename_json_ClientSSLCipher_to_cloudflare_logpush_http_request_client_ssl_cipher_91147e80 field: json.ClientSSLCipher target_field: cloudflare_logpush.http_request.client.ssl.cipher ignore_missing: true - set: + tag: set_tls_cipher_d8c6958f field: tls.cipher copy_from: cloudflare_logpush.http_request.client.ssl.cipher ignore_empty_value: true - rename: + tag: rename_json_ClientSSLProtocol_to_cloudflare_logpush_http_request_client_ssl_protocol_146b34d6 field: json.ClientSSLProtocol target_field: cloudflare_logpush.http_request.client.ssl.protocol ignore_missing: true - dissect: field: cloudflare_logpush.http_request.client.ssl.protocol + tag: dissect_client_ssl_protocol pattern: '%{tls.version_protocol}v%{tls.version}' ignore_missing: true ignore_failure: true - lowercase: + tag: lowercase_tls_version_protocol_b840a1f9 field: tls.version_protocol ignore_missing: true - convert: + tag: convert_json_ClientSrcPort_to_cloudflare_logpush_http_request_client_src_port_3d226ea5 field: json.ClientSrcPort target_field: cloudflare_logpush.http_request.client.src.port - if: ctx.json?.ClientSrcPort != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0552caab field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: source.port + tag: set_source_port_from_client_src_port + copy_from: cloudflare_logpush.http_request.client.src.port + ignore_empty_value: true - convert: + tag: convert_json_ClientTCPRTTMs_to_cloudflare_logpush_http_request_client_tcp_rtt_ms_ae6167b8 field: json.ClientTCPRTTMs target_field: cloudflare_logpush.http_request.client.tcp_rtt.ms - if: ctx.json?.ClientTCPRTTMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_d64f5576 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ClientXRequestedWith_to_cloudflare_logpush_http_request_client_xrequested_with_4fbd479f field: json.ClientXRequestedWith target_field: cloudflare_logpush.http_request.client.xrequested_with ignore_missing: true - rename: + tag: rename_json_Cookies_to_cloudflare_logpush_http_request_cookies_1ddf2546 field: json.Cookies target_field: cloudflare_logpush.http_request.cookies ignore_missing: true - - rename: + - convert: + tag: convert_json_EdgeCFConnectingO2O_to_cloudflare_logpush_http_request_edge_cf_connecting_o2o_3c573565 field: json.EdgeCFConnectingO2O target_field: cloudflare_logpush.http_request.edge.cf_connecting_o2o + type: boolean ignore_missing: true + on_failure: + - append: + tag: append_error_message_8a86f07b + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_EdgeColoCode_to_cloudflare_logpush_http_request_edge_colo_code_78f5d638 field: json.EdgeColoCode target_field: cloudflare_logpush.http_request.edge.colo.code ignore_missing: true - convert: + tag: convert_json_EdgeColoID_to_cloudflare_logpush_http_request_edge_colo_id_23567a5d field: json.EdgeColoID target_field: cloudflare_logpush.http_request.edge.colo.id - if: ctx.json?.EdgeColoID != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ee6472bb field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_EdgePathingOp_to_cloudflare_logpush_http_request_edge_pathing_op_d3b079a8 field: json.EdgePathingOp target_field: cloudflare_logpush.http_request.edge.pathing.op ignore_missing: true - rename: + tag: rename_json_EdgePathingSrc_to_cloudflare_logpush_http_request_edge_pathing_src_247c1a48 field: json.EdgePathingSrc target_field: cloudflare_logpush.http_request.edge.pathing.src ignore_missing: true - rename: + tag: rename_json_EdgePathingStatus_to_cloudflare_logpush_http_request_edge_pathing_status_efbc6d66 field: json.EdgePathingStatus target_field: cloudflare_logpush.http_request.edge.pathing.status ignore_missing: true - rename: + tag: rename_json_EdgeRateLimitAction_to_cloudflare_logpush_http_request_edge_rate_limit_action_ee838a82 field: json.EdgeRateLimitAction target_field: cloudflare_logpush.http_request.edge.rate.limit.action ignore_missing: true - convert: + tag: convert_json_EdgeRateLimitID_to_cloudflare_logpush_http_request_edge_rate_limit_id_08f59c5d field: json.EdgeRateLimitID target_field: cloudflare_logpush.http_request.edge.rate.limit.id - if: ctx.json?.EdgeRateLimitID != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2897a9cb field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_EdgeRequestHost_to_cloudflare_logpush_http_request_edge_request_host_92a86166 field: json.EdgeRequestHost target_field: cloudflare_logpush.http_request.edge.request.host ignore_missing: true - convert: + tag: convert_json_EdgeResponseBodyBytes_to_cloudflare_logpush_http_request_edge_response_body_bytes_e749ab58 field: json.EdgeResponseBodyBytes target_field: cloudflare_logpush.http_request.edge.response.body_bytes - if: ctx.json?.EdgeResponseBodyBytes != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_26361976 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_EdgeResponseBytes_to_cloudflare_logpush_http_request_edge_response_bytes_0d7be1cd field: json.EdgeResponseBytes target_field: cloudflare_logpush.http_request.edge.response.bytes - if: ctx.json?.EdgeResponseBytes != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ac846933 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: http.response.bytes + tag: set_http_response_bytes_from_edge_response_bytes + copy_from: cloudflare_logpush.http_request.edge.response.bytes + ignore_empty_value: true - convert: + tag: convert_json_EdgeResponseCompressionRatio_to_cloudflare_logpush_http_request_edge_response_compression_ratio_504133a9 field: json.EdgeResponseCompressionRatio target_field: cloudflare_logpush.http_request.edge.response.compression_ratio - if: ctx.json?.EdgeResponseCompressionRatio != '' type: double ignore_missing: true on_failure: - append: + tag: append_error_message_d444eaf3 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_EdgeServerIP_to_cloudflare_logpush_http_request_edge_server_ip_2a497892 field: json.EdgeServerIP target_field: cloudflare_logpush.http_request.edge.server.ip if: ctx.json?.EdgeServerIP != '' @@ -596,230 +733,273 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b0769278 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_EdgeTimeToFirstByteMs_to_cloudflare_logpush_http_request_edge_time_to_first_byte_ms_6df0c524 field: json.EdgeTimeToFirstByteMs target_field: cloudflare_logpush.http_request.edge.time_to_first_byte.ms - if: ctx.json?.EdgeTimeToFirstByteMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_d0ac58aa field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_SecurityActions_to_cloudflare_logpush_http_request_firewall_matches_action_5da10512 field: json.SecurityActions target_field: cloudflare_logpush.http_request.firewall.matches.action ignore_missing: true - rename: + tag: rename_json_FirewallMatchesActions_to_cloudflare_logpush_http_request_firewall_matches_action_d95e64b7 field: json.FirewallMatchesActions target_field: cloudflare_logpush.http_request.firewall.matches.action ignore_missing: true if: ctx.cloudflare_logpush?.http_request?.firewall?.matches?.action == null - rename: + tag: rename_json_SecurityRuleIDs_to_cloudflare_logpush_http_request_firewall_matches_rule_id_a6a1429b field: json.SecurityRuleIDs target_field: cloudflare_logpush.http_request.firewall.matches.rule_id ignore_missing: true - rename: + tag: rename_json_FirewallMatchesRuleIDs_to_cloudflare_logpush_http_request_firewall_matches_rule_id_e5128330 field: json.FirewallMatchesRuleIDs target_field: cloudflare_logpush.http_request.firewall.matches.rule_id ignore_missing: true if: ctx.cloudflare_logpush?.http_request?.firewall?.matches?.rule_id == null - rename: + tag: rename_json_SecuritySources_to_cloudflare_logpush_http_request_firewall_matches_sources_3be8a2a9 field: json.SecuritySources target_field: cloudflare_logpush.http_request.firewall.matches.sources ignore_missing: true - rename: + tag: rename_json_FirewallMatchesSources_to_cloudflare_logpush_http_request_firewall_matches_sources_4c6b7d3e field: json.FirewallMatchesSources target_field: cloudflare_logpush.http_request.firewall.matches.sources ignore_missing: true if: ctx.cloudflare_logpush?.http_request?.firewall?.matches?.sources == null - rename: + tag: rename_json_JA3Hash_to_cloudflare_logpush_http_request_ja3_hash_e226d5df field: json.JA3Hash target_field: cloudflare_logpush.http_request.ja3_hash ignore_missing: true + - set: + field: tls.client.ja3 + tag: set_tls_client_ja3_from_ja3_hash + copy_from: cloudflare_logpush.http_request.ja3_hash + ignore_empty_value: true - rename: + tag: rename_json_JA4_to_cloudflare_logpush_http_request_ja4_59bce106 field: json.JA4 target_field: cloudflare_logpush.http_request.ja4 ignore_missing: true - rename: + tag: rename_json_JA4Signals_to_cloudflare_logpush_http_request_ja4_signals_d95118dd field: json.JA4Signals target_field: cloudflare_logpush.http_request.ja4_signals ignore_missing: true - convert: + tag: convert_json_OriginDNSResponseTimeMs_to_cloudflare_logpush_http_request_origin_dns_response_time_ms_238f4dc1 field: json.OriginDNSResponseTimeMs target_field: cloudflare_logpush.http_request.origin.dns_response_time.ms - if: ctx.json?.OriginDNSResponseTimeMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_1f662457 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginRequestHeaderSendDurationMs_to_cloudflare_logpush_http_request_origin_request_header_send_duration_ms_c405f76e field: json.OriginRequestHeaderSendDurationMs target_field: cloudflare_logpush.http_request.origin.request_header_send_duration.ms - if: ctx.json?.OriginRequestHeaderSendDurationMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_d93093cc field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginResponseBytes_to_cloudflare_logpush_http_request_origin_response_bytes_79b5b47b field: json.OriginResponseBytes target_field: cloudflare_logpush.http_request.origin.response.bytes - if: ctx.json?.OriginResponseBytes != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ed99c121 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginResponseDurationMs_to_cloudflare_logpush_http_request_origin_response_duration_ms_71a69769 field: json.OriginResponseDurationMs target_field: cloudflare_logpush.http_request.origin.response.duration.ms - if: ctx.json?.OriginResponseDurationMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_105e6a3f field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginResponseHeaderReceiveDurationMs_to_cloudflare_logpush_http_request_origin_response_header_receive_duration_ms_f2a7abbf field: json.OriginResponseHeaderReceiveDurationMs target_field: cloudflare_logpush.http_request.origin.response.header_receive_duration.ms - if: ctx.json?.OriginResponseHeaderReceiveDurationMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_5e5033a5 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginResponseStatus_to_cloudflare_logpush_http_request_origin_response_status_853ca57d field: json.OriginResponseStatus target_field: cloudflare_logpush.http_request.origin.response.status - if: ctx.json?.OriginResponseStatus != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_a4cd8cbb field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginResponseTime_to_cloudflare_logpush_http_request_origin_response_time_13cb781d field: json.OriginResponseTime target_field: cloudflare_logpush.http_request.origin.response.time - if: ctx.json?.OriginResponseTime != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c822a97b field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_OriginSSLProtocol_to_cloudflare_logpush_http_request_origin_ssl_protocol_beee50a5 field: json.OriginSSLProtocol target_field: cloudflare_logpush.http_request.origin.ssl_protocol ignore_missing: true - convert: + tag: convert_json_OriginTCPHandshakeDurationMs_to_cloudflare_logpush_http_request_origin_tcp_handshake_duration_ms_af335fe5 field: json.OriginTCPHandshakeDurationMs target_field: cloudflare_logpush.http_request.origin.tcp_handshake_duration.ms - if: ctx.json?.OriginTCPHandshakeDurationMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_72cc2dcb field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_OriginTLSHandshakeDurationMs_to_cloudflare_logpush_http_request_origin_tls_handshake_duration_ms_8da6ac79 field: json.OriginTLSHandshakeDurationMs target_field: cloudflare_logpush.http_request.origin.tls_handshake_duration.ms - if: ctx.json?.OriginTLSHandshakeDurationMs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_98b7c29f field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ParentRayID_to_cloudflare_logpush_http_request_parent_ray_id_f27e6a53 field: json.ParentRayID target_field: cloudflare_logpush.http_request.parent_ray.id ignore_missing: true - rename: + tag: rename_json_RayID_to_cloudflare_logpush_http_request_ray_id_5aae985a field: json.RayID target_field: cloudflare_logpush.http_request.ray.id ignore_missing: true - set: + tag: set_event_id_a8495d8d field: event.id copy_from: cloudflare_logpush.http_request.ray.id ignore_empty_value: true - rename: + tag: rename_json_RequestHeaders_to_cloudflare_logpush_http_request_request_headers_34660b94 field: json.RequestHeaders - target_field: cloudflare_logpush.http_request.request.header + target_field: cloudflare_logpush.http_request.request.headers ignore_missing: true - rename: + tag: rename_json_ResponseHeaders_to_cloudflare_logpush_http_request_response_headers_c65dbfec field: json.ResponseHeaders - target_field: cloudflare_logpush.http_request.response.header + target_field: cloudflare_logpush.http_request.response.headers ignore_missing: true - rename: + tag: rename_json_SecurityLevel_to_cloudflare_logpush_http_request_security_level_b8abdf6d field: json.SecurityLevel target_field: cloudflare_logpush.http_request.security_level ignore_missing: true - convert: + tag: convert_json_SmartRouteColoID_to_cloudflare_logpush_http_request_smart_route_colo_id_66ce0128 field: json.SmartRouteColoID target_field: cloudflare_logpush.http_request.smart_route.colo.id - if: ctx.json?.SmartRouteColoID != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_acc3f3ee field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_UpperTierColoID_to_cloudflare_logpush_http_request_upper_tier_colo_id_16454d92 field: json.UpperTierColoID target_field: cloudflare_logpush.http_request.upper_tier.colo.id - if: ctx.json?.UpperTierColoID != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_544372a8 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_SecurityAction_to_cloudflare_logpush_http_request_waf_action_5e276a4c field: json.SecurityAction target_field: cloudflare_logpush.http_request.waf.action ignore_missing: true - rename: + tag: rename_json_WAFAction_to_cloudflare_logpush_http_request_waf_action_d16779f0 field: json.WAFAction target_field: cloudflare_logpush.http_request.waf.action ignore_missing: true if: ctx.cloudflare_logpush?.http_request?.waf?.action == null - rename: + tag: rename_json_WAFFlags_to_cloudflare_logpush_http_request_waf_flag_f6f072ab field: json.WAFFlags target_field: cloudflare_logpush.http_request.waf.flag ignore_missing: true - rename: + tag: rename_json_WAFMatchedVar_to_cloudflare_logpush_http_request_waf_matched_var_d988df35 field: json.WAFMatchedVar target_field: cloudflare_logpush.http_request.waf.matched_var ignore_missing: true - rename: + tag: rename_json_WAFProfile_to_cloudflare_logpush_http_request_waf_profile_2fd3fefe field: json.WAFProfile target_field: cloudflare_logpush.http_request.waf.profile ignore_missing: true - rename: + tag: rename_json_SecurityRuleID_to_cloudflare_logpush_http_request_waf_rule_id_a0ada520 field: json.SecurityRuleID target_field: cloudflare_logpush.http_request.waf.rule.id ignore_missing: true - rename: + tag: rename_json_WAFRuleID_to_cloudflare_logpush_http_request_waf_rule_id_e3b580fa field: json.WAFRuleID target_field: cloudflare_logpush.http_request.waf.rule.id ignore_missing: true if: ctx.cloudflare_logpush?.http_request?.waf?.rule?.id == null - rename: + tag: rename_json_SecurityRuleDescription_to_cloudflare_logpush_http_request_waf_rule_message_8ceb7d17 field: json.SecurityRuleDescription target_field: cloudflare_logpush.http_request.waf.rule.message ignore_missing: true - rename: + tag: rename_json_WAFRuleMessage_to_cloudflare_logpush_http_request_waf_rule_message_dbbafec4 field: json.WAFRuleMessage target_field: cloudflare_logpush.http_request.waf.rule.message ignore_missing: true @@ -828,116 +1008,226 @@ processors: field: json.WAFAttackScore target_field: cloudflare_logpush.http_request.waf.score.global type: long - if: ctx.json?.WAFAttackScore != '' ignore_missing: true tag: convert_waf_score_global on_failure: - append: + tag: append_error_message_df678495 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: field: json.WAFRCEAttackScore target_field: cloudflare_logpush.http_request.waf.score.rce type: long - if: ctx.json?.WAFRCEAttackScore != '' ignore_missing: true tag: convert_waf_score_rce on_failure: - append: + tag: append_error_message_dbfbe5a3 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: field: json.WAFSQLiAttackScore target_field: cloudflare_logpush.http_request.waf.score.sqli type: long - if: ctx.json?.WAFSQLiAttackScore != '' ignore_missing: true tag: convert_waf_score_sqli on_failure: - append: + tag: append_error_message_e0da2880 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: field: json.WAFXSSAttackScore target_field: cloudflare_logpush.http_request.waf.score.xss type: long - if: ctx.json?.WAFXSSAttackScore != '' ignore_missing: true tag: convert_waf_score_xss on_failure: - append: + tag: append_error_message_90bea843 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_WorkerCPUTime_to_cloudflare_logpush_http_request_worker_cpu_time_e75c505a field: json.WorkerCPUTime target_field: cloudflare_logpush.http_request.worker.cpu_time - if: ctx.json?.WorkerCPUTime != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0b102990 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_WorkerStatus_to_cloudflare_logpush_http_request_worker_status_a56133c4 field: json.WorkerStatus target_field: cloudflare_logpush.http_request.worker.status ignore_missing: true - - rename: + - convert: + tag: convert_json_WorkerSubrequest_to_cloudflare_logpush_http_request_worker_subrequest_value_7706e15e field: json.WorkerSubrequest target_field: cloudflare_logpush.http_request.worker.subrequest.value + type: boolean ignore_missing: true + on_failure: + - append: + tag: append_error_message_52c6be14 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_WorkerSubrequestCount_to_cloudflare_logpush_http_request_worker_subrequest_count_d61893fb field: json.WorkerSubrequestCount target_field: cloudflare_logpush.http_request.worker.subrequest.count - if: ctx.json?.WorkerSubrequestCount != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_da2d1ae1 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_WorkerWallTimeUs_to_cloudflare_logpush_http_request_worker_wall_time_us_2198e815 field: json.WorkerWallTimeUs target_field: cloudflare_logpush.http_request.worker.wall_time_us - if: ctx.json?.WorkerWallTimeUs != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_e4ace5bb field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_ZoneID_to_cloudflare_logpush_http_request_zone_id_4909bd85 field: json.ZoneID target_field: cloudflare_logpush.http_request.zone.id - if: ctx.json?.ZoneID != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_a6429433 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ZoneName_to_cloudflare_logpush_http_request_zone_name_21fed8d2 field: json.ZoneName target_field: cloudflare_logpush.http_request.zone.name ignore_missing: true + - rename: + tag: rename_json_FraudAttack_to_cloudflare_logpush_http_request_fraud_attack_8a45d67a + field: json.FraudAttack + target_field: cloudflare_logpush.http_request.fraud.attack + ignore_missing: true + - convert: + tag: convert_json_FraudDetectionIDs_to_cloudflare_logpush_http_request_fraud_detection_ids_05ff211e + field: json.FraudDetectionIDs + target_field: cloudflare_logpush.http_request.fraud.detection_ids + type: long + ignore_missing: true + on_failure: + - append: + tag: append_error_message_ee7a733c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + tag: rename_json_FraudDetectionTags_to_cloudflare_logpush_http_request_fraud_detection_tags_2dcf7c41 + field: json.FraudDetectionTags + target_field: cloudflare_logpush.http_request.fraud.detection_tags + ignore_missing: true + - rename: + tag: rename_json_FraudEmailRisk_to_cloudflare_logpush_http_request_fraud_email_risk_1caf224b + field: json.FraudEmailRisk + target_field: cloudflare_logpush.http_request.fraud.email_risk + ignore_missing: true + - rename: + tag: rename_json_FraudUserID_to_cloudflare_logpush_http_request_fraud_user_id_eb218e7f + field: json.FraudUserID + target_field: cloudflare_logpush.http_request.fraud.user_id + ignore_missing: true + - rename: + tag: rename_json_JSDetectionPassed_to_cloudflare_logpush_http_request_js_detection_passed_09fbbdc6 + field: json.JSDetectionPassed + target_field: cloudflare_logpush.http_request.js_detection_passed + ignore_missing: true + - rename: + tag: rename_json_PayPerCrawlStatus_to_cloudflare_logpush_http_request_pay_per_crawl_status_962677d5 + field: json.PayPerCrawlStatus + target_field: cloudflare_logpush.http_request.pay_per_crawl_status + ignore_missing: true + - rename: + tag: rename_json_VerifiedBotCategory_to_cloudflare_logpush_http_request_verified_bot_category_286debc8 + field: json.VerifiedBotCategory + target_field: cloudflare_logpush.http_request.verified_bot_category + ignore_missing: true + - rename: + tag: rename_json_WebAssetsLabelsManaged_to_cloudflare_logpush_http_request_web_assets_labels_managed_681f6654 + field: json.WebAssetsLabelsManaged + target_field: cloudflare_logpush.http_request.web_assets.labels_managed + ignore_missing: true + - rename: + tag: rename_json_WebAssetsOperationID_to_cloudflare_logpush_http_request_web_assets_operation_id_d3e3500e + field: json.WebAssetsOperationID + target_field: cloudflare_logpush.http_request.web_assets.operation_id + ignore_missing: true + - rename: + tag: rename_json_WorkerScriptName_to_cloudflare_logpush_http_request_worker_script_name_a6f79051 + field: json.WorkerScriptName + target_field: cloudflare_logpush.http_request.worker.script_name + ignore_missing: true - append: + tag: append_related_hash_8ac40ba9 field: related.hash value: '{{{cloudflare_logpush.http_request.ja3_hash}}}' if: ctx.cloudflare_logpush?.http_request?.ja3_hash != null allow_duplicates: false - ignore_failure: true - append: + tag: append_related_hash_69d4e071 + field: related.hash + value: '{{{cloudflare_logpush.http_request.ja4}}}' + if: ctx.cloudflare_logpush?.http_request?.ja4 != null + allow_duplicates: false + - append: + tag: append_related_hash_70f73950 + field: related.hash + value: '{{{cloudflare_logpush.http_request.client.mtls.auth.fingerprint}}}' + if: ctx.cloudflare_logpush?.http_request?.client?.mtls?.auth?.fingerprint != null + allow_duplicates: false + - append: + tag: append_related_hosts_995ec422 + field: related.hosts + value: '{{{cloudflare_logpush.http_request.client.request.host}}}' + if: ctx.cloudflare_logpush?.http_request?.client?.request?.host != null + allow_duplicates: false + - append: + tag: append_related_hosts_3478de2e + field: related.hosts + value: '{{{cloudflare_logpush.http_request.edge.request.host}}}' + if: ctx.cloudflare_logpush?.http_request?.edge?.request?.host != null + allow_duplicates: false + - append: + tag: append_related_user_7ed39041 + field: related.user + value: '{{{cloudflare_logpush.http_request.fraud.user_id}}}' + if: ctx.cloudflare_logpush?.http_request?.fraud?.user_id != null + allow_duplicates: false + - append: + tag: append_related_ip_f8980c4d field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null + value: '{{{cloudflare_logpush.http_request.client.ip}}}' + if: ctx.cloudflare_logpush?.http_request?.client?.ip != null allow_duplicates: false - ignore_failure: true - append: + tag: append_related_ip_31f510cd field: related.ip - value: '{{{destination.ip}}}' - if: ctx.destination?.ip != null + value: '{{{cloudflare_logpush.http_request.origin.ip}}}' + if: ctx.cloudflare_logpush?.http_request?.origin?.ip != null + allow_duplicates: false + - append: + tag: append_related_ip_10ccce4e + field: related.ip + value: '{{{cloudflare_logpush.http_request.edge.server.ip}}}' + if: ctx.cloudflare_logpush?.http_request?.edge?.server?.ip != null allow_duplicates: false - ignore_failure: true - remove: tag: remove_json_conf field: @@ -945,6 +1235,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_c0e686cf field: - cloudflare_logpush.http_request.origin.ip - cloudflare_logpush.http_request.client.request.method @@ -955,33 +1246,58 @@ processors: - cloudflare_logpush.http_request.client.ip - cloudflare_logpush.http_request.client.request.user.agent - cloudflare_logpush.http_request.client.ssl.cipher + - cloudflare_logpush.http_request.client.device.type + - cloudflare_logpush.http_request.client.request.bytes + - cloudflare_logpush.http_request.edge.response.bytes + - cloudflare_logpush.http_request.client.src.port + - cloudflare_logpush.http_request.ja3_hash if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/magic_ids/elasticsearch/ingest_pipeline/default.yml index a2579ccfb50..02677ab975c 100644 --- a/packages/cloudflare_logpush/data_stream/magic_ids/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/magic_ids/elasticsearch/ingest_pipeline/default.yml @@ -2,36 +2,43 @@ description: Pipeline for parsing Cloudflare Magic IDS logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 - tag: set_ecs_version + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true - tag: rename_event_original + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - tag: json_event_original - set: field: event.kind value: event tag: set_event_kind - set: field: event.category - value: - - network - - intrusion_detection + value: + - network + - intrusion_detection tag: set_event_category - set: field: event.type value: [info] tag: set_event_type -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -40,28 +47,29 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# Timestamp + # Timestamp + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_df40dd8a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' @@ -72,30 +80,48 @@ processors: timezone: UTC tag: date_event_timestamp on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - remove: + tag: remove_json_Timestamp_6625d522 + field: json.Timestamp + ignore_missing: true + - append: + tag: append_error_message_31244f95 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: cloudflare_logpush.magic_ids.timestamp copy_from: "@timestamp" tag: set_timestamp + ignore_empty_value: true -# Handle destination IP and port - - rename: + # Handle destination IP and port + - convert: field: json.DestinationIP target_field: cloudflare_logpush.magic_ids.destination.ip + type: ip + if: ctx.json?.DestinationIP != '' ignore_missing: true - tag: rename_destination_ip + tag: convert_magic_ids_destination_ip_to_ip + on_failure: + - remove: + tag: remove_json_DestinationIP_da8097eb + field: json.DestinationIP + ignore_missing: true + - append: + tag: append_error_message_04240565 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: destination.ip copy_from: cloudflare_logpush.magic_ids.destination.ip ignore_empty_value: true tag: set_destination_ip -# Geo enrichment (destination IP) + # Geo enrichment (destination IP) - geoip: field: destination.ip target_field: destination.geo tag: geo_destination_ip + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb tag: geo_destination_as @@ -115,33 +141,55 @@ processors: target_field: destination.as.organization.name ignore_missing: true tag: rename_destination_as_org - - rename: + - convert: field: json.DestinationPort target_field: cloudflare_logpush.magic_ids.destination.port + type: long ignore_missing: true - tag: rename_destination_port + tag: convert_magic_ids_destination_port_to_long + on_failure: + - remove: + tag: remove_json_DestinationPort_c4f9fce7 + field: json.DestinationPort + ignore_missing: true + - append: + tag: append_error_message_2827f163 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: destination.port copy_from: cloudflare_logpush.magic_ids.destination.port ignore_empty_value: true tag: set_destination_port -# Handle source IP and port - - rename: + # Handle source IP and port + - convert: field: json.SourceIP target_field: cloudflare_logpush.magic_ids.source.ip + type: ip + if: ctx.json?.SourceIP != '' ignore_missing: true - tag: rename_source_ip + tag: convert_magic_ids_source_ip_to_ip + on_failure: + - remove: + tag: remove_json_SourceIP_1c89585b + field: json.SourceIP + ignore_missing: true + - append: + tag: append_error_message_9c4e142c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: source.ip copy_from: cloudflare_logpush.magic_ids.source.ip ignore_empty_value: true tag: set_source_ip -# Geo enrichment (source IP) + # Geo enrichment (source IP) - geoip: field: source.ip target_field: source.geo tag: geo_source_ip + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb tag: geo_source_as @@ -161,11 +209,21 @@ processors: target_field: source.as.organization.name ignore_missing: true tag: rename_source_as_org - - rename: + - convert: field: json.SourcePort target_field: cloudflare_logpush.magic_ids.source.port + type: long ignore_missing: true - tag: rename_source_port + tag: convert_magic_ids_source_port_to_long + on_failure: + - remove: + tag: remove_json_SourcePort_dcac03a4 + field: json.SourcePort + ignore_missing: true + - append: + tag: append_error_message_ccdd2861 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: source.port copy_from: cloudflare_logpush.magic_ids.source.port @@ -194,19 +252,19 @@ processors: ignore_empty_value: true tag: set_event_action - append: - field: event.type - value: allowed - allow_duplicates: false - if: ctx.event?.action == 'pass' - tag: append_event_type_allowed + field: event.type + value: allowed + allow_duplicates: false + if: ctx.event?.action == 'pass' + tag: append_event_type_allowed - append: - field: event.type - value: denied - allow_duplicates: false - if: ctx.event?.action == 'block' - tag: append_event_type_block + field: event.type + value: denied + allow_duplicates: false + if: ctx.event?.action == 'block' + tag: append_event_type_block -# Custom fields + # Custom fields - rename: field: json.ColoCode target_field: cloudflare_logpush.magic_ids.colo.code @@ -217,37 +275,57 @@ processors: target_field: cloudflare_logpush.magic_ids.colo.city ignore_missing: true tag: rename_colo_city - - rename: + - convert: field: json.SignatureID target_field: cloudflare_logpush.magic_ids.signature.id + type: long ignore_missing: true - tag: rename_signature_id + tag: convert_magic_ids_signature_id_to_long + on_failure: + - remove: + tag: remove_json_SignatureID_ef5cfa87 + field: json.SignatureID + ignore_missing: true + - append: + tag: append_error_message_edb26dcb + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.SignatureMessage target_field: cloudflare_logpush.magic_ids.signature.message ignore_missing: true tag: rename_signature_message - - rename: + - convert: field: json.SignatureRevision target_field: cloudflare_logpush.magic_ids.signature.revision + type: long ignore_missing: true - tag: rename_signature_revision + tag: convert_magic_ids_signature_revision_to_long + on_failure: + - remove: + tag: remove_json_SignatureRevision_944410fb + field: json.SignatureRevision + ignore_missing: true + - append: + tag: append_error_message_b72c25ff + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' -# Create related fields + # Create related fields - append: field: related.ip - value: '{{{destination.ip}}}' - if: ctx.destination?.ip != null && ctx.destination.ip != '' + value: '{{{cloudflare_logpush.magic_ids.destination.ip}}}' + if: ctx.cloudflare_logpush?.magic_ids?.destination?.ip != null allow_duplicates: false - tag: append_related_destination_ip + tag: append_destination_ip_into_related_ip - append: field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null && ctx.source.ip != '' + value: '{{{cloudflare_logpush.magic_ids.source.ip}}}' + if: ctx.cloudflare_logpush?.magic_ids?.source?.ip != null allow_duplicates: false - tag: append_related_source_ip + tag: append_source_ip_into_related_ip -# Clean resulting event + # Clean resulting event - remove: tag: remove_json_conf field: @@ -264,34 +342,57 @@ processors: - cloudflare_logpush.magic_ids.source.port - cloudflare_logpush.magic_ids.transport if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true tag: remove_duplicated_custom_fields + on_failure: + - append: + tag: append_error_message_f7da9d35 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: - description: Drops null/empty values recursively. - tag: script_drop_empty_fields + tag: script_to_drop_null_values lang: painless + description: This script processor iterates over the whole document to remove fields with null values. source: |- - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log b/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log index f5babd62b87..8dd9bc8ac4e 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log @@ -1,3 +1,3 @@ {"ClientIPASN":"13335","ClientIPASNDescription":"CLOUDFLARENET","ClientIPCountry":"US","LastKnownGoodColoCode":"SJC","Phase":"connection","Timestamp":"2021-07-27T00:01:07Z","Type":"network-error"} {"ClientIPASN":"13335","ClientIPASNDescription":"CLOUDFLARENET","ClientIPCountry":"US","LastKnownGoodColoCode":"SJC","Phase":"connection","Timestamp":"1627344067","Type":"network-error"} -{"ClientIPASN":"13335","ClientIPASNDescription":"CLOUDFLARENET","ClientIPCountry":"US","LastKnownGoodColoCode":"SJC","Phase":"connection","Timestamp":"1627344067000000000","Type":"network-error"} \ No newline at end of file +{"ClientIPASN":"13335","ClientIPASNDescription":"CLOUDFLARENET","ClientIPCountry":"US","LastKnownGoodColoCode":"SJC","Phase":"connection","Timestamp":"1627344067000000000","Type":"network-error"} diff --git a/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml index 36efb8c409d..551b91d3a99 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare NEL Report logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,27 +45,28 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_df40dd8a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' @@ -61,11 +75,18 @@ processors: - ISO8601 - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC + tag: date_json_timestamp on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - remove: + tag: remove_json_Timestamp_9c461744 + field: json.Timestamp + ignore_missing: true + - append: + tag: append_error_message_ef9ca3c1 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_nel_report_timestamp_26266fd5 field: cloudflare_logpush.nel_report.timestamp copy_from: '@timestamp' ignore_empty_value: true @@ -73,36 +94,47 @@ processors: field: json.Type target_field: cloudflare_logpush.nel_report.error.type ignore_missing: true + tag: rename_json_type - set: field: error.type copy_from: cloudflare_logpush.nel_report.error.type ignore_empty_value: true + tag: set_error_type - convert: field: json.ClientIPASN target_field: cloudflare_logpush.nel_report.client.ip.asn.value - if: ctx.json?.ClientIPASN != '' type: long ignore_missing: true + tag: convert_client_ip_asn on_failure: + - remove: + tag: remove_json_ClientIPASN_6786aab2 + field: json.ClientIPASN + ignore_missing: true - append: + tag: append_error_message_1fec64e9 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.ClientIPASNDescription target_field: cloudflare_logpush.nel_report.client.ip.asn.description ignore_missing: true + tag: rename_client_ip_asn_description - rename: field: json.ClientIPCountry target_field: cloudflare_logpush.nel_report.client.ip.country ignore_missing: true + tag: rename_client_ip_country - rename: field: json.LastKnownGoodColoCode target_field: cloudflare_logpush.nel_report.last_known_good.colo.code ignore_missing: true + tag: rename_last_known_good_colo_code - rename: field: json.Phase target_field: cloudflare_logpush.nel_report.phase ignore_missing: true + tag: rename_phase - remove: tag: remove_json_conf field: @@ -114,32 +146,57 @@ processors: - cloudflare_logpush.nel_report.timestamp - cloudflare_logpush.nel_report.error.type if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true + tag: remove_duplicated_custom_fields + on_failure: + - append: + tag: append_error_message_eafb8646 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/sample_logs/network_analytics.log b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/sample_logs/network_analytics.log index 4196c40179b..d85cf018e0a 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/sample_logs/network_analytics.log +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/sample_logs/network_analytics.log @@ -1 +1 @@ -{"AttackCampaignID":"xyz987","AttackID":"abc777","ColoCountry":"AD","ColoGeoHash":"gbuun","ColoID":46,"ColoName":"SJC","Datetime":"2021-07-27T00:01:07Z","DestinationASN":1900,"DestinationASNDescription":"asn description","DestinationCountry":"AD","DestinationGeoHash":"gbuun","DestinationPort":0,"Direction":"ingress","GREChecksum":10,"GREEthertype":10,"GREHeaderLength":1024,"GREKey":10,"GRESequenceNumber":10,"GREVersion":10,"ICMPChecksum":10,"ICMPCode":10,"ICMPType":10,"IPDestinationAddress":"175.16.199.0","IPDestinationSubnet":"/24","IPFragmentOffset":1480,"IPHeaderLength":20,"IPMoreFragments":1480,"IPProtocol":6,"IPProtocolName":"tcp","IPSourceAddress":"67.43.156.0","IPSourceSubnet":"/24","IPTotalLength":1024,"IPTotalLengthBuckets":10,"IPTtl":240,"IPTtlBuckets":2,"IPv4Checksum":0,"IPv4DontFragment":0,"IPv4Dscp":46,"IPv4Ecn":1,"IPv4Identification":1,"IPv4Options":1,"IPv6Dscp":46,"IPv6Ecn":1,"IPv6ExtensionHeaders":"header","IPv6FlowLabel":1,"IPv6Identification":1,"MitigationReason":"BLOCKED","MitigationScope":"local","MitigationSystem":"flowtrackd","Outcome":"pass","ProtocolState":"OPEN","RuleID":"rule1","RulesetID":"3b64149bfa6e4220bbbc2bd6db589552","RulesetOverrideID":"id1","SampleInterval":1,"SourceASN":1500,"SourceASNDescription":"Source ASN Description","SourceCountry":"AD","SourceGeoHash":"gbuun","SourcePort":0,"TCPAcknowledgementNumber":1000,"TCPChecksum":10,"TCPDataOffset":0,"TCPFlags":1,"TCPFlagsString":"Human-readable flags string","TCPMss":512,"TCPOptions":"mss","TCPSackBlocks":1,"TCPSacksPermitted":1,"TCPSequenceNumber":100,"TCPTimestampEcr":100,"TCPTimestampValue":100,"TCPUrgentPointer":10,"TCPWindowScale":10,"TCPWindowSize":10,"UDPChecksum":10,"UDPPayloadLength":10,"Verdict":"pass"} \ No newline at end of file +{"AttackCampaignID":"xyz987","AttackID":"abc777","ColoCountry":"AD","ColoGeoHash":"gbuun","ColoID":46,"ColoName":"SJC","Datetime":"2021-07-27T00:01:07Z","DestinationASN":1900,"DestinationASNDescription":"asn description","DestinationCountry":"AD","DestinationGeoHash":"gbuun","DestinationPort":1234,"Direction":"ingress","GREChecksum":10,"GREEthertype":10,"GREHeaderLength":1024,"GREKey":10,"GRESequenceNumber":10,"GREVersion":10,"ICMPChecksum":10,"ICMPCode":10,"ICMPType":10,"IPDestinationAddress":"175.16.199.0","IPDestinationSubnet":"/24","IPFragmentOffset":1480,"IPHeaderLength":20,"IPMoreFragments":1480,"IPProtocol":6,"IPProtocolName":"tcp","IPSourceAddress":"67.43.156.0","IPSourceSubnet":"/24","IPTotalLength":1024,"IPTotalLengthBuckets":10,"IPTtl":240,"IPTtlBuckets":2,"IPv4Checksum":0,"IPv4DontFragment":0,"IPv4Dscp":46,"IPv4Ecn":1,"IPv4Identification":1,"IPv4Options":1,"IPv6Dscp":46,"IPv6Ecn":1,"IPv6ExtensionHeaders":"header","IPv6FlowLabel":1,"IPv6Identification":1,"MitigationReason":"BLOCKED","MitigationScope":"local","MitigationSystem":"flowtrackd","Outcome":"pass","ProtocolState":"OPEN","RuleID":"rule1","RulesetID":"3b64149bfa6e4220bbbc2bd6db589552","RulesetOverrideID":"id1","SampleInterval":1,"SourceASN":1500,"SourceASNDescription":"Source ASN Description","SourceCountry":"AD","SourceGeoHash":"gbuun","SourcePort":5678,"TCPAcknowledgementNumber":1000,"TCPChecksum":10,"TCPDataOffset":0,"TCPFlags":1,"TCPFlagsString":"Human-readable flags string","TCPMss":512,"TCPOptions":"mss","TCPSackBlocks":1,"TCPSacksPermitted":1,"TCPSequenceNumber":100,"TCPTimestampEcr":100,"TCPTimestampValue":100,"TCPUrgentPointer":10,"TCPWindowScale":10,"TCPWindowSize":10,"UDPChecksum":10,"UDPPayloadLength":10,"Verdict":"pass"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log b/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log index b2a7141fb9f..cc48fc672a1 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log @@ -1,3 +1,3 @@ {"AccountID":"e1836771179f98aabb828da5ea69a111","BytesReceived":679,"BytesSent":2333,"ClientTCPHandshakeDurationMs":12,"ClientTLSCipher":"TLS_AES_128_GCM_SHA256","ClientTLSHandshakeDurationMs":125,"ClientTLSVersion":"TLS 1.3","ConnectionCloseReason":"CLIENT_CLOSED","ConnectionReuse":false,"DestinationTunnelID":"00000000-0000-0000-0000-000000000000","DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","EgressColoName":"ORD","EgressIP":"2a02:cf40::23","EgressPort":41052,"EgressRuleID":"00000000-0000-0000-0000-000000000000","EgressRuleName":"Egress Rule 1","Email":"user@test.com","IngressColoName":"ORD","Offramp":"INTERNET","OriginIP":"89.160.20.129","OriginPort":80,"OriginTLSCertificateIssuer":"DigiCert Inc","OriginTLSCertificateValidationResult":"VALID","OriginTLSCipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","OriginTLSHandshakeDurationMs":130,"OriginTLSVersion":"TLS 1.2","Protocol":"TCP","RuleEvaluationDurationMs":10,"SessionEndTime":"2023-05-04T11:29:14Z","SessionID":"18881f179300007fb0d06d6400000001","SessionStartTime":"2023-05-04T11:29:14Z","SourceInternalIP":"1.128.0.1","SourceIP":"67.43.156.2","SourcePort":52994,"UserID":"166befbb-00e3-5e20-bd6e-27245723949f","VirtualNetworkID":"0ce99869-63d3-4d5d-bdaf-d4f33df964aa"} {"AccountID":"e1836771179f98aabb828da5ea69a111","BytesReceived":679,"BytesSent":2333,"ClientTCPHandshakeDurationMs":12,"ClientTLSCipher":"TLS_AES_128_GCM_SHA256","ClientTLSHandshakeDurationMs":125,"ClientTLSVersion":"TLS 1.3","ConnectionCloseReason":"CLIENT_CLOSED","ConnectionReuse":false,"DestinationTunnelID":"00000000-0000-0000-0000-000000000000","DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","EgressColoName":"ORD","EgressIP":"2a02:cf40::23","EgressPort":41052,"EgressRuleID":"00000000-0000-0000-0000-000000000000","EgressRuleName":"Egress Rule 1","Email":"user@test.com","IngressColoName":"ORD","Offramp":"INTERNET","OriginIP":"89.160.20.129","OriginPort":80,"OriginTLSCertificateIssuer":"DigiCert Inc","OriginTLSCertificateValidationResult":"VALID","OriginTLSCipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","OriginTLSHandshakeDurationMs":130,"OriginTLSVersion":"TLS 1.2","Protocol":"TCP","RuleEvaluationDurationMs":10,"SessionEndTime":"1683199754","SessionID":"18881f179300007fb0d06d6400000001","SessionStartTime":"1683199754","SourceInternalIP":"1.128.0.1","SourceIP":"67.43.156.2","SourcePort":52994,"UserID":"166befbb-00e3-5e20-bd6e-27245723949f","VirtualNetworkID":"0ce99869-63d3-4d5d-bdaf-d4f33df964aa"} -{"AccountID":"e1836771179f98aabb828da5ea69a111","BytesReceived":679,"BytesSent":2333,"ClientTCPHandshakeDurationMs":12,"ClientTLSCipher":"TLS_AES_128_GCM_SHA256","ClientTLSHandshakeDurationMs":125,"ClientTLSVersion":"TLS 1.3","ConnectionCloseReason":"CLIENT_CLOSED","ConnectionReuse":false,"DestinationTunnelID":"00000000-0000-0000-0000-000000000000","DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","EgressColoName":"ORD","EgressIP":"2a02:cf40::23","EgressPort":41052,"EgressRuleID":"00000000-0000-0000-0000-000000000000","EgressRuleName":"Egress Rule 1","Email":"user@test.com","IngressColoName":"ORD","Offramp":"INTERNET","OriginIP":"89.160.20.129","OriginPort":80,"OriginTLSCertificateIssuer":"DigiCert Inc","OriginTLSCertificateValidationResult":"VALID","OriginTLSCipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","OriginTLSHandshakeDurationMs":130,"OriginTLSVersion":"TLS 1.2","Protocol":"TCP","RuleEvaluationDurationMs":10,"SessionEndTime":"1683199754000000000","SessionID":"18881f179300007fb0d06d6400000001","SessionStartTime":"1683199754000000000","SourceInternalIP":"1.128.0.1","SourceIP":"67.43.156.2","SourcePort":52994,"UserID":"166befbb-00e3-5e20-bd6e-27245723949f","VirtualNetworkID":"0ce99869-63d3-4d5d-bdaf-d4f33df964aa"} \ No newline at end of file +{"AccountID":"e1836771179f98aabb828da5ea69a111","BytesReceived":679,"BytesSent":2333,"ClientTCPHandshakeDurationMs":12,"ClientTLSCipher":"TLS_AES_128_GCM_SHA256","ClientTLSHandshakeDurationMs":125,"ClientTLSVersion":"TLS 1.3","ConnectionCloseReason":"CLIENT_CLOSED","ConnectionReuse":false,"DestinationTunnelID":"00000000-0000-0000-0000-000000000000","DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","EgressColoName":"ORD","EgressIP":"2a02:cf40::23","EgressPort":41052,"EgressRuleID":"00000000-0000-0000-0000-000000000000","EgressRuleName":"Egress Rule 1","Email":"user@test.com","IngressColoName":"ORD","Offramp":"INTERNET","OriginIP":"89.160.20.129","OriginPort":80,"OriginTLSCertificateIssuer":"DigiCert Inc","OriginTLSCertificateValidationResult":"VALID","OriginTLSCipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","OriginTLSHandshakeDurationMs":130,"OriginTLSVersion":"TLS 1.2","Protocol":"TCP","RuleEvaluationDurationMs":10,"SessionEndTime":"1683199754000000000","SessionID":"18881f179300007fb0d06d6400000001","SessionStartTime":"1683199754000000000","SourceInternalIP":"1.128.0.1","SourceIP":"67.43.156.2","SourcePort":52994,"UserID":"166befbb-00e3-5e20-bd6e-27245723949f","VirtualNetworkID":"0ce99869-63d3-4d5d-bdaf-d4f33df964aa"} diff --git a/packages/cloudflare_logpush/data_stream/network_session/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/network_session/elasticsearch/ingest_pipeline/default.yml index 0afdad2049a..960b309106a 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/network_session/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Zero Trust Network Session logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_ed69613e field: event.category value: [network, session] - set: + tag: set_event_type_8fd37e4e field: event.type value: [connection] - set: + tag: set_event_kind_de80643c field: event.kind value: event -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,29 +45,44 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# ECS fields + # ECS fields + - convert: + tag: convert_json_SessionStartTime_15b1559d + field: json.SessionStartTime + type: long + if: ctx.json?.SessionStartTime != null && ctx.json.SessionStartTime != '' + ignore_failure: true + - convert: + tag: convert_json_SessionEndTime_5c99a0b6 + field: json.SessionEndTime + type: long + if: ctx.json?.SessionEndTime != null && ctx.json.SessionEndTime != '' + ignore_failure: true - script: lang: painless tag: painless_session_start_time_to_milli - if: ctx.json?.SessionStartTime != null source: | - try { - long t; - if (ctx.json.SessionStartTime instanceof String) { - t = Long.parseLong(ctx.json.SessionStartTime); - } else if (ctx.json.SessionStartTime instanceof Number) { - t = (long)(ctx.json.SessionStartTime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.SessionStartTime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.SessionStartTime = t*(long)(1e3) + def convertToMillis(long timestamp) { + if (timestamp > (long)(1e18)) { + return timestamp/(long)(1e6) + } else if (timestamp < (long)(1e10)) { + return timestamp*(long)(1e3) } + return timestamp + } + if (ctx.json?.SessionStartTime != null && ctx.json.SessionStartTime instanceof Number) { + ctx.json.SessionStartTime = convertToMillis(ctx.json.SessionStartTime); + } + if (ctx.json?.SessionEndTime != null && ctx.json.SessionEndTime instanceof Number) { + ctx.json.SessionEndTime = convertToMillis(ctx.json.SessionEndTime); } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_687a5d42 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_SessionStartTime_4818ec3f field: json.SessionStartTime if: ctx.json?.SessionStartTime != null && ctx.json.SessionStartTime != '' formats: @@ -63,36 +91,29 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_3b048279 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: - field: json.SessionStartTime + tag: set_session_start_efb4fc52 + field: cloudflare_logpush.network_session.session.start copy_from: "@timestamp" - - script: - lang: painless - tag: painless_session_end_time_to_milli - if: ctx.json?.SessionEndTime != null - source: | - try { - long t; - if (ctx.json.SessionEndTime instanceof String) { - t = Long.parseLong(ctx.json.SessionEndTime); - } else if (ctx.json.SessionEndTime instanceof Number) { - t = (long)(ctx.json.SessionEndTime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.SessionEndTime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.SessionEndTime = t*(long)(1e3) - } - } - catch (Exception e) {} + ignore_empty_value: true + - set: + tag: set_event_start_6e95ec84 + field: event.start + copy_from: cloudflare_logpush.network_session.session.start + ignore_empty_value: true + - set: + tag: set_cloudflare_logpush_network_session_timestamp_34bb3f1a + field: cloudflare_logpush.network_session.timestamp + copy_from: "@timestamp" + ignore_empty_value: true - date: + tag: date_json_SessionEndTime_to_json_SessionEndTime_fd82cbd6 field: json.SessionEndTime - target_field: json.SessionEndTime + target_field: cloudflare_logpush.network_session.session.end if: ctx.json?.SessionEndTime != null && ctx.json.SessionEndTime != '' formats: - UNIX_MS @@ -100,61 +121,111 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_2c5842b8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: - field: cloudflare_logpush.network_session.timestamp - copy_from: "@timestamp" - - rename: + tag: set_event_end_d2325484 + field: event.end + copy_from: cloudflare_logpush.network_session.session.end + ignore_empty_value: true + - script: + lang: painless + description: Calculates event.duration from event.start and event.end + if: ctx.event?.start != null && ctx.event?.end != null && ctx.event?.duration == null + source: >- + ZonedDateTime start = ZonedDateTime.parse(ctx.event.start); + ZonedDateTime end = ZonedDateTime.parse(ctx.event.end); + ctx.event.duration = ChronoUnit.NANOS.between(start, end); + tag: calculate_event_duration + - convert: + tag: convert_json_BytesReceived_to_cloudflare_logpush_network_session_destination_bytes_4522d0e1 field: json.BytesReceived target_field: cloudflare_logpush.network_session.destination.bytes + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_5ce8d6fb + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_bytes_e38f3e4c field: destination.bytes copy_from: cloudflare_logpush.network_session.destination.bytes ignore_empty_value: true - - rename: + - convert: + tag: convert_json_BytesSent_to_cloudflare_logpush_network_session_source_bytes_c5ef2c67 field: json.BytesSent target_field: cloudflare_logpush.network_session.source.bytes + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_23fe5bb1 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_bytes_1d92f200 field: source.bytes copy_from: cloudflare_logpush.network_session.source.bytes ignore_empty_value: true - rename: + tag: rename_json_DetectedProtocol_to_cloudflare_logpush_network_session_detected_protocol_1c65f314 field: json.DetectedProtocol target_field: cloudflare_logpush.network_session.detected_protocol ignore_missing: true - rename: + tag: rename_json_DeviceID_to_cloudflare_logpush_network_session_host_id_26f4a4bd field: json.DeviceID target_field: cloudflare_logpush.network_session.host.id ignore_missing: true - set: + tag: set_host_id_8ded5f2c field: host.id copy_from: cloudflare_logpush.network_session.host.id ignore_empty_value: true + - set: + field: device.id + tag: set_device_id_from_host_id + copy_from: cloudflare_logpush.network_session.host.id + ignore_empty_value: true - rename: + tag: rename_json_DeviceName_to_cloudflare_logpush_network_session_host_name_145bb719 field: json.DeviceName target_field: cloudflare_logpush.network_session.host.name ignore_missing: true - set: + tag: set_host_name_840dd844 field: host.name copy_from: cloudflare_logpush.network_session.host.name ignore_empty_value: true - - rename: + - convert: + tag: convert_json_OriginIP_to_cloudflare_logpush_network_session_destination_ip_8dcc0373 field: json.OriginIP target_field: cloudflare_logpush.network_session.destination.ip + type: ip ignore_missing: true + if: ctx.json?.OriginIP != '' + on_failure: + - append: + tag: append_error_message_31b5f3bd + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_ip_94459dee field: destination.ip copy_from: cloudflare_logpush.network_session.destination.ip ignore_empty_value: true -# Geo enrichment (destination IP) + # Geo enrichment (destination IP) - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo + ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -163,74 +234,86 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_OriginPort_to_cloudflare_logpush_network_session_destination_port_8b8fc264 field: json.OriginPort target_field: cloudflare_logpush.network_session.destination.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_84222e32 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port_5f15990a field: destination.port copy_from: cloudflare_logpush.network_session.destination.port ignore_empty_value: true - rename: + tag: rename_json_OriginTLSCertificateIssuer_to_cloudflare_logpush_network_session_tls_server_certificate_issuer_94eda834 field: json.OriginTLSCertificateIssuer target_field: cloudflare_logpush.network_session.tls.server.certificate.issuer ignore_missing: true - set: + tag: set_tls_server_issuer_84dfe341 field: tls.server.issuer copy_from: cloudflare_logpush.network_session.tls.server.certificate.issuer ignore_empty_value: true - rename: + tag: rename_json_Protocol_to_cloudflare_logpush_network_session_transport_932973e8 field: json.Protocol target_field: cloudflare_logpush.network_session.transport ignore_missing: true - set: + tag: set_network_transport_3f402e30 field: network.transport copy_from: cloudflare_logpush.network_session.transport ignore_empty_value: true - rename: + tag: rename_json_SessionID_to_cloudflare_logpush_network_session_session_id_d026538b field: json.SessionID target_field: cloudflare_logpush.network_session.session.id ignore_missing: true - set: + tag: set_event_id_9ca1813e field: event.id copy_from: cloudflare_logpush.network_session.session.id ignore_empty_value: true - - rename: - field: json.SessionStartTime - target_field: cloudflare_logpush.network_session.session.start - ignore_missing: true - - set: - field: event.start - copy_from: cloudflare_logpush.network_session.session.start - ignore_empty_value: true - - rename: - field: json.SessionEndTime - target_field: cloudflare_logpush.network_session.session.end - ignore_missing: true - - set: - field: event.end - copy_from: cloudflare_logpush.network_session.session.end - ignore_empty_value: true - - rename: + - convert: + tag: convert_json_SourceIP_to_cloudflare_logpush_network_session_source_ip_025a4038 field: json.SourceIP target_field: cloudflare_logpush.network_session.source.ip + type: ip ignore_missing: true + if: ctx.json?.SourceIP != '' + on_failure: + - append: + tag: append_error_message_f55015d6 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_60b5c944 field: source.ip copy_from: cloudflare_logpush.network_session.source.ip ignore_empty_value: true -# Geo enrichment (source IP) + # Geo enrichment (source IP) - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo + ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -239,165 +322,330 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - - rename: + - convert: + tag: convert_json_SourcePort_to_cloudflare_logpush_network_session_source_port_f4c6cbe2 field: json.SourcePort target_field: cloudflare_logpush.network_session.source.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_2bb9309c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_port_14399d1c field: source.port copy_from: cloudflare_logpush.network_session.source.port ignore_empty_value: true - rename: + tag: rename_json_UserID_to_cloudflare_logpush_network_session_user_id_48a9a5b3 field: json.UserID target_field: cloudflare_logpush.network_session.user.id ignore_missing: true - set: + tag: set_user_id_d4461d6c field: user.id copy_from: cloudflare_logpush.network_session.user.id ignore_empty_value: true - rename: + tag: rename_json_Email_to_cloudflare_logpush_network_session_user_email_1f12a13c field: json.Email target_field: cloudflare_logpush.network_session.user.email ignore_missing: true - set: + tag: set_user_email_a8fe96ba field: user.email copy_from: cloudflare_logpush.network_session.user.email ignore_empty_value: true - rename: + tag: rename_json_VirtualNetworkID_to_cloudflare_logpush_network_session_vlan_id_0d1d80e9 field: json.VirtualNetworkID target_field: cloudflare_logpush.network_session.vlan.id ignore_missing: true - set: + tag: set_network_vlan_id_ee700abc field: network.vlan.id copy_from: cloudflare_logpush.network_session.vlan.id ignore_empty_value: true -# Custom fields + # Custom fields - rename: + tag: rename_json_AccountID_to_cloudflare_logpush_network_session_account_id_10bbb2fa field: json.AccountID target_field: cloudflare_logpush.network_session.account_id ignore_missing: true - - rename: + - convert: + tag: convert_json_ClientTCPHandshakeDurationMs_to_cloudflare_logpush_network_session_tcp_client_handshake_time_ms_69cd1747 field: json.ClientTCPHandshakeDurationMs target_field: cloudflare_logpush.network_session.tcp.client.handshake_time_ms + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_4c1381f5 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ConnectionCloseReason_to_cloudflare_logpush_network_session_tcp_connection_close_reason_75e8a337 field: json.ConnectionCloseReason target_field: cloudflare_logpush.network_session.tcp.connection.close_reason ignore_missing: true - - rename: + - convert: + tag: convert_json_ConnectionReuse_to_cloudflare_logpush_network_session_tcp_connection_reuse_61a298b9 field: json.ConnectionReuse target_field: cloudflare_logpush.network_session.tcp.connection.reuse + type: boolean ignore_missing: true + on_failure: + - append: + tag: append_error_message_8c08ccfb + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_DestinationTunnelID_to_cloudflare_logpush_network_session_destination_tunnel_id_34922a94 field: json.DestinationTunnelID target_field: cloudflare_logpush.network_session.destination.tunnel_id ignore_missing: true - rename: + tag: rename_json_EgressColoName_to_cloudflare_logpush_network_session_egress_colo_name_469fcc54 field: json.EgressColoName target_field: cloudflare_logpush.network_session.egress.colo_name ignore_missing: true - - rename: + - convert: + tag: convert_json_EgressIP_to_cloudflare_logpush_network_session_egress_ip_69991e02 field: json.EgressIP target_field: cloudflare_logpush.network_session.egress.ip + type: ip ignore_missing: true - - rename: + if: ctx.json?.EgressIP != '' + on_failure: + - append: + tag: append_error_message_8fc9cf48 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_EgressPort_to_cloudflare_logpush_network_session_egress_port_34a05ac2 field: json.EgressPort target_field: cloudflare_logpush.network_session.egress.port + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_b504a68c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_EgressRuleID_to_cloudflare_logpush_network_session_egress_rule_id_88aeb3b5 field: json.EgressRuleID target_field: cloudflare_logpush.network_session.egress.rule.id ignore_missing: true - rename: + tag: rename_json_EgressRuleName_to_cloudflare_logpush_network_session_egress_rule_name_930f5f11 field: json.EgressRuleName target_field: cloudflare_logpush.network_session.egress.rule.name ignore_missing: true - rename: + tag: rename_json_IngressColoName_to_cloudflare_logpush_network_session_ingress_colo_name_c5ed9816 field: json.IngressColoName target_field: cloudflare_logpush.network_session.ingress.colo_name ignore_missing: true - rename: + tag: rename_json_Offramp_to_cloudflare_logpush_network_session_offramp_7546864d field: json.Offramp target_field: cloudflare_logpush.network_session.offramp ignore_missing: true - - rename: + - convert: + tag: convert_json_RuleEvaluationDurationMs_to_cloudflare_logpush_network_session_rule_evaluation_time_ms_7fd60435 field: json.RuleEvaluationDurationMs target_field: cloudflare_logpush.network_session.rule_evaluation.time_ms + type: long ignore_missing: true - - rename: + on_failure: + - append: + tag: append_error_message_624bf6db + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_SourceInternalIP_to_cloudflare_logpush_network_session_source_internal_ip_dd0c67a2 field: json.SourceInternalIP target_field: cloudflare_logpush.network_session.source.internal_ip + type: ip + ignore_missing: true + if: ctx.json?.SourceInternalIP != '' + on_failure: + - append: + tag: append_error_message_a4010a0c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_InitialOriginIP_to_cloudflare_logpush_network_session_initial_origin_ip_45718b53 + field: json.InitialOriginIP + target_field: cloudflare_logpush.network_session.initial_origin_ip + type: ip + ignore_missing: true + if: ctx.json?.InitialOriginIP != '' + on_failure: + - append: + tag: append_error_message_34d648d9 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + tag: rename_json_RegistrationID_to_cloudflare_logpush_network_session_registration_id_5daa239a + field: json.RegistrationID + target_field: cloudflare_logpush.network_session.registration_id ignore_missing: true - rename: + tag: rename_json_ResolvedFQDN_to_cloudflare_logpush_network_session_resolved_fqdn_f2d2e5e4 + field: json.ResolvedFQDN + target_field: cloudflare_logpush.network_session.resolved_fqdn + ignore_missing: true + - rename: + tag: rename_json_SNI_to_cloudflare_logpush_network_session_sni_ed6abd1d + field: json.SNI + target_field: cloudflare_logpush.network_session.sni + ignore_missing: true + - set: + field: tls.client.server_name + tag: set_tls_client_server_name_from_sni + copy_from: cloudflare_logpush.network_session.sni + ignore_empty_value: true + - rename: + tag: rename_json_ClientTLSCipher_to_cloudflare_logpush_network_session_tls_client_cipher_4f9c2371 field: json.ClientTLSCipher target_field: cloudflare_logpush.network_session.tls.client.cipher ignore_missing: true - - rename: + - set: + field: tls.cipher + tag: set_tls_cipher_from_client_tls_cipher + copy_from: cloudflare_logpush.network_session.tls.client.cipher + ignore_empty_value: true + - convert: + tag: convert_json_ClientTLSHandshakeDurationMs_to_cloudflare_logpush_network_session_tls_client_handshake_time_ms_39ad139f field: json.ClientTLSHandshakeDurationMs target_field: cloudflare_logpush.network_session.tls.client.handshake_time_ms + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_b6dfdbb5 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ClientTLSVersion_to_cloudflare_logpush_network_session_tls_client_version_ed81718d field: json.ClientTLSVersion target_field: cloudflare_logpush.network_session.tls.client.version ignore_missing: true + - dissect: + field: cloudflare_logpush.network_session.tls.client.version + tag: dissect_client_tls_version + pattern: '%{tls.version_protocol}v%{tls.version}' + ignore_missing: true + ignore_failure: true + - lowercase: + field: tls.version_protocol + tag: lowercase_tls_version_protocol + ignore_missing: true - rename: + tag: rename_json_OriginTLSCertificateValidationResult_to_cloudflare_logpush_network_session_tls_server_certificate_validation_result_8626f7e1 field: json.OriginTLSCertificateValidationResult target_field: cloudflare_logpush.network_session.tls.server.certificate.validation_result ignore_missing: true - rename: + tag: rename_json_OriginTLSCipher_to_cloudflare_logpush_network_session_tls_server_cipher_4039a3d8 field: json.OriginTLSCipher target_field: cloudflare_logpush.network_session.tls.server.cipher ignore_missing: true - - rename: + - convert: + tag: convert_json_OriginTLSHandshakeDurationMs_to_cloudflare_logpush_network_session_tls_server_handshake_time_ms_9a1c6b76 field: json.OriginTLSHandshakeDurationMs target_field: cloudflare_logpush.network_session.tls.server.handshake_time_ms + type: long ignore_missing: true + on_failure: + - append: + tag: append_error_message_5c18e91c + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_OriginTLSVersion_to_cloudflare_logpush_network_session_tls_server_version_6509c1f8 field: json.OriginTLSVersion target_field: cloudflare_logpush.network_session.tls.server.version ignore_missing: true -# Create related fields + # Create related fields - append: + tag: append_related_ip_536684c7 field: related.ip - value: "{{{source.ip}}}" - if: ctx.source?.ip != null + value: '{{{cloudflare_logpush.network_session.source.ip}}}' + if: ctx.cloudflare_logpush?.network_session?.source?.ip != null allow_duplicates: false - append: + tag: append_related_ip_0c197e61 field: related.ip - value: "{{{destination.ip}}}" - if: ctx.destination?.ip != null + value: '{{{cloudflare_logpush.network_session.destination.ip}}}' + if: ctx.cloudflare_logpush?.network_session?.destination?.ip != null allow_duplicates: false - append: + tag: append_related_ip_e5930da7 field: related.ip - value: "{{{cloudflare_logpush.network_session.egress.ip}}}" + value: '{{{cloudflare_logpush.network_session.egress.ip}}}' if: ctx.cloudflare_logpush?.network_session?.egress?.ip != null allow_duplicates: false - append: + tag: append_related_ip_3b19c28f + field: related.ip + value: '{{{cloudflare_logpush.network_session.source.internal_ip}}}' + if: ctx.cloudflare_logpush?.network_session?.source?.internal_ip != null + allow_duplicates: false + - append: + tag: append_related_ip_c98bd3f4 + field: related.ip + value: '{{{cloudflare_logpush.network_session.initial_origin_ip}}}' + if: ctx.cloudflare_logpush?.network_session?.initial_origin_ip != null + allow_duplicates: false + - append: + tag: append_related_hosts_3b861d29 + field: related.hosts + value: '{{{cloudflare_logpush.network_session.host.id}}}' + if: ctx.cloudflare_logpush?.network_session?.host?.id != null + allow_duplicates: false + - append: + tag: append_related_hosts_36bad091 + field: related.hosts + value: '{{{cloudflare_logpush.network_session.host.name}}}' + if: ctx.cloudflare_logpush?.network_session?.host?.name != null + allow_duplicates: false + - append: + tag: append_related_hosts_857bdfa8 field: related.hosts - value: "{{{host.id}}}" - if: ctx.host?.id != null + value: '{{{cloudflare_logpush.network_session.resolved_fqdn}}}' + if: ctx.cloudflare_logpush?.network_session?.resolved_fqdn != null allow_duplicates: false - append: + tag: append_related_hosts_09a22ef4 field: related.hosts - value: "{{{host.name}}}" - if: ctx.host?.name != null + value: '{{{cloudflare_logpush.network_session.sni}}}' + if: ctx.cloudflare_logpush?.network_session?.sni != null allow_duplicates: false - append: + tag: append_related_user_a0633cd9 field: related.user - value: "{{{user.id}}}" - if: ctx.user?.id != null + value: '{{{cloudflare_logpush.network_session.user.id}}}' + if: ctx.cloudflare_logpush?.network_session?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_d7e7b58f field: related.user - value: "{{{user.email}}}" - if: ctx.user?.email != null + value: '{{{cloudflare_logpush.network_session.user.email}}}' + if: ctx.cloudflare_logpush?.network_session?.user?.email != null allow_duplicates: false - remove: tag: remove_json_conf @@ -406,6 +654,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_0b964e75 field: - cloudflare_logpush.network_session.timestamp - cloudflare_logpush.network_session.destination.bytes @@ -424,33 +673,54 @@ processors: - cloudflare_logpush.network_session.user.id - cloudflare_logpush.network_session.user.email - cloudflare_logpush.network_session.vlan.id + - cloudflare_logpush.network_session.tls.client.cipher + - cloudflare_logpush.network_session.sni if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. + tag: script_to_drop_null_values lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml index b77f019ffc4..c52194d10d1 100644 --- a/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Page Shield Event logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,30 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_Timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_76b0386a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Timestamp_70be028a field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' formats: @@ -62,49 +77,51 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - append: + tag: append_error_message_d6556db8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_page_shield_events_timestamp_15ff66de field: cloudflare_logpush.page_shield_events.timestamp copy_from: '@timestamp' ignore_empty_value: true - rename: + tag: rename_json_Action_to_cloudflare_logpush_page_shield_events_action_249c1317 field: json.Action target_field: cloudflare_logpush.page_shield_events.action ignore_missing: true - set: + tag: set_event_action_b99711ae field: event.action copy_from: cloudflare_logpush.page_shield_events.action ignore_empty_value: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - rename: + tag: rename_json_CSPDirective_to_cloudflare_logpush_page_shield_events_csp_directive_ec50c63e field: json.CSPDirective target_field: cloudflare_logpush.page_shield_events.csp_directive ignore_missing: true - rename: + tag: rename_json_Host_to_cloudflare_logpush_page_shield_events_host_664c8787 field: json.Host target_field: cloudflare_logpush.page_shield_events.host ignore_missing: true - set: + tag: set_host_name_122ed077 field: host.name copy_from: cloudflare_logpush.page_shield_events.host ignore_empty_value: true - - append: - field: related.hosts - value: '{{{host.name}}}' - if: ctx.host?.name != null && ctx.host.name != '' - allow_duplicates: false - rename: + tag: rename_json_PageURL_to_cloudflare_logpush_page_shield_events_page_url_529399de field: json.PageURL target_field: cloudflare_logpush.page_shield_events.page_url ignore_missing: true - rename: + tag: rename_json_URL_to_cloudflare_logpush_page_shield_events_url_32905bc9 field: json.URL target_field: cloudflare_logpush.page_shield_events.url ignore_missing: true @@ -115,16 +132,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_843a9dcb field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_PolicyID_to_cloudflare_logpush_page_shield_events_policy_id_8eed8fc8 field: json.PolicyID target_field: cloudflare_logpush.page_shield_events.policy_id ignore_missing: true - rename: + tag: rename_json_ResourceType_to_cloudflare_logpush_page_shield_events_resource_type_085e4bce field: json.ResourceType target_field: cloudflare_logpush.page_shield_events.resource_type ignore_missing: true @@ -136,19 +153,31 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_de7594a2 field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_URLHost_to_cloudflare_logpush_page_shield_events_url_host_ed944944 field: json.URLHost target_field: cloudflare_logpush.page_shield_events.url_host ignore_missing: true - set: + tag: set_url_domain_086e7313 field: url.domain copy_from: cloudflare_logpush.page_shield_events.url_host ignore_empty_value: true + - append: + tag: append_related_hosts_d7df75fa + field: related.hosts + value: '{{{cloudflare_logpush.page_shield_events.host}}}' + if: ctx.cloudflare_logpush?.page_shield_events?.host != null + allow_duplicates: false + - append: + tag: append_related_hosts_7ba9ae92 + field: related.hosts + value: '{{{cloudflare_logpush.page_shield_events.url_host}}}' + if: ctx.cloudflare_logpush?.page_shield_events?.url_host != null + allow_duplicates: false - remove: tag: remove_json_conf field: @@ -156,37 +185,47 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_61810b29 field: - cloudflare_logpush.page_shield_events.action - cloudflare_logpush.page_shield_events.host - cloudflare_logpush.page_shield_events.url - cloudflare_logpush.page_shield_events.timestamp - if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') - ignore_failure: true + if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); - set: field: event.kind - value: pipeline_error tag: set_pipeline_error_into_event_kind + value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -194,12 +233,10 @@ processors: on_failure: - append: field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/elasticsearch/ingest_pipeline/default.yml index 60a07dbcb03..d72fba2e647 100644 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/elasticsearch/ingest_pipeline/default.yml @@ -2,19 +2,26 @@ description: Pipeline for parsing Cloudflare Sinkhole HTTP logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 - tag: set_ecs_version + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true - tag: rename_event_original + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - tag: json_event_original - set: field: event.kind value: event @@ -27,9 +34,9 @@ processors: field: event.type value: [info] tag: set_event_type -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -38,28 +45,29 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# Timestamp + # Timestamp + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_df40dd8a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' @@ -70,30 +78,40 @@ processors: timezone: UTC tag: date_event_timestamp on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_ccdbdce0 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: cloudflare_logpush.sinkhole_http.timestamp copy_from: "@timestamp" tag: set_timestamp + ignore_empty_value: true -# Handle destination IP - - rename: + # Handle destination IP + - convert: field: json.DestAddr target_field: cloudflare_logpush.sinkhole_http.destination.ip + type: ip ignore_missing: true - tag: rename_destination_ip + if: ctx.json?.DestAddr != '' + tag: convert_sinkhole_http_destination_ip_to_ip + on_failure: + - append: + tag: append_error_message_32f305b3 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: destination.ip copy_from: cloudflare_logpush.sinkhole_http.destination.ip ignore_empty_value: true tag: set_destination_ip -# Geo enrichment (destination IP) + # Geo enrichment (destination IP) - geoip: field: destination.ip target_field: destination.geo tag: geo_destination_ip + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb tag: geo_destination_as @@ -114,22 +132,30 @@ processors: ignore_missing: true tag: rename_destination_as_org -# Handle source IP - - rename: + # Handle source IP + - convert: field: json.SrcAddr target_field: cloudflare_logpush.sinkhole_http.source.ip + type: ip ignore_missing: true - tag: rename_source_ip + if: ctx.json?.SrcAddr != '' + tag: convert_sinkhole_http_source_ip_to_ip + on_failure: + - append: + tag: append_error_message_e995d2cb + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: source.ip copy_from: cloudflare_logpush.sinkhole_http.source.ip ignore_empty_value: true tag: set_source_ip -# Geo enrichment (source IP) + # Geo enrichment (source IP) - geoip: field: source.ip target_field: source.geo tag: geo_source_ip + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb tag: geo_source_as @@ -150,7 +176,7 @@ processors: ignore_missing: true tag: rename_source_as_org -# HTTP fields + # HTTP fields - rename: field: json.Body target_field: cloudflare_logpush.sinkhole_http.request.body.content @@ -161,11 +187,17 @@ processors: copy_from: cloudflare_logpush.sinkhole_http.request.body.content ignore_empty_value: true tag: set_http_body - - rename: + - convert: field: json.BodyLength target_field: cloudflare_logpush.sinkhole_http.request.body.bytes + type: long ignore_missing: true - tag: rename_http_body_bytes + tag: convert_sinkhole_http_request_body_bytes_to_long + on_failure: + - append: + tag: append_error_message_e7877929 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: http.request.body.bytes copy_from: cloudflare_logpush.sinkhole_http.request.body.bytes @@ -203,14 +235,17 @@ processors: ignore_empty_value: true tag: set_http_host - rename: + tag: rename_json_URI_to_cloudflare_logpush_sinkhole_http_request_uri_fcb3a925 field: json.URI target_field: cloudflare_logpush.sinkhole_http.request.uri ignore_missing: true - rename: + tag: rename_json_URL_to_cloudflare_logpush_sinkhole_http_request_url_44a29715 field: json.URL target_field: cloudflare_logpush.sinkhole_http.request.url ignore_missing: true - uri_parts: + tag: uri_parts_cloudflare_logpush_sinkhole_http_request_url_to_url_4d919288 field: cloudflare_logpush.sinkhole_http.request.url target_field: url if: ctx.cloudflare_logpush?.sinkhole_http?.request?.url != null && ctx.cloudflare_logpush.sinkhole_http.request.url != '' @@ -220,6 +255,7 @@ processors: ignore_missing: true tag: rename_user_agent - user_agent: + tag: user_agent_cloudflare_logpush_sinkhole_http_user_agent_3af01702 field: cloudflare_logpush.sinkhole_http.user_agent ignore_missing: true - rename: @@ -233,7 +269,7 @@ processors: ignore_empty_value: true tag: set_username -# Custom fields + # Custom fields - rename: field: json.AccountID target_field: cloudflare_logpush.sinkhole_http.account_id @@ -265,33 +301,33 @@ processors: ignore_missing: true tag: rename_sinkhole_id -# Create related fields + # Create related fields - append: field: related.ip - value: '{{{destination.ip}}}' - if: ctx.destination?.ip != null && ctx.destination.ip != '' + value: '{{{cloudflare_logpush.sinkhole_http.destination.ip}}}' + if: ctx.cloudflare_logpush?.sinkhole_http?.destination?.ip != null allow_duplicates: false tag: append_related_destination_ip - append: field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null && ctx.source.ip != '' + value: '{{{cloudflare_logpush.sinkhole_http.source.ip}}}' + if: ctx.cloudflare_logpush?.sinkhole_http?.source?.ip != null allow_duplicates: false tag: append_related_source_ip - append: field: related.hosts - value: '{{{host.name}}}' - if: ctx.host?.name != null && ctx.host.name != '' + value: '{{{cloudflare_logpush.sinkhole_http.host.name}}}' + if: ctx.cloudflare_logpush?.sinkhole_http?.host?.name != null allow_duplicates: false tag: append_related_host - append: field: related.user - value: '{{{user.name}}}' - if: ctx.user?.name != null && ctx.user.name != '' + value: '{{{cloudflare_logpush.sinkhole_http.user.name}}}' + if: ctx.cloudflare_logpush?.sinkhole_http?.user?.name != null allow_duplicates: false tag: append_related_user -# Clean resulting event + # Clean resulting event - remove: tag: remove_json_conf field: @@ -313,34 +349,52 @@ processors: - cloudflare_logpush.sinkhole_http.request.uri - cloudflare_logpush.sinkhole_http.request.url if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true tag: remove_duplicated_custom_fields - script: - description: Drops null/empty values recursively. - tag: script_drop_empty_fields + tag: script_to_drop_null_values lang: painless + description: This script processor iterates over the whole document to remove fields with null values. source: |- - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml index 2c8ee1ee7c8..df3c9db58eb 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Spectrum Event logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,60 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true + - convert: + tag: convert_json_ConnectTimestamp_be5367bc + field: json.ConnectTimestamp + type: long + if: ctx.json?.ConnectTimestamp != null && ctx.json.ConnectTimestamp != '' + ignore_failure: true + - convert: + tag: convert_json_DisconnectTimestamp_84281438 + field: json.DisconnectTimestamp + type: long + if: ctx.json?.DisconnectTimestamp != null && ctx.json.DisconnectTimestamp != '' + ignore_failure: true - script: lang: painless tag: painless_timestamp_to_milli - if: ctx.json?.Timestamp != null source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); + def convertToMillis(long timestamp) { + if (timestamp > (long)(1e18)) { + return timestamp/(long)(1e6) + } else if (timestamp < (long)(1e10)) { + return timestamp*(long)(1e3) + } + return timestamp + } + if (ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number) { + ctx.json.Timestamp = convertToMillis(ctx.json.Timestamp); + } + if (ctx.json?.ConnectTimestamp != null && ctx.json.ConnectTimestamp instanceof Number) { + if (ctx.json.ConnectTimestamp == 0) { + ctx.json.ConnectTimestamp = null; } else { - return; + ctx.json.ConnectTimestamp = convertToMillis(ctx.json.ConnectTimestamp); } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) + } + if (ctx.json?.DisconnectTimestamp != null && ctx.json.DisconnectTimestamp instanceof Number) { + if (ctx.json.DisconnectTimestamp == 0) { + ctx.json.DisconnectTimestamp = null; + } else { + ctx.json.DisconnectTimestamp = convertToMillis(ctx.json.DisconnectTimestamp); } } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_1608873d + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Timestamp_70be028a field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' formats: @@ -62,43 +107,19 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_d6556db8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_spectrum_event_timestamp_eb12da0f field: cloudflare_logpush.spectrum_event.timestamp copy_from: '@timestamp' ignore_empty_value: true - - script: - lang: painless - tag: painless_connect_timestamp_to_milli - if: ctx.json?.ConnectTimestamp != null - source: | - if (ctx.json.ConnectTimestamp == "0" || - ctx.json.ConnectTimestamp == 0 || - ctx.json.ConnectTimestamp == "") { - ctx.json.ConnectTimestamp = null; - return; - } - try { - long t; - if (ctx.json.ConnectTimestamp instanceof String) { - t = Long.parseLong(ctx.json.ConnectTimestamp); - } else if (ctx.json.ConnectTimestamp instanceof Number) { - t = (long)(ctx.json.ConnectTimestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.ConnectTimestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.ConnectTimestamp = t*(long)(1e3) - } - } - catch (Exception e) {} - date: + tag: date_json_ConnectTimestamp_to_cloudflare_logpush_spectrum_event_connect_time_526de421 field: json.ConnectTimestamp - if: ctx.json?.ConnectTimestamp != null + if: ctx.json?.ConnectTimestamp != null && ctx.json.ConnectTimestamp != '' formats: - ISO8601 - uuuu-MM-dd'T'HH:mm:ssX @@ -110,42 +131,18 @@ processors: target_field: cloudflare_logpush.spectrum_event.connect.time on_failure: - append: + tag: append_error_message_ff3b02f3 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_event_start_0859f146 field: event.start copy_from: cloudflare_logpush.spectrum_event.connect.time ignore_empty_value: true - - script: - lang: painless - tag: painless_disconnect_timestamp_to_milli - if: ctx.json?.DisconnectTimestamp != null - source: | - if (ctx.json?.DisconnectTimestamp == "0" || - ctx.json?.DisconnectTimestamp == 0 || - ctx.json?.DisconnectTimestamp == "") { - ctx.json.DisconnectTimestamp = null; - return; - } - try { - long t; - if (ctx.json.DisconnectTimestamp instanceof String) { - t = Long.parseLong(ctx.json.DisconnectTimestamp); - } else if (ctx.json.DisconnectTimestamp instanceof Number) { - t = (long)(ctx.json.DisconnectTimestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.DisconnectTimestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.DisconnectTimestamp = t*(long)(1e3) - } - } - catch (Exception e) {} - date: + tag: date_json_DisconnectTimestamp_to_cloudflare_logpush_spectrum_event_disconnect_time_9ed3f259 field: json.DisconnectTimestamp - if: ctx.json?.DisconnectTimestamp != null + if: ctx.json?.DisconnectTimestamp != null && ctx.json.DisconnectTimestamp != '' formats: - ISO8601 - uuuu-MM-dd'T'HH:mm:ssX @@ -157,271 +154,335 @@ processors: target_field: cloudflare_logpush.spectrum_event.disconnect.time on_failure: - append: + tag: append_error_message_9f6c295b field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_event_end_74af7957 field: event.end copy_from: cloudflare_logpush.spectrum_event.disconnect.time ignore_empty_value: true + - script: + lang: painless + description: Calculates event.duration from event.start and event.end + if: ctx.event?.start != null && ctx.event?.end != null && ctx.event?.duration == null + source: >- + ZonedDateTime start = ZonedDateTime.parse(ctx.event.start); + ZonedDateTime end = ZonedDateTime.parse(ctx.event.end); + ctx.event.duration = ChronoUnit.NANOS.between(start, end); + tag: calculate_event_duration - rename: + tag: rename_json_Event_to_cloudflare_logpush_spectrum_event_action_a1437c14 field: json.Event target_field: cloudflare_logpush.spectrum_event.action ignore_missing: true - set: + tag: set_event_action_f87e3943 field: event.action copy_from: cloudflare_logpush.spectrum_event.action ignore_empty_value: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - convert: + tag: convert_json_OriginBytes_to_cloudflare_logpush_spectrum_event_origin_bytes_f0d69873 field: json.OriginBytes target_field: cloudflare_logpush.spectrum_event.origin.bytes - if: ctx.json?.OriginBytes != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0931fbdd field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_bytes_37d6657d field: destination.bytes copy_from: cloudflare_logpush.spectrum_event.origin.bytes ignore_empty_value: true - convert: + tag: convert_json_OriginIP_to_cloudflare_logpush_spectrum_event_origin_ip_905b3a2c field: json.OriginIP target_field: cloudflare_logpush.spectrum_event.origin.ip - if: ctx.json?.OriginIP != '' type: ip ignore_missing: true + if: ctx.json?.OriginIP != '' on_failure: - append: + tag: append_error_message_74a73b2a field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_ip_cbefb767 field: destination.ip copy_from: cloudflare_logpush.spectrum_event.origin.ip ignore_empty_value: true - convert: + tag: convert_json_OriginPort_to_cloudflare_logpush_spectrum_event_origin_port_1698678b field: json.OriginPort target_field: cloudflare_logpush.spectrum_event.origin.port - if: ctx.json?.OriginPort != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_38ec1e55 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port_078de763 field: destination.port copy_from: cloudflare_logpush.spectrum_event.origin.port ignore_empty_value: true - rename: + tag: rename_json_Application_to_cloudflare_logpush_spectrum_event_application_4a0e3d0e field: json.Application target_field: cloudflare_logpush.spectrum_event.application ignore_missing: true - set: + tag: set_event_id_fe813204 field: event.id copy_from: cloudflare_logpush.spectrum_event.application ignore_empty_value: true - convert: + tag: convert_json_Status_to_cloudflare_logpush_spectrum_event_status_f6b72271 field: json.Status target_field: cloudflare_logpush.spectrum_event.status - if: ctx.json?.Status != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_3dd6cdc3 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_http_response_status_code_8eb3b996 field: http.response.status_code copy_from: cloudflare_logpush.spectrum_event.status ignore_empty_value: true - convert: + tag: convert_json_ClientAsn_to_cloudflare_logpush_spectrum_event_client_asn_d42ef38d field: json.ClientAsn target_field: cloudflare_logpush.spectrum_event.client.asn - if: ctx.json?.ClientAsn != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_21d09607 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_as_number_43c84eca field: source.as.number copy_from: cloudflare_logpush.spectrum_event.client.asn ignore_empty_value: true - convert: + tag: convert_json_ClientBytes_to_cloudflare_logpush_spectrum_event_client_bytes_2e430a65 field: json.ClientBytes target_field: cloudflare_logpush.spectrum_event.client.bytes - if: ctx.json?.ClientBytes != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ffbb9457 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_bytes_61e86c9f field: source.bytes copy_from: cloudflare_logpush.spectrum_event.client.bytes ignore_empty_value: true - rename: + tag: rename_json_ClientCountry_to_cloudflare_logpush_spectrum_event_client_country_7e53c63a field: json.ClientCountry target_field: cloudflare_logpush.spectrum_event.client.country ignore_missing: true - set: + tag: set_source_geo_country_iso_code_a5c289a6 field: source.geo.country_iso_code copy_from: cloudflare_logpush.spectrum_event.client.country ignore_empty_value: true - convert: + tag: convert_json_ClientIP_to_cloudflare_logpush_spectrum_event_client_ip_f360f739 field: json.ClientIP target_field: cloudflare_logpush.spectrum_event.client.ip - if: ctx.json?.ClientIP != '' type: ip ignore_missing: true + if: ctx.json?.ClientIP != '' on_failure: - append: + tag: append_error_message_a0a1c567 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_a8992b97 field: source.ip copy_from: cloudflare_logpush.spectrum_event.client.ip ignore_empty_value: true - convert: + tag: convert_json_ClientPort_to_cloudflare_logpush_spectrum_event_client_port_65d05837 field: json.ClientPort target_field: cloudflare_logpush.spectrum_event.client.port - if: ctx.json?.ClientPort != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_adcf6e99 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_port_8f859f67 field: source.port copy_from: cloudflare_logpush.spectrum_event.client.port ignore_empty_value: true - rename: + tag: rename_json_ClientMatchedIpFirewall_to_cloudflare_logpush_spectrum_event_client_matched_ip_firewall_2a24f1a2 field: json.ClientMatchedIpFirewall target_field: cloudflare_logpush.spectrum_event.client.matched_ip_firewall ignore_missing: true - rename: + tag: rename_json_ClientProto_to_cloudflare_logpush_spectrum_event_client_protocol_ab852cb8 field: json.ClientProto target_field: cloudflare_logpush.spectrum_event.client.protocol ignore_missing: true - set: + tag: set_network_transport_26276eaf field: network.transport copy_from: cloudflare_logpush.spectrum_event.client.protocol ignore_empty_value: true - lowercase: + tag: lowercase_network_transport_bc8c1c12 field: network.transport ignore_missing: true - convert: + tag: convert_json_ClientTcpRtt_to_cloudflare_logpush_spectrum_event_client_tcp_rtt_5f939cf4 field: json.ClientTcpRtt target_field: cloudflare_logpush.spectrum_event.client.tcp_rtt - if: ctx.json?.ClientTcpRtt != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_87e331ee field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ClientTlsCipher_to_cloudflare_logpush_spectrum_event_client_tls_cipher_abbca2ae field: json.ClientTlsCipher target_field: cloudflare_logpush.spectrum_event.client.tls.cipher ignore_missing: true + - set: + field: tls.cipher + tag: set_tls_cipher_from_client_tls_cipher + copy_from: cloudflare_logpush.spectrum_event.client.tls.cipher + ignore_empty_value: true - rename: + tag: rename_json_ClientTlsClientHelloServerName_to_cloudflare_logpush_spectrum_event_client_tls_client_hello_server_name_c9681c0f field: json.ClientTlsClientHelloServerName target_field: cloudflare_logpush.spectrum_event.client.tls.client_hello_server_name ignore_missing: true - rename: + tag: rename_json_ClientTlsProtocol_to_cloudflare_logpush_spectrum_event_client_tls_protocol_a7adcec0 field: json.ClientTlsProtocol target_field: cloudflare_logpush.spectrum_event.client.tls.protocol ignore_missing: true - dissect: if: ctx.cloudflare_logpush?.spectrum_event?.client?.tls?.protocol != null && ctx.cloudflare_logpush.spectrum_event.client.tls.protocol != 'none' && ctx.cloudflare_logpush.spectrum_event.client.tls.protocol != 'unknown' field: cloudflare_logpush.spectrum_event.client.tls.protocol + tag: dissect_client_tls_protocol pattern: '%{tls.version_protocol}v%{tls.version}' ignore_missing: true + ignore_failure: true - lowercase: field: tls.version_protocol + tag: lowercase_tls_version_protocol ignore_missing: true - rename: + tag: rename_json_ClientTlsStatus_to_cloudflare_logpush_spectrum_event_client_tls_status_7dd6b8c0 field: json.ClientTlsStatus target_field: cloudflare_logpush.spectrum_event.client.tls.status ignore_missing: true - rename: + tag: rename_json_ColoCode_to_cloudflare_logpush_spectrum_event_colo_code_13aaf6cc field: json.ColoCode target_field: cloudflare_logpush.spectrum_event.colo.code ignore_missing: true - - rename: + - convert: field: json.IpFirewall target_field: cloudflare_logpush.spectrum_event.ip_firewall + type: boolean ignore_missing: true + tag: convert_spectrum_event_ip_firewall_to_boolean + on_failure: + - append: + tag: append_error_message_f464f2aa + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_OriginProto_to_cloudflare_logpush_spectrum_event_origin_protocol_b28a7b72 field: json.OriginProto target_field: cloudflare_logpush.spectrum_event.origin.protocol ignore_missing: true - convert: + tag: convert_json_OriginTcpRtt_to_cloudflare_logpush_spectrum_event_origin_tcp_rtt_38e194a4 field: json.OriginTcpRtt target_field: cloudflare_logpush.spectrum_event.origin.tcp_rtt - if: ctx.json?.OriginTcpRtt != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ff262a56 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_OriginTlsCipher_to_cloudflare_logpush_spectrum_event_origin_tls_cipher_233b05dc field: json.OriginTlsCipher target_field: cloudflare_logpush.spectrum_event.origin.tls.cipher ignore_missing: true - rename: + tag: rename_json_OriginTlsFingerprint_to_cloudflare_logpush_spectrum_event_origin_tls_fingerprint_6157979e field: json.OriginTlsFingerprint target_field: cloudflare_logpush.spectrum_event.origin.tls.fingerprint ignore_missing: true - rename: + tag: rename_json_OriginTlsMode_to_cloudflare_logpush_spectrum_event_origin_tls_mode_2f051e70 field: json.OriginTlsMode target_field: cloudflare_logpush.spectrum_event.origin.tls.mode ignore_missing: true - rename: + tag: rename_json_OriginTlsProtocol_to_cloudflare_logpush_spectrum_event_origin_tls_protocol_97e4e7ae field: json.OriginTlsProtocol target_field: cloudflare_logpush.spectrum_event.origin.tls.protocol ignore_missing: true - rename: + tag: rename_json_OriginTlsStatus_to_cloudflare_logpush_spectrum_event_origin_tls_status_5cae09ea field: json.OriginTlsStatus target_field: cloudflare_logpush.spectrum_event.origin.tls.status - ignore_failure: true + ignore_missing: true - rename: + tag: rename_json_ProxyProtocol_to_cloudflare_logpush_spectrum_event_proxy_protocol_13f4be6c field: json.ProxyProtocol target_field: cloudflare_logpush.spectrum_event.proxy.protocol ignore_missing: true - append: - field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null - allow_duplicates: false - ignore_failure: true - - append: + tag: append_related_ip_c9e172e9 field: related.ip value: '{{{cloudflare_logpush.spectrum_event.client.ip}}}' if: ctx.cloudflare_logpush?.spectrum_event?.client?.ip != null allow_duplicates: false - ignore_failure: true - - append: - field: related.ip - value: '{{{destination.ip}}}' - if: ctx.destination?.ip != null - allow_duplicates: false - ignore_failure: true - append: + tag: append_related_ip_d0673639 field: related.ip value: '{{{cloudflare_logpush.spectrum_event.origin.ip}}}' if: ctx.cloudflare_logpush?.spectrum_event?.origin?.ip != null allow_duplicates: false - ignore_failure: true + - append: + tag: append_related_hash_a97d3f39 + field: related.hash + value: '{{{cloudflare_logpush.spectrum_event.origin.tls.fingerprint}}}' + if: ctx.cloudflare_logpush?.spectrum_event?.origin?.tls?.fingerprint != null + allow_duplicates: false - community_id: + tag: community_id_d2308e7a target_field: network.community_id ignore_failure: true - remove: @@ -431,6 +492,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_33a92778 field: - cloudflare_logpush.spectrum_event.timestamp - cloudflare_logpush.spectrum_event.origin.bytes @@ -444,33 +506,53 @@ processors: - cloudflare_logpush.spectrum_event.client.country - cloudflare_logpush.spectrum_event.client.ip - cloudflare_logpush.spectrum_event.client.port + - cloudflare_logpush.spectrum_event.client.tls.cipher if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/workers_trace/elasticsearch/ingest_pipeline/default.yml index 6b1260b88bd..48f615d4632 100644 --- a/packages/cloudflare_logpush/data_stream/workers_trace/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/workers_trace/elasticsearch/ingest_pipeline/default.yml @@ -2,19 +2,26 @@ description: Pipeline for parsing Cloudflare Workers Trace Event logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 - tag: set_ecs_version + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true - tag: rename_event_original + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - tag: json_event_original - set: field: event.category value: [web] @@ -27,9 +34,9 @@ processors: field: event.kind value: event tag: set_event_kind -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -49,14 +56,15 @@ processors: timezone: UTC tag: date_event_timestamp on_failure: - - append: - field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + - append: + tag: append_error_message_2af7750e + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: cloudflare_logpush.workers_trace.timestamp copy_from: "@timestamp" tag: set_timestamp + ignore_empty_value: true # ECS fields - rename: @@ -134,9 +142,32 @@ processors: target_field: cloudflare_logpush.workers_trace.script.version ignore_missing: true tag: rename_script_version + - convert: + field: json.CPUTimeMs + target_field: cloudflare_logpush.workers_trace.cpu_time_ms + type: long + ignore_missing: true + tag: convert_cpu_time_ms + on_failure: + - append: + tag: append_error_message_9e7f0b47 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: json.WallTimeMs + target_field: cloudflare_logpush.workers_trace.wall_time_ms + type: long + ignore_missing: true + tag: convert_wall_time_ms + on_failure: + - append: + tag: append_error_message_2962d579 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: description: Convert Workers Trace Events keys to snake case recursively. + if: ctx.cloudflare_logpush?.workers_trace != null tag: script_convert_snake_case lang: painless source: |- @@ -167,6 +198,11 @@ processors: } ctx.cloudflare_logpush['workers_trace'] = keysToSnakeCase(ctx.cloudflare_logpush.workers_trace); + on_failure: + - append: + tag: append_error_message_986de937 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.id @@ -189,7 +225,7 @@ processors: if: ctx.cloudflare_logpush?.workers_trace?.event?.request?.url != null tag: uri_parts_http_url - # Clean event + # Clean event - remove: tag: remove_json_conf field: @@ -206,34 +242,52 @@ processors: - cloudflare_logpush.workers_trace.timestamp - cloudflare_logpush.workers_trace.type if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true tag: remove_duplicated_custom_fields - script: - description: Drops null/empty values recursively. - tag: script_drop_empty_fields + tag: script_to_drop_null_values lang: painless + description: This script processor iterates over the whole document to remove fields with null values. source: |- - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags From 30d8b0ed50ab9942d12c030048cbbcfe14751cb3 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:35:48 +0530 Subject: [PATCH 11/14] implement dynamic DNS mapping and pipeline improvements for dns and dns_firewall data streams Add dynamic mapping for dns.response_code and dns.question.type to follow IANA keyword representations in dns and dns_firewall data streams. Also applies the same pipeline improvements as the previous commit (timestamp consolidation, convert processors, null removal script, standardized error.message, and processor tags) to these two data streams. --- .../elasticsearch/ingest_pipeline/default.yml | 319 ++++++++++--- .../elasticsearch/ingest_pipeline/default.yml | 431 ++++++++++++++---- 2 files changed, 599 insertions(+), 151 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml index 1886ddb6a8b..349ac75a635 100644 --- a/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/dns/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare DNS logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,28 +45,30 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_df40dd8a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: + tag: date_json_Timestamp_70be028a field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' formats: @@ -62,14 +77,17 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_d6556db8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_dns_timestamp_9e5e1fac field: cloudflare_logpush.dns.timestamp copy_from: '@timestamp' ignore_empty_value: true - convert: + tag: convert_json_SourceIP_to_cloudflare_logpush_dns_source_ip_4d627d9e field: json.SourceIP target_field: cloudflare_logpush.dns.source.ip if: ctx.json?.SourceIP != '' @@ -77,35 +95,157 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f1d22510 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_a6b54f72 field: source.ip copy_from: cloudflare_logpush.dns.source.ip ignore_empty_value: true - rename: + tag: rename_json_QueryName_to_cloudflare_logpush_dns_query_name_c1af2645 field: json.QueryName target_field: cloudflare_logpush.dns.query.name ignore_missing: true - set: + tag: set_dns_question_name_2b839091 field: dns.question.name copy_from: cloudflare_logpush.dns.query.name ignore_empty_value: true - convert: field: json.QueryType target_field: cloudflare_logpush.dns.query.type - if: ctx.json?.QueryType != '' type: long ignore_missing: true + tag: convert_dns_query_type on_failure: - append: + tag: append_error_message_31b17e89 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - script: + lang: painless + tag: script_map_dns_question_type + if: ctx.cloudflare_logpush?.dns?.query?.type != null + params: + "0": Reserved + "1": A + "2": NS + "3": MD + "4": MF + "5": CNAME + "6": SOA + "7": MB + "8": MG + "9": MR + "10": "NULL" + "11": WKS + "12": PTR + "13": HINFO + "14": MINFO + "15": MX + "16": TXT + "17": RP + "18": AFSDB + "19": X25 + "20": ISDN + "21": RT + "22": NSAP + "23": NSAP-PTR + "24": SIG + "25": KEY + "26": PX + "27": GPOS + "28": AAAA + "29": LOC + "30": NXT + "31": EID + "32": NIMLOC + "33": SRV + "34": ATMA + "35": NAPTR + "36": KX + "37": CERT + "38": A6 + "39": DNAME + "40": SINK + "41": OPT + "42": APL + "43": DS + "44": SSHFP + "45": IPSECKEY + "46": RRSIG + "47": NSEC + "48": DNSKEY + "49": DHCID + "50": NSEC3 + "51": NSEC3PARAM + "52": TLSA + "53": SMIMEA + "55": HIP + "56": NINFO + "57": RKEY + "58": TALINK + "59": CDS + "60": CDNSKEY + "61": OPENPGPKEY + "62": CSYNC + "63": ZONEMD + "64": SVCB + "65": HTTPS + "66": DSYNC + "67": HHIT + "68": BRID + "99": SPF + "100": UINFO + "101": UID + "102": GID + "103": UNSPEC + "104": NID + "105": L32 + "106": L64 + "107": LP + "108": EUI48 + "109": EUI64 + "128": NXNAME + "249": TKEY + "250": TSIG + "251": IXFR + "252": AXFR + "253": MAILB + "254": MAILA + "255": ANY + "256": URI + "257": CAA + "258": AVC + "259": DOA + "260": AMTRELAY + "261": RESINFO + "262": WALLET + "263": CLA + "264": IPN + "32768": TA + "32769": DLV + source: >- + def queryType = String.valueOf(ctx.cloudflare_logpush.dns.query.type); + ctx.dns = ctx.dns ?: [:]; + ctx.dns.question = ctx.dns.question ?: [:]; + if (params.containsKey(queryType)) { + ctx.dns.question.type = params[queryType]; + } + on_failure: + - append: + tag: append_error_message_bef90659 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ColoCode_to_cloudflare_logpush_dns_colo_code_81807dd5 field: json.ColoCode target_field: cloudflare_logpush.dns.colo.code ignore_missing: true - convert: + tag: convert_json_EDNSSubnet_to_cloudflare_logpush_dns_edns_subnet_03051799 field: json.EDNSSubnet target_field: cloudflare_logpush.dns.edns.subnet if: ctx.json?.EDNSSubnet != '' @@ -113,50 +253,90 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c042b207 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_EDNSSubnetLength_to_cloudflare_logpush_dns_edns_subnet_length_d25956dd field: json.EDNSSubnetLength target_field: cloudflare_logpush.dns.edns.subnet_length - if: ctx.json?.EDNSSubnetLength != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_89468473 field: error.message - value: '{{{_ingest.on_failure_message}}}' - - rename: + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_ResponseCached_to_cloudflare_logpush_dns_response_cached_b8f6f6be field: json.ResponseCached target_field: cloudflare_logpush.dns.response.cached + type: boolean ignore_missing: true + on_failure: + - append: + tag: append_error_message_b9a22af4 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: field: json.ResponseCode target_field: cloudflare_logpush.dns.response.code - if: ctx.json?.ResponseCode != '' type: long ignore_missing: true + tag: convert_dns_response_code on_failure: - append: + tag: append_error_message_1d696893 field: error.message - value: '{{{_ingest.on_failure_message}}}' - - append: - field: related.hosts - value: '{{{dns.query.name}}}' - if: ctx.dns?.query?.name != null - allow_duplicates: false - ignore_failure: true + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - script: + lang: painless + tag: script_map_dns_response_code + if: ctx.cloudflare_logpush?.dns?.response?.code != null + params: + "0": NoError + "1": FormErr + "2": ServFail + "3": NXDomain + "4": NotImp + "5": Refused + "6": YXDomain + "7": YXRRSet + "8": NXRRSet + "9": NotAuth + "10": NotZone + "11": DSOTYPENI + "16": BADVERS + "17": BADKEY + "18": BADTIME + "19": BADMODE + "20": BADNAME + "21": BADALG + "22": BADTRUNC + "23": BADCOOKIE + source: >- + def code = String.valueOf(ctx.cloudflare_logpush.dns.response.code); + ctx.dns = ctx.dns ?: [:]; + if (params.containsKey(code)) { + ctx.dns.response_code = params[code]; + } + on_failure: + - append: + tag: append_error_message_635a8012 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_c321162b field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null + value: '{{{cloudflare_logpush.dns.source.ip}}}' + if: ctx.cloudflare_logpush?.dns?.source?.ip != null allow_duplicates: false - ignore_failure: true - append: + tag: append_related_ip_ca3cb0e9 field: related.ip value: '{{{cloudflare_logpush.dns.edns.subnet}}}' if: ctx.cloudflare_logpush?.dns?.edns?.subnet != null allow_duplicates: false - ignore_failure: true - remove: tag: remove_json_conf field: @@ -164,6 +344,7 @@ processors: - _conf ignore_missing: true - remove: + tag: remove_8d04016b field: - cloudflare_logpush.dns.timestamp - cloudflare_logpush.dns.query.name @@ -172,29 +353,49 @@ processors: ignore_failure: true ignore_missing: true - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/elasticsearch/ingest_pipeline/default.yml index 8318b6bc32f..1212968fa38 100644 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/elasticsearch/ingest_pipeline/default.yml @@ -2,19 +2,26 @@ description: Pipeline for parsing Cloudflare DNS Firewall logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: 8.11.0 - tag: set_ecs_version + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true - tag: rename_event_original + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - tag: json_event_original - set: field: event.kind value: event @@ -27,9 +34,9 @@ processors: field: event.type value: [info] tag: set_event_type -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -38,28 +45,29 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true -# Timestamp + # Timestamp + - convert: + tag: convert_json_Timestamp_7242ddce + field: json.Timestamp + type: long + if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' + ignore_failure: true - script: lang: painless tag: painless_timestamp_to_milli - if: ctx.json?.Timestamp != null + if: ctx.json?.Timestamp != null && ctx.json.Timestamp instanceof Number source: | - try { - long t; - if (ctx.json.Timestamp instanceof String) { - t = Long.parseLong(ctx.json.Timestamp); - } else if (ctx.json.Timestamp instanceof Number) { - t = (long)(ctx.json.Timestamp); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Timestamp = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Timestamp = t*(long)(1e3) - } + long t = (long)(ctx.json.Timestamp); + if (t > (long)(1e18)) { + ctx.json.Timestamp = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Timestamp = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_df40dd8a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.Timestamp if: ctx.json?.Timestamp != null && ctx.json.Timestamp != '' @@ -70,30 +78,40 @@ processors: timezone: UTC tag: date_event_timestamp on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - append: + tag: append_error_message_ccdbdce0 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: cloudflare_logpush.dns_firewall.timestamp copy_from: "@timestamp" tag: set_timestamp + ignore_empty_value: true -# Handle source IP - - rename: + # Handle source IP + - convert: + tag: convert_json_SourceIP_to_cloudflare_logpush_dns_firewall_source_ip_88e15363 field: json.SourceIP target_field: cloudflare_logpush.dns_firewall.source.ip + if: ctx.json?.SourceIP != '' + type: ip ignore_missing: true - tag: rename_source_ip + on_failure: + - append: + tag: append_error_message_63371be1 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: source.ip copy_from: cloudflare_logpush.dns_firewall.source.ip ignore_empty_value: true tag: set_source_ip -# Geo enrichment (source IP) + # Geo enrichment (source IP) - geoip: field: source.ip target_field: source.geo tag: geo_source_ip + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb tag: geo_source_as @@ -114,7 +132,7 @@ processors: ignore_missing: true tag: rename_source_as_org -# DNS ECS fields + # DNS ECS fields - rename: field: json.QueryName target_field: cloudflare_logpush.dns_firewall.question.name @@ -129,37 +147,65 @@ processors: - convert: field: json.ClientResponseCode target_field: cloudflare_logpush.dns_firewall.response.code - if: ctx.json?.ClientResponseCode != '' type: string ignore_missing: true tag: convert_dns_response_code on_failure: - append: + tag: append_error_message_6eacf3d0 field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: field: json.UpstreamResponseCode target_field: cloudflare_logpush.dns_firewall.upstream.response_code - if: ctx.json?.UpstreamResponseCode != '' type: string ignore_missing: true tag: convert_dns_upstream_response_code on_failure: - append: + tag: append_error_message_b90db852 field: error.message - value: '{{{_ingest.on_failure_message}}}' - - set: - field: dns.response_code - copy_from: cloudflare_logpush.dns_firewall.response.code - if: ctx.cloudflare_logpush?.dns_firewall?.response?.code != null - tag: set_dns_response_code - - set: - field: dns.response_code - copy_from: cloudflare_logpush.dns_firewall.upstream.response_code - if: ctx.cloudflare_logpush?.dns_firewall?.response_code == null && ctx.cloudflare_logpush?.dns_firewall?.upstream?.response_code != null - tag: set_dns_upstream_response_code + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - script: + lang: painless + tag: script_map_dns_upstream_response_code + params: + "0": NoError + "1": FormErr + "2": ServFail + "3": NXDomain + "4": NotImp + "5": Refused + "6": YXDomain + "7": YXRRSet + "8": NXRRSet + "9": NotAuth + "10": NotZone + "11": DSOTYPENI + "16": BADVERS + "17": BADKEY + "18": BADTIME + "19": BADMODE + "20": BADNAME + "21": BADALG + "22": BADTRUNC + "23": BADCOOKIE + source: >- + ctx.dns = ctx.dns ?: [:]; + def code1 = ctx.cloudflare_logpush?.dns_firewall?.response?.code; + def code2 = ctx.cloudflare_logpush?.dns_firewall?.upstream?.response_code; + if (code1 != null && params.containsKey(code1)) { + ctx.dns.response_code = params[code1]; + } else if (code2 != null && params.containsKey(code2)) { + ctx.dns.response_code = params[code2]; + } + on_failure: + - append: + tag: append_error_message_2f5264e2 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' -# Custom fields + # Custom fields - rename: field: json.ClusterID target_field: cloudflare_logpush.dns_firewall.cluster_id @@ -170,36 +216,73 @@ processors: target_field: cloudflare_logpush.dns_firewall.colo.code ignore_missing: true tag: rename_colo_code - - rename: + - convert: + tag: convert_json_EDNSSubnet_to_cloudflare_logpush_dns_firewall_edns_subnet_83710560 field: json.EDNSSubnet target_field: cloudflare_logpush.dns_firewall.edns.subnet + if: ctx.json?.EDNSSubnet != '' + type: ip ignore_missing: true - tag: rename_edns_subnet - - rename: + on_failure: + - append: + tag: append_error_message_a2539ba2 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_EDNSSubnetLength_to_cloudflare_logpush_dns_firewall_edns_subnet_length_eb5cf4fe field: json.EDNSSubnetLength target_field: cloudflare_logpush.dns_firewall.edns.subnet_length + type: long ignore_missing: true - tag: rename_edns_subnet_length - - rename: + on_failure: + - append: + tag: append_error_message_10bb6f88 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_QueryDO_to_cloudflare_logpush_dns_firewall_question_dnssec_ok_7cf1a881 field: json.QueryDO target_field: cloudflare_logpush.dns_firewall.question.dnssec_ok + type: boolean ignore_missing: true - tag: rename_question_do - - rename: + on_failure: + - append: + tag: append_error_message_c35decc3 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_QueryRD_to_cloudflare_logpush_dns_firewall_question_recursion_desired_934d6fc8 field: json.QueryRD target_field: cloudflare_logpush.dns_firewall.question.recursion_desired + type: boolean ignore_missing: true - tag: rename_question_recursion_desired - - rename: + on_failure: + - append: + tag: append_error_message_90cd681a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_QuerySize_to_cloudflare_logpush_dns_firewall_question_size_23ed3ba7 field: json.QuerySize target_field: cloudflare_logpush.dns_firewall.question.size + type: long ignore_missing: true - tag: rename_question_size - - rename: + on_failure: + - append: + tag: append_error_message_bf1eaefd + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_QueryTCP_to_cloudflare_logpush_dns_firewall_question_tcp_6d9fe0d7 field: json.QueryTCP target_field: cloudflare_logpush.dns_firewall.question.tcp + type: boolean ignore_missing: true - tag: rename_question_tcp + on_failure: + - append: + tag: append_error_message_d940735d + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: network.transport value: tcp @@ -210,58 +293,204 @@ processors: value: udp if: ctx.cloudflare_logpush?.dns_firewall?.question?.tcp == false tag: set_network_transport_udp - - rename: + - convert: field: json.QueryType target_field: cloudflare_logpush.dns_firewall.question.type + type: long ignore_missing: true - tag: rename_question_type - - rename: + tag: convert_dns_query_type + on_failure: + - append: + tag: append_error_message_563ddffa + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - script: + lang: painless + tag: script_map_dns_question_type + if: ctx.cloudflare_logpush?.dns_firewall?.question?.type != null + params: + "0": Reserved + "1": A + "2": NS + "3": MD + "4": MF + "5": CNAME + "6": SOA + "7": MB + "8": MG + "9": MR + "10": "NULL" + "11": WKS + "12": PTR + "13": HINFO + "14": MINFO + "15": MX + "16": TXT + "17": RP + "18": AFSDB + "19": X25 + "20": ISDN + "21": RT + "22": NSAP + "23": NSAP-PTR + "24": SIG + "25": KEY + "26": PX + "27": GPOS + "28": AAAA + "29": LOC + "30": NXT + "31": EID + "32": NIMLOC + "33": SRV + "34": ATMA + "35": NAPTR + "36": KX + "37": CERT + "38": A6 + "39": DNAME + "40": SINK + "41": OPT + "42": APL + "43": DS + "44": SSHFP + "45": IPSECKEY + "46": RRSIG + "47": NSEC + "48": DNSKEY + "49": DHCID + "50": NSEC3 + "51": NSEC3PARAM + "52": TLSA + "53": SMIMEA + "55": HIP + "56": NINFO + "57": RKEY + "58": TALINK + "59": CDS + "60": CDNSKEY + "61": OPENPGPKEY + "62": CSYNC + "63": ZONEMD + "64": SVCB + "65": HTTPS + "66": DSYNC + "67": HHIT + "68": BRID + "99": SPF + "100": UINFO + "101": UID + "102": GID + "103": UNSPEC + "104": NID + "105": L32 + "106": L64 + "107": LP + "108": EUI48 + "109": EUI64 + "128": NXNAME + "249": TKEY + "250": TSIG + "251": IXFR + "252": AXFR + "253": MAILB + "254": MAILA + "255": ANY + "256": URI + "257": CAA + "258": AVC + "259": DOA + "260": AMTRELAY + "261": RESINFO + "262": WALLET + "263": CLA + "264": IPN + "32768": TA + "32769": DLV + source: >- + def queryType = String.valueOf(ctx.cloudflare_logpush.dns_firewall.question.type); + ctx.dns = ctx.dns ?: [:]; + ctx.dns.question = ctx.dns.question ?: [:]; + if (params.containsKey(queryType)) { + ctx.dns.question.type = params[queryType]; + } + on_failure: + - append: + tag: append_error_message_dd62aa59 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_ResponseCached_to_cloudflare_logpush_dns_firewall_response_cached_5b0a8dad field: json.ResponseCached target_field: cloudflare_logpush.dns_firewall.response.cached + type: boolean ignore_missing: true - tag: rename_dns_response_cached + on_failure: + - append: + tag: append_error_message_b8a0296f + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.ResponseReason target_field: cloudflare_logpush.dns_firewall.response.reason ignore_missing: true tag: rename_dns_response_reason - - rename: + - convert: + tag: convert_json_ResponseCachedStale_to_cloudflare_logpush_dns_firewall_response_cached_stale_8c38a756 field: json.ResponseCachedStale target_field: cloudflare_logpush.dns_firewall.response.cached_stale + type: boolean ignore_missing: true - tag: rename_dns_response_cached_stale - - rename: + on_failure: + - append: + tag: append_error_message_cb8ad2b8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_UpstreamIP_to_cloudflare_logpush_dns_firewall_upstream_ip_0e68065b field: json.UpstreamIP target_field: cloudflare_logpush.dns_firewall.upstream.ip + if: ctx.json?.UpstreamIP != '' + type: ip ignore_missing: true - tag: rename_dns_upstream_ip - - rename: + on_failure: + - append: + tag: append_error_message_e39afed1 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + tag: convert_json_UpstreamResponseTimeMs_to_cloudflare_logpush_dns_firewall_upstream_response_time_ms_adc04a95 field: json.UpstreamResponseTimeMs target_field: cloudflare_logpush.dns_firewall.upstream.response_time_ms + type: long ignore_missing: true - tag: rename_dns_upstream_response_time_ms + on_failure: + - append: + tag: append_error_message_8d671b4f + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' -# Create related fields + # Create related fields - append: field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null && ctx.source.ip != '' + value: '{{{cloudflare_logpush.dns_firewall.source.ip}}}' + if: ctx.cloudflare_logpush?.dns_firewall?.source?.ip != null allow_duplicates: false tag: append_related_source_ip - append: field: related.ip value: '{{{cloudflare_logpush.dns_firewall.edns.subnet}}}' - if: ctx.cloudflare_logpush?.dns_firewall?.edns?.subnet != null && ctx.cloudflare_logpush.dns_firewall.edns.subnet != '' + if: ctx.cloudflare_logpush?.dns_firewall?.edns?.subnet != null allow_duplicates: false tag: append_related_edns_subnet - append: field: related.ip value: '{{{cloudflare_logpush.dns_firewall.upstream.ip}}}' - if: ctx.cloudflare_logpush?.dns_firewall?.upstream?.ip != null && ctx.cloudflare_logpush.dns_firewall.upstream.ip != '' + if: ctx.cloudflare_logpush?.dns_firewall?.upstream?.ip != null allow_duplicates: false tag: append_related_upstream_ip -# Clean resulting event + # Clean resulting event - remove: tag: remove_json_conf field: @@ -273,36 +502,54 @@ processors: - cloudflare_logpush.dns_firewall.timestamp - cloudflare_logpush.dns_firewall.source.ip - cloudflare_logpush.dns_firewall.question.name - - cloudflare_logpush.dns_firewall.response.code if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true tag: remove_duplicated_custom_fields - script: - description: Drops null/empty values recursively. - tag: script_drop_empty_fields + tag: script_to_drop_null_values lang: painless + description: This script processor iterates over the whole document to remove fields with null values. source: |- - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); } - dropEmptyFields(ctx); + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags From e42d8b66168ece0cda62678342517bdd029b4468 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:36:12 +0530 Subject: [PATCH 12/14] add ignore_failure to community_id processor and apply pipeline improvements for network_analytics Add ignore_failure: true to the community_id processor in the network_analytics pipeline to resolve build failures. Also applies the same pipeline improvements (timestamp consolidation, convert processors, null removal script, standardized error.message, and processor tags) to this data stream. --- .../elasticsearch/ingest_pipeline/default.yml | 531 +++++++++++------- 1 file changed, 327 insertions(+), 204 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml index 92ae19e136f..617a215bbf5 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml @@ -2,28 +2,41 @@ description: Pipeline for parsing Cloudflare Network Analytics logs. processors: - set: + tag: set_ecs_version_b777da29 field: ecs.version - value: '8.11.0' + value: '9.3.0' - rename: field: message + tag: rename_message_to_event_original target_field: event.original ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] -## AWS S3 input does _id Based Deduplication and generates "_id" by default. -## When "Data Deduplication" is not enabled, this field must be removed. -## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation + ## AWS S3 input does _id Based Deduplication and generates "_id" by default. + ## When "Data Deduplication" is not enabled, this field must be removed. + ## https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation - remove: field: _id tag: remove_id_based_deduplication @@ -32,29 +45,31 @@ processors: must be removed. if: ctx._conf?.enable_deduplication == false && ctx.input?.type == 'aws-s3' ignore_missing: true + - convert: + tag: convert_json_Datetime_712ea5a9 + field: json.Datetime + type: long + if: ctx.json?.Datetime != null && ctx.json.Datetime != '' + ignore_failure: true - script: lang: painless tag: painless_datetime_to_milli - if: ctx.json?.Datetime != null + if: ctx.json?.Datetime != null && ctx.json.Datetime instanceof Number source: | - try { - long t; - if (ctx.json.Datetime instanceof String) { - t = Long.parseLong(ctx.json.Datetime); - } else if (ctx.json.Datetime instanceof Number) { - t = (long)(ctx.json.Datetime); - } else { - return; - } - if (t > (long)(1e18)) { - ctx.json.Datetime = t/(long)(1e6) - } else if (t < (long)(1e10)) { - ctx.json.Datetime = t*(long)(1e3) - } + long t = (long)(ctx.json.Datetime); + if (t > (long)(1e18)) { + ctx.json.Datetime = t/(long)(1e6) + } else if (t < (long)(1e10)) { + ctx.json.Datetime = t*(long)(1e3) } - catch (Exception e) {} + on_failure: + - append: + tag: append_error_message_6318b156 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.Datetime + tag: date_json_datetime if: ctx.json?.Datetime != null && ctx.json.Datetime != '' formats: - UNIX_MS @@ -62,22 +77,31 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ timezone: UTC on_failure: - - append: - field: error.message - value: "{{{_ingest.on_failure_message}}}" + - remove: + tag: remove_json_Datetime_fff9e67e + field: json.Datetime + ignore_missing: true + - append: + tag: append_error_message_a5ae079a + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_cloudflare_logpush_network_analytics_timestamp_5d354fd6 field: cloudflare_logpush.network_analytics.timestamp copy_from: '@timestamp' ignore_empty_value: true - set: + tag: set_cloudflare_logpush_network_analytics_outcome_4476b418 field: cloudflare_logpush.network_analytics.outcome value: success if: ctx.json?.Outcome == 'pass' - set: + tag: set_cloudflare_logpush_network_analytics_outcome_cfd333fb field: cloudflare_logpush.network_analytics.outcome value: failure if: ctx.json?.Outcome == 'drop' - set: + tag: set_event_outcome_61f32c56 field: event.outcome copy_from: cloudflare_logpush.network_analytics.outcome ignore_empty_value: true @@ -85,15 +109,15 @@ processors: tag: convert_json_destination_asn field: json.DestinationASN target_field: cloudflare_logpush.network_analytics.destination.asn - if: ctx.json?.DestinationASN != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8c387d1e field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_as_number_d9408f17 field: destination.as.number copy_from: cloudflare_logpush.network_analytics.destination.asn ignore_empty_value: true @@ -106,10 +130,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_07b2ad6a field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_ip_84cf708a field: destination.ip copy_from: cloudflare_logpush.network_analytics.destination.ip ignore_empty_value: true @@ -117,37 +142,63 @@ processors: tag: convert_json_destination_port field: json.DestinationPort target_field: cloudflare_logpush.network_analytics.destination.port - if: ctx.json?.DestinationPort != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_3b604b03 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port_3c5d6b46 field: destination.port copy_from: cloudflare_logpush.network_analytics.destination.port ignore_empty_value: true - rename: + tag: rename_json_Direction_to_cloudflare_logpush_network_analytics_direction_cbfd61b5 field: json.Direction target_field: cloudflare_logpush.network_analytics.direction ignore_missing: true - set: + tag: set_network_direction_21150cc8 field: network.direction copy_from: cloudflare_logpush.network_analytics.direction ignore_empty_value: true - rename: + tag: rename_json_IPProtocolName_to_cloudflare_logpush_network_analytics_ip_protocol_name_1e143b25 field: json.IPProtocolName target_field: cloudflare_logpush.network_analytics.ip.protocol.name ignore_missing: true - set: + tag: set_network_transport_90b70a4d field: network.transport copy_from: cloudflare_logpush.network_analytics.ip.protocol.name ignore_empty_value: true - lowercase: + tag: lowercase_network_transport_bc8c1c12 field: network.transport ignore_missing: true + - rename: + tag: rename_json_DNSQueryName_to_cloudflare_logpush_network_analytics_dns_query_name_edc75b81 + field: json.DNSQueryName + target_field: cloudflare_logpush.network_analytics.dns.query.name + ignore_missing: true + - rename: + tag: rename_json_DNSQueryType_to_cloudflare_logpush_network_analytics_dns_query_type_37315fd3 + field: json.DNSQueryType + target_field: cloudflare_logpush.network_analytics.dns.query.type + ignore_missing: true + - convert: + tag: convert_json_pfp_custom_tag + field: json.PFPCustomTag + target_field: cloudflare_logpush.network_analytics.pfp_custom_tag + type: long + ignore_missing: true + on_failure: + - append: + tag: append_error_message_479b27c8 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_ip_source_address field: json.IPSourceAddress @@ -157,10 +208,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e09cfcb6 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_a7297980 field: source.ip copy_from: cloudflare_logpush.network_analytics.source.ip ignore_empty_value: true @@ -168,15 +220,15 @@ processors: tag: convert_json_source_asn field: json.SourceASN target_field: cloudflare_logpush.network_analytics.source.asn - if: ctx.json?.SourceASN != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_179a2a0f field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_as_number_3022b95d field: source.as.number copy_from: cloudflare_logpush.network_analytics.source.asn ignore_empty_value: true @@ -184,55 +236,75 @@ processors: tag: convert_json_source_port field: json.SourcePort target_field: cloudflare_logpush.network_analytics.source.port - if: ctx.json?.SourcePort != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_392dab8c field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_port_7623a0b8 field: source.port copy_from: cloudflare_logpush.network_analytics.source.port ignore_empty_value: true - rename: + tag: rename_json_RuleID_to_cloudflare_logpush_network_analytics_rule_id_2b8c6abd field: json.RuleID target_field: cloudflare_logpush.network_analytics.rule.id ignore_missing: true - set: + tag: set_rule_id_5728b9d8 field: rule.id copy_from: cloudflare_logpush.network_analytics.rule.id ignore_empty_value: true - rename: + tag: rename_json_AttackCampaignID_to_cloudflare_logpush_network_analytics_attack_campaign_id_472be873 field: json.AttackCampaignID target_field: cloudflare_logpush.network_analytics.attack.campaign.id ignore_missing: true + - set: + field: threat.group.id + tag: set_threat_group_id_from_attack_campaign_id + copy_from: cloudflare_logpush.network_analytics.attack.campaign.id + ignore_empty_value: true - rename: + tag: rename_json_AttackID_to_cloudflare_logpush_network_analytics_attack_id_3d175e85 field: json.AttackID target_field: cloudflare_logpush.network_analytics.attack.id ignore_missing: true + - set: + field: threat.indicator.id + tag: set_threat_indicator_id_from_attack_id + value: ['{{{cloudflare_logpush.network_analytics.attack.id}}}'] + if: ctx.cloudflare_logpush?.network_analytics?.attack?.id != null && ctx.cloudflare_logpush?.network_analytics?.attack?.id != '' - rename: + tag: rename_json_AttackVector_to_cloudflare_logpush_network_analytics_attack_vector_ee323f41 field: json.AttackVector target_field: cloudflare_logpush.network_analytics.attack.vector ignore_missing: true - rename: + tag: rename_json_ColoCity_to_cloudflare_logpush_network_analytics_colo_city_a53dc7a7 field: json.ColoCity target_field: cloudflare_logpush.network_analytics.colo.city ignore_missing: true - rename: + tag: rename_json_ColoCode_to_cloudflare_logpush_network_analytics_colo_code_c7a63d3b field: json.ColoCode target_field: cloudflare_logpush.network_analytics.colo.code ignore_missing: true - rename: + tag: rename_json_ColoCountry_to_cloudflare_logpush_network_analytics_colo_country_d6750495 field: json.ColoCountry target_field: cloudflare_logpush.network_analytics.colo.country ignore_missing: true - rename: + tag: rename_json_ColoGeoHash_to_cloudflare_logpush_network_analytics_colo_geo_hash_f884fb06 field: json.ColoGeoHash target_field: cloudflare_logpush.network_analytics.colo.geo_hash ignore_missing: true - set: + tag: set_cloudflare_logpush_network_analytics_colo_geo_location_6969330e field: cloudflare_logpush.network_analytics.colo.geo_location copy_from: cloudflare_logpush.network_analytics.colo.geo_hash ignore_empty_value: true @@ -240,35 +312,40 @@ processors: tag: convert_json_colo_id field: json.ColoID target_field: cloudflare_logpush.network_analytics.colo.id - if: ctx.json?.ColoID != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_f36483c0 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ColoName_to_cloudflare_logpush_network_analytics_colo_name_4eba22ef field: json.ColoName target_field: cloudflare_logpush.network_analytics.colo.name ignore_missing: true - rename: + tag: rename_json_DestinationASNDescription_to_cloudflare_logpush_network_analytics_destination_as_number_description_6c53e992 field: json.DestinationASNDescription target_field: cloudflare_logpush.network_analytics.destination.as.number.description ignore_missing: true - rename: + tag: rename_json_DestinationASNName_to_cloudflare_logpush_network_analytics_destination_as_number_name_f0a751ba field: json.DestinationASNName target_field: cloudflare_logpush.network_analytics.destination.as.number.name ignore_missing: true - rename: + tag: rename_json_DestinationCountry_to_cloudflare_logpush_network_analytics_destination_country_41b25d39 field: json.DestinationCountry target_field: cloudflare_logpush.network_analytics.destination.country ignore_missing: true - rename: + tag: rename_json_DestinationGeoHash_to_cloudflare_logpush_network_analytics_destination_geo_hash_3760770c field: json.DestinationGeoHash target_field: cloudflare_logpush.network_analytics.destination.geo_hash ignore_missing: true - set: + tag: set_cloudflare_logpush_network_analytics_destination_geo_location_7feda026 field: cloudflare_logpush.network_analytics.destination.geo_location copy_from: cloudflare_logpush.network_analytics.destination.geo_hash ignore_empty_value: true @@ -276,111 +353,103 @@ processors: tag: convert_json_gre_checksum field: json.GREChecksum target_field: cloudflare_logpush.network_analytics.gre.checksum - if: ctx.json?.GREChecksum != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_d6c994e7 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_gre_ethertype field: json.GREEthertype target_field: cloudflare_logpush.network_analytics.gre.ether.type - if: ctx.json?.GREEthertype != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_e7b8d652 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_gre_header_length field: json.GREHeaderLength target_field: cloudflare_logpush.network_analytics.gre.header.length - if: ctx.json?.GREHeaderLength != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8fb446d8 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_key field: json.GREKey target_field: cloudflare_logpush.network_analytics.gre.key - if: ctx.json?.GREKey != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_bc84aa22 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_sequence_number field: json.GRESequenceNumber target_field: cloudflare_logpush.network_analytics.gre.sequence.number - if: ctx.json?.GRESequenceNumber != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_268ef8c0 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_gre_version field: json.GREVersion target_field: cloudflare_logpush.network_analytics.gre.version - if: ctx.json?.GREVersion != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_3f5275bc field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_icmp_checksum field: json.ICMPChecksum target_field: cloudflare_logpush.network_analytics.icmp.checksum - if: ctx.json?.ICMPChecksum != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_b1f69c64 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_icmp_code field: json.ICMPCode target_field: cloudflare_logpush.network_analytics.icmp.code - if: ctx.json?.ICMPCode != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_f8ea7192 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_icmp_type field: json.ICMPType target_field: cloudflare_logpush.network_analytics.icmp.type - if: ctx.json?.ICMPType != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_96a9655b field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPDestinationSubnet_to_cloudflare_logpush_network_analytics_ip_destination_subnet_82888579 field: json.IPDestinationSubnet target_field: cloudflare_logpush.network_analytics.ip.destination.subnet ignore_missing: true @@ -388,51 +457,48 @@ processors: tag: convert_json_ip_fragment_offset field: json.IPFragmentOffset target_field: cloudflare_logpush.network_analytics.ip.fragment.offset - if: ctx.json?.IPFragmentOffset != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_b76f8c89 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ip_header_length field: json.IPHeaderLength target_field: cloudflare_logpush.network_analytics.ip.header.length - if: ctx.json?.IPHeaderLength != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_715942ab field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ip_more_fragments field: json.IPMoreFragments target_field: cloudflare_logpush.network_analytics.ip.more.fragments - if: ctx.json?.IPMoreFragments != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_a98dfab4 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ip_protocol field: json.IPProtocol target_field: cloudflare_logpush.network_analytics.ip.protocol.value - if: ctx.json?.IPProtocol != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_e48f827a field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPSourceSubnet_to_cloudflare_logpush_network_analytics_ip_source_subnet_a13f22f5 field: json.IPSourceSubnet target_field: cloudflare_logpush.network_analytics.ip.source.subnet ignore_missing: true @@ -440,37 +506,38 @@ processors: tag: convert_json_ip_total_length field: json.IPTotalLength target_field: cloudflare_logpush.network_analytics.ip.total.length.value - if: ctx.json?.IPTotalLength != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2c02da8f field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ip_total_length_buckets field: json.IPTotalLengthBuckets target_field: cloudflare_logpush.network_analytics.ip.total.length.buckets - if: ctx.json?.IPTotalLengthBuckets != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_197cc77c field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPTTL_to_cloudflare_logpush_network_analytics_ip_ttl_value_a7e5501d field: json.IPTTL target_field: cloudflare_logpush.network_analytics.ip.ttl.value if: ctx.json?.IPTTL != null && ctx.json.IPTTL != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_ip_ttl_value_118eaadb field: cloudflare_logpush.network_analytics.ip.ttl.value value: '{{{json.IPTtl}}}' if: ctx.cloudflare_logpush?.network_analytics?.ip?.ttl?.value != null && ctx.json?.IPTtl != null && ctx.json.IPTtl != '' && ctx.json.IPTtl != ctx.cloudflare_logpush.network_analytics.ip.ttl.value allow_duplicates: false - rename: + tag: rename_json_IPTtl_to_cloudflare_logpush_network_analytics_ip_ttl_value_5661224a field: json.IPTtl target_field: cloudflare_logpush.network_analytics.ip.ttl.value if: ctx.cloudflare_logpush?.network_analytics?.ip?.ttl?.value == null && ctx.json?.IPTtl != null && ctx.json.IPTtl != '' @@ -478,25 +545,27 @@ processors: - convert: tag: convert_ip_ttl_value field: cloudflare_logpush.network_analytics.ip.ttl.value - if: ctx.cloudflare_logpush?.network_analytics?.ip?.ttl?.value != null && ctx.cloudflare_logpush.network_analytics.ip.ttl.value != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_17060000 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPTTLBuckets_to_cloudflare_logpush_network_analytics_ip_ttl_buckets_a9999f1a field: json.IPTTLBuckets target_field: cloudflare_logpush.network_analytics.ip.ttl.buckets if: ctx.json?.IPTTLBuckets != null && ctx.json.IPTTLBuckets != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_ip_ttl_buckets_36d32073 field: cloudflare_logpush.network_analytics.ip.ttl.buckets value: '{{{json.IPTtlBuckets}}}' if: ctx.cloudflare_logpush?.network_analytics?.ip?.ttl?.buckets != null && ctx.json?.IPTtlBuckets != null && ctx.json.IPTtlBuckets != '' && ctx.json.IPTtlBuckets != ctx.cloudflare_logpush.network_analytics.ip.ttl.buckets allow_duplicates: false - rename: + tag: rename_json_IPTtlBuckets_to_cloudflare_logpush_network_analytics_ip_ttl_buckets_1321b5b5 field: json.IPTtlBuckets target_field: cloudflare_logpush.network_analytics.ip.ttl.buckets if: ctx.cloudflare_logpush?.network_analytics?.ip?.ttl?.buckets == null && ctx.json?.IPTtlBuckets != null && ctx.json.IPTtlBuckets != '' @@ -504,49 +573,49 @@ processors: - convert: tag: convert_ip_ttl_buckets field: cloudflare_logpush.network_analytics.ip.ttl.buckets - if: ctx.cloudflare_logpush?.network_analytics?.ip?.ttl?.buckets != null && ctx.cloudflare_logpush.network_analytics.ip.ttl.buckets != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6bf19134 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ipv4_checksum field: json.IPv4Checksum target_field: cloudflare_logpush.network_analytics.ipv4.checksum - if: ctx.json?.IPv4Checksum != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c79f5fbe field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ipv4_dont_fragment field: json.IPv4DontFragment target_field: cloudflare_logpush.network_analytics.ipv4.dont_fragment - if: ctx.json?.IPv4DontFragment != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_9b4f4eaa field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPv4DSCP_to_cloudflare_logpush_network_analytics_ipv4_dscp_d9a537ca field: json.IPv4DSCP target_field: cloudflare_logpush.network_analytics.ipv4.dscp if: ctx.json?.IPv4DSCP != null && ctx.json.IPv4DSCP != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_ipv4_dscp_f86dedeb field: cloudflare_logpush.network_analytics.ipv4.dscp value: '{{{json.IPv4Dscp}}}' if: ctx.cloudflare_logpush?.network_analytics?.ipv4?.dscp != null && ctx.json?.IPv4Dscp != null && ctx.json.IPv4Dscp != '' && ctx.json.IPv4Dscp != ctx.cloudflare_logpush.network_analytics.ipv4.dscp allow_duplicates: false - rename: + tag: rename_json_IPv4Dscp_to_cloudflare_logpush_network_analytics_ipv4_dscp_59bdb9b1 field: json.IPv4Dscp target_field: cloudflare_logpush.network_analytics.ipv4.dscp if: ctx.cloudflare_logpush?.network_analytics?.ipv4?.dscp == null && ctx.json?.IPv4Dscp != null && ctx.json.IPv4Dscp != '' @@ -554,25 +623,27 @@ processors: - convert: tag: convert_ipv4_dscp field: cloudflare_logpush.network_analytics.ipv4.dscp - if: ctx.cloudflare_logpush?.network_analytics?.ipv4?.dscp != null && ctx.cloudflare_logpush.network_analytics.ipv4.dscp != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_594a6bcb field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPv4ECN_to_cloudflare_logpush_network_analytics_ipv4_ecn_f6376678 field: json.IPv4ECN target_field: cloudflare_logpush.network_analytics.ipv4.ecn if: ctx.json?.IPv4ECN != null && ctx.json.IPv4ECN != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_ipv4_ecn_e639006d field: cloudflare_logpush.network_analytics.ipv4.ecn value: '{{{json.IPv4Ecn}}}' if: ctx.cloudflare_logpush?.network_analytics?.ipv4?.ecn != null && ctx.json?.IPv4Ecn != null && ctx.json.IPv4Ecn != '' && ctx.json.IPv4Ecn != ctx.cloudflare_logpush.network_analytics.ipv4.ecn allow_duplicates: false - rename: + tag: rename_json_IPv4Ecn_to_cloudflare_logpush_network_analytics_ipv4_ecn_e9c2129f field: json.IPv4Ecn target_field: cloudflare_logpush.network_analytics.ipv4.ecn if: ctx.cloudflare_logpush?.network_analytics?.ipv4?.ecn == null && ctx.json?.IPv4Ecn != null && ctx.json.IPv4Ecn != '' @@ -580,49 +651,50 @@ processors: - convert: tag: convert_ipv4_ecn field: cloudflare_logpush.network_analytics.ipv4.ecn - if: ctx.cloudflare_logpush?.network_analytics?.ipv4?.ecn != null && ctx.cloudflare_logpush.network_analytics.ipv4.ecn != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_e61ebc85 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ipv4_identification field: json.IPv4Identification target_field: cloudflare_logpush.network_analytics.ipv4.identification - if: ctx.json?.IPv4Identification != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_75491593 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ipv4_options field: json.IPv4Options target_field: cloudflare_logpush.network_analytics.ipv4.options - if: ctx.json?.IPv4Options != '' + if: ctx.json?.IPv4Options != null && ctx.json.IPv4Options != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_9973f13e field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPv6DSCP_to_cloudflare_logpush_network_analytics_ipv6_dscp_29df8bce field: json.IPv6DSCP target_field: cloudflare_logpush.network_analytics.ipv6.dscp if: ctx.json?.IPv6DSCP != null && ctx.json.IPv6DSCP != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_ipv6_dscp_c6b9d3f9 field: cloudflare_logpush.network_analytics.ipv6.dscp value: '{{{json.IPv6Dscp}}}' if: ctx.cloudflare_logpush?.network_analytics?.ipv6?.dscp != null && ctx.json?.IPv6Dscp != null && ctx.json.IPv6Dscp != '' && ctx.json.IPv6Dscp != ctx.cloudflare_logpush.network_analytics.ipv6.dscp allow_duplicates: false - rename: + tag: rename_json_IPv6Dscp_to_cloudflare_logpush_network_analytics_ipv6_dscp_9377b07f field: json.IPv6Dscp target_field: cloudflare_logpush.network_analytics.ipv6.dscp if: ctx.cloudflare_logpush?.network_analytics?.ipv6?.dscp == null && ctx.json?.IPv6Dscp != null && ctx.json.IPv6Dscp != '' @@ -630,25 +702,27 @@ processors: - convert: tag: convert_ipv6_dscp field: cloudflare_logpush.network_analytics.ipv6.dscp - if: ctx.cloudflare_logpush?.network_analytics?.ipv6?.dscp != null && ctx.cloudflare_logpush.network_analytics.ipv6.dscp != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_85bf8cbf field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPv6ECN_to_cloudflare_logpush_network_analytics_ipv6_ecn_7091f9d8 field: json.IPv6ECN target_field: cloudflare_logpush.network_analytics.ipv6.ecn if: ctx.json?.IPv6ECN != null && ctx.json.IPv6ECN != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_ipv6_ecn_8abd78a3 field: cloudflare_logpush.network_analytics.ipv6.ecn value: '{{{json.IPv6Ecn}}}' if: ctx.cloudflare_logpush?.network_analytics?.ipv6?.ecn != null && ctx.json?.IPv6Ecn != null && ctx.json.IPv6Ecn != '' && ctx.json.IPv6Ecn != ctx.cloudflare_logpush.network_analytics.ipv6.ecn allow_duplicates: false - rename: + tag: rename_json_IPv6Ecn_to_cloudflare_logpush_network_analytics_ipv6_ecn_7256fda5 field: json.IPv6Ecn target_field: cloudflare_logpush.network_analytics.ipv6.ecn if: ctx.cloudflare_logpush?.network_analytics?.ipv6?.ecn == null && ctx.json?.IPv6Ecn != null && ctx.json.IPv6Ecn != '' @@ -656,15 +730,15 @@ processors: - convert: tag: convert_ipv6_ecn field: cloudflare_logpush.network_analytics.ipv6.ecn - if: ctx.cloudflare_logpush?.network_analytics?.ipv6?.ecn != null && ctx.cloudflare_logpush.network_analytics.ipv6.ecn != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_448b096d field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_IPv6ExtensionHeaders_to_cloudflare_logpush_network_analytics_ipv6_extension_headers_cdd8e128 field: json.IPv6ExtensionHeaders target_field: cloudflare_logpush.network_analytics.ipv6.extension_headers ignore_missing: true @@ -672,51 +746,56 @@ processors: tag: convert_json_ipv6_flow_label field: json.IPv6FlowLabel target_field: cloudflare_logpush.network_analytics.ipv6.flow_label - if: ctx.json?.IPv6FlowLabel != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_f720b077 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_ipv6_identification field: json.IPv6Identification target_field: cloudflare_logpush.network_analytics.ipv6.identification - if: ctx.json?.IPv6Identification != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_d48801c5 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_MitigationReason_to_cloudflare_logpush_network_analytics_mitigation_reason_59208f9b field: json.MitigationReason target_field: cloudflare_logpush.network_analytics.mitigation.reason ignore_missing: true - rename: + tag: rename_json_MitigationScope_to_cloudflare_logpush_network_analytics_mitigation_scope_6d29f385 field: json.MitigationScope target_field: cloudflare_logpush.network_analytics.mitigation.scope ignore_missing: true - rename: + tag: rename_json_MitigationSystem_to_cloudflare_logpush_network_analytics_mitigation_system_0ac311db field: json.MitigationSystem target_field: cloudflare_logpush.network_analytics.mitigation.system ignore_missing: true - rename: + tag: rename_json_ProtocolState_to_cloudflare_logpush_network_analytics_protocol_state_1e631304 field: json.ProtocolState target_field: cloudflare_logpush.network_analytics.protocol_state ignore_missing: true - rename: + tag: rename_json_RuleName_to_cloudflare_logpush_network_analytics_rule_name_4d8b0585 field: json.RuleName target_field: cloudflare_logpush.network_analytics.rule.name ignore_missing: true - rename: + tag: rename_json_RulesetID_to_cloudflare_logpush_network_analytics_rule_set_id_ce25cb79 field: json.RulesetID target_field: cloudflare_logpush.network_analytics.rule.set.id ignore_missing: true - rename: + tag: rename_json_RulesetOverrideID_to_cloudflare_logpush_network_analytics_rule_set_override_id_eee4e7fb field: json.RulesetOverrideID target_field: cloudflare_logpush.network_analytics.rule.set.override.id ignore_missing: true @@ -724,31 +803,35 @@ processors: tag: convert_json_sample_interval field: json.SampleInterval target_field: cloudflare_logpush.network_analytics.sample_interval - if: ctx.json?.SampleInterval != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6f644e40 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_SourceASNName_to_cloudflare_logpush_network_analytics_source_as_number_name_b995e894 field: json.SourceASNName target_field: cloudflare_logpush.network_analytics.source.as.number.name ignore_missing: true - rename: + tag: rename_json_SourceASNDescription_to_cloudflare_logpush_network_analytics_source_as_number_description_b4edf388 field: json.SourceASNDescription target_field: cloudflare_logpush.network_analytics.source.as.number.description ignore_missing: true - rename: + tag: rename_json_SourceCountry_to_cloudflare_logpush_network_analytics_source_country_ffe895c9 field: json.SourceCountry target_field: cloudflare_logpush.network_analytics.source.country ignore_missing: true - rename: + tag: rename_json_SourceGeoHash_to_cloudflare_logpush_network_analytics_source_geo_hash_1f37e21e field: json.SourceGeoHash target_field: cloudflare_logpush.network_analytics.source.geo_hash ignore_missing: true - set: + tag: set_cloudflare_logpush_network_analytics_source_geo_location_d48297d2 field: cloudflare_logpush.network_analytics.source.geo_location copy_from: cloudflare_logpush.network_analytics.source.geo_hash ignore_empty_value: true @@ -756,65 +839,65 @@ processors: tag: convert_json_tcp_acknowledgement_number field: json.TCPAcknowledgementNumber target_field: cloudflare_logpush.network_analytics.tcp.acknowledgement_number - if: ctx.json?.TCPAcknowledgementNumber != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ee72c534 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_checksum field: json.TCPChecksum target_field: cloudflare_logpush.network_analytics.tcp.checksum - if: ctx.json?.TCPChecksum != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_67af8966 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_data_offset field: json.TCPDataOffset target_field: cloudflare_logpush.network_analytics.tcp.dataoffset - if: ctx.json?.TCPDataOffset != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_9bf24e53 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_flags field: json.TCPFlags target_field: cloudflare_logpush.network_analytics.tcp.flags.value - if: ctx.json?.TCPFlags != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2d02092b field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_TCPFlagsString_to_cloudflare_logpush_network_analytics_tcp_flags_string_df4680fd field: json.TCPFlagsString target_field: cloudflare_logpush.network_analytics.tcp.flags.string ignore_missing: true - rename: + tag: rename_json_TCPMSS_to_cloudflare_logpush_network_analytics_tcp_mss_f53c9bd6 field: json.TCPMSS target_field: cloudflare_logpush.network_analytics.tcp.mss if: ctx.json?.TCPMSS != null && ctx.json.TCPMSS != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_tcp_mss_33576750 field: cloudflare_logpush.network_analytics.tcp.mss value: '{{{json.TCPMss}}}' if: ctx.cloudflare_logpush?.network_analytics?.tcp?.mss != null && ctx.json?.TCPMss != null && ctx.json.TCPMss != '' && ctx.json.TCPMss != ctx.cloudflare_logpush.network_analytics.tcp.mss allow_duplicates: false - rename: + tag: rename_json_TCPMss_to_cloudflare_logpush_network_analytics_tcp_mss_2b90b11c field: json.TCPMss target_field: cloudflare_logpush.network_analytics.tcp.mss if: ctx.cloudflare_logpush?.network_analytics?.tcp?.mss == null && ctx.json?.TCPMss != null && ctx.json.TCPMss != '' @@ -822,41 +905,49 @@ processors: - convert: tag: convert_tcp_mss field: cloudflare_logpush.network_analytics.tcp.mss - if: ctx.cloudflare_logpush?.network_analytics?.tcp?.mss != null && ctx.cloudflare_logpush.network_analytics.tcp.mss != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_5c02a0a1 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_TCPOptions_to_cloudflare_logpush_network_analytics_tcp_options_0cd59237 field: json.TCPOptions target_field: cloudflare_logpush.network_analytics.tcp.options ignore_missing: true - split: + tag: split_json_TCPSACKBlocks_e086a612 field: json.TCPSACKBlocks separator: ', *' if: ctx.json?.TCPSACKBlocks instanceof String && ctx.json.TCPSACKBlocks != '' - - rename: - field: json.TCPSACKBlocks - target_field: cloudflare_logpush.network_analytics.tcp.sack.blocks - ignore_missing: true - split: + tag: split_json_TCPSackBlocks_42519872 field: json.TCPSackBlocks separator: ', *' if: ctx.json?.TCPSackBlocks instanceof String && ctx.json.TCPSackBlocks != '' - script: tag: script_raise_non_string_tcp_sack_blocks_to_array source: |- - if (ctx.json?.TCPSACKBlocks != null && !(ctx.json.TCPSACKBlocks instanceof List)) { + if (ctx.json?.TCPSACKBlocks != null && ctx.json.TCPSACKBlocks != '' && !(ctx.json.TCPSACKBlocks instanceof List)) { ctx.json.TCPSACKBlocks = [ctx.json.TCPSACKBlocks]; } - if (ctx.json?.TCPSackBlocks != null && !(ctx.json.TCPSackBlocks instanceof List)) { + if (ctx.json?.TCPSackBlocks != null && ctx.json.TCPSackBlocks != '' && !(ctx.json.TCPSackBlocks instanceof List)) { ctx.json.TCPSackBlocks = [ctx.json.TCPSackBlocks]; } + on_failure: + - append: + tag: append_error_message_3b61f5c9 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + tag: rename_json_TCPSACKBlocks_to_cloudflare_logpush_network_analytics_tcp_sack_blocks_fce9ee1d + field: json.TCPSACKBlocks + target_field: cloudflare_logpush.network_analytics.tcp.sack.blocks + ignore_missing: true - foreach: - tag: + tag: foreach_tcp_sack_blocks field: json.TCPSackBlocks if: ctx.json?.TCPSackBlocks instanceof List processor: @@ -872,20 +963,23 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_47583c58 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_TCPSACKPermitted_to_cloudflare_logpush_network_analytics_tcp_sack_permitted_324342d0 field: json.TCPSACKPermitted target_field: cloudflare_logpush.network_analytics.tcp.sack.permitted if: ctx.json?.TCPSACKPermitted != null && ctx.json.TCPSACKPermitted != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_tcp_sack_permitted_b734c4b6 field: cloudflare_logpush.network_analytics.tcp.sack.permitted value: '{{{json.TCPSacksPermitted}}}' if: ctx.cloudflare_logpush?.network_analytics?.tcp?.sack?.permitted != null && ctx.json?.TCPSacksPermitted != null && ctx.json.TCPSacksPermitted != '' && ctx.json.TCPSacksPermitted != ctx.cloudflare_logpush.network_analytics.tcp.sack.permitted allow_duplicates: false - rename: + tag: rename_json_TCPSacksPermitted_to_cloudflare_logpush_network_analytics_tcp_sack_permitted_b2646965 field: json.TCPSacksPermitted target_field: cloudflare_logpush.network_analytics.tcp.sack.permitted if: ctx.cloudflare_logpush?.network_analytics?.tcp?.sack?.permitted == null && ctx.json?.TCPSacksPermitted != null && ctx.json.TCPSacksPermitted != '' @@ -893,37 +987,38 @@ processors: - convert: tag: convert_tcp_sack_permitted field: cloudflare_logpush.network_analytics.tcp.sack.permitted - if: ctx.cloudflare_logpush?.network_analytics?.tcp?.sack?.permitted != null && ctx.cloudflare_logpush.network_analytics.tcp.sack.permitted != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0dd72184 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_sequence_number field: json.TCPSequenceNumber target_field: cloudflare_logpush.network_analytics.tcp.sequence_number - if: ctx.json?.TCPSequenceNumber != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8b898e0b field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_TCPTimestampECR_to_cloudflare_logpush_network_analytics_tcp_timestamp_ecr_b92c89a2 field: json.TCPTimestampECR target_field: cloudflare_logpush.network_analytics.tcp.timestamp.ecr if: ctx.json?.TCPTimestampECR != null && ctx.json.TCPTimestampECR != '' ignore_missing: true - append: + tag: append_cloudflare_logpush_network_analytics_tcp_timestamp_ecr_e0048eae field: cloudflare_logpush.network_analytics.tcp.timestamp.ecr value: '{{{json.TCPTimestampEcr}}}' if: ctx.cloudflare_logpush?.network_analytics?.tcp?.timestamp?.ecr != null && ctx.json?.TCPTimestampEcr != null && ctx.json.TCPTimestampEcr != '' && ctx.json.TCPTimestampEcr != ctx.cloudflare_logpush.network_analytics.tcp.timestamp.ecr allow_duplicates: false - rename: + tag: rename_json_TCPTimestampEcr_to_cloudflare_logpush_network_analytics_tcp_timestamp_ecr_7cf8e2a0 field: json.TCPTimestampEcr target_field: cloudflare_logpush.network_analytics.tcp.timestamp.ecr if: ctx.cloudflare_logpush?.network_analytics?.tcp?.timestamp?.ecr == null && ctx.json?.TCPTimestampEcr != null && ctx.json.TCPTimestampEcr != '' @@ -931,121 +1026,122 @@ processors: - convert: tag: convert_tcp_timestamp_ecr field: cloudflare_logpush.network_analytics.tcp.timestamp.ecr - if: ctx.cloudflare_logpush?.network_analytics?.tcp?.timestamp?.ecr != null && ctx.cloudflare_logpush.network_analytics.tcp.timestamp.ecr != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8b17b78a field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_timestamp_value field: json.TCPTimestampValue target_field: cloudflare_logpush.network_analytics.tcp.timestamp.value - if: ctx.json?.TCPTimestampValue != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c0380ca1 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_urgent_pointer field: json.TCPUrgentPointer target_field: cloudflare_logpush.network_analytics.tcp.urgent_pointer - if: ctx.json?.TCPUrgentPointer != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_36c60d3d field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_window_scale field: json.TCPWindowScale target_field: cloudflare_logpush.network_analytics.tcp.window.scale - if: ctx.json?.TCPWindowScale != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2643abc6 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_tcp_window_size field: json.TCPWindowSize target_field: cloudflare_logpush.network_analytics.tcp.window.size - if: ctx.json?.TCPWindowSize != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_d598ad6b field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_udp_checksum field: json.UDPChecksum target_field: cloudflare_logpush.network_analytics.udp.checksum - if: ctx.json?.UDPChecksum != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_bff0a08e field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: tag: convert_json_udp_payload_length field: json.UDPPayloadLength target_field: cloudflare_logpush.network_analytics.udp.payload_length - if: ctx.json?.UDPPayloadLength != '' type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2b2b9a71 field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_Verdict_to_cloudflare_logpush_network_analytics_verdict_78867681 field: json.Verdict target_field: cloudflare_logpush.network_analytics.verdict ignore_missing: true - append: field: related.hash + tag: append_source_geo_hash_into_related_hash value: '{{{cloudflare_logpush.network_analytics.source.geo_hash}}}' if: ctx.cloudflare_logpush?.network_analytics?.source?.geo_hash != null allow_duplicates: false - ignore_failure: true - append: field: related.hash + tag: append_destination_geo_hash_into_related_hash value: '{{{cloudflare_logpush.network_analytics.destination.geo_hash}}}' if: ctx.cloudflare_logpush?.network_analytics?.destination?.geo_hash != null allow_duplicates: false - ignore_failure: true - append: field: related.hash + tag: append_colo_geo_hash_into_related_hash value: '{{{cloudflare_logpush.network_analytics.colo.geo_hash}}}' if: ctx.cloudflare_logpush?.network_analytics?.colo?.geo_hash != null allow_duplicates: false - ignore_failure: true + - append: + field: related.hosts + tag: append_colo_name_into_related_hosts + value: '{{{cloudflare_logpush.network_analytics.colo.name}}}' + if: ctx.cloudflare_logpush?.network_analytics?.colo?.name != null + allow_duplicates: false - append: field: related.ip - value: '{{{source.ip}}}' - if: ctx.source?.ip != null + tag: append_source_ip_into_related_ip + value: '{{{cloudflare_logpush.network_analytics.source.ip}}}' + if: ctx.cloudflare_logpush?.network_analytics?.source?.ip != null allow_duplicates: false - ignore_failure: true - append: field: related.ip - value: '{{{destination.ip}}}' - if: ctx.destination?.ip != null + tag: append_destination_ip_into_related_ip + value: '{{{cloudflare_logpush.network_analytics.destination.ip}}}' + if: ctx.cloudflare_logpush?.network_analytics?.destination?.ip != null allow_duplicates: false - ignore_failure: true - community_id: + tag: community_id_network_flow target_field: network.community_id ignore_failure: true - remove: @@ -1067,33 +1163,60 @@ processors: - cloudflare_logpush.network_analytics.source.asn - cloudflare_logpush.network_analytics.source.port - cloudflare_logpush.network_analytics.rule.id + - cloudflare_logpush.network_analytics.attack.id + - cloudflare_logpush.network_analytics.attack.campaign.id if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - ignore_failure: true ignore_missing: true + tag: remove_duplicated_custom_fields + on_failure: + - append: + tag: append_error_message_2b2c8ee6 + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: - description: Drops null/empty values recursively. - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); } - return false; - } - dropEmptyFields(ctx); + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + tag: append_tags_9fe66b2c + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message - value: |- - Processor "{{{ _ingest.on_failure_processor_type }}}" with tag {{{_ingest.on_failure_processor_tag}}} in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind + tag: set_pipeline_error_to_event_kind value: pipeline_error - append: field: tags From ccd6d97fd19ce76dd4e13720cfe1b8cf5ffd44db Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 16:38:41 +0530 Subject: [PATCH 13/14] add changelog entry for cloudflare_logpush pipeline improvements --- packages/cloudflare_logpush/changelog.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml index bb8fdeb7f13..6e92bda0c68 100644 --- a/packages/cloudflare_logpush/changelog.yml +++ b/packages/cloudflare_logpush/changelog.yml @@ -1,4 +1,10 @@ # newer versions go on top +- version: "1.44.0" + changes: + - description: | + Add support for new fields, improve error handling, and ECS field mappings across all data streams. + type: enhancement + link: https://github.com/elastic/integrations/pull/18257 - version: "1.43.3" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. From 5754cd37a991b81090c763069de4702a9e05fd48 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt Date: Thu, 9 Apr 2026 18:23:13 +0530 Subject: [PATCH 14/14] run pipeline and system tests --- ...-pipeline-access-request.log-expected.json | 6 +- .../access_request/sample_event.json | 54 ++-- .../test-pipeline-audit.log-expected.json | 15 +- .../data_stream/audit/sample_event.json | 54 ++-- .../test-pipeline-casb.log-expected.json | 6 +- .../data_stream/casb/sample_event.json | 49 ++-- ...-pipeline-device-posture.log-expected.json | 33 ++- .../device_posture/sample_event.json | 68 ++--- ...line-dlp-forensic-copies.log-expected.json | 2 +- .../dlp_forensic_copies/sample_event.json | 40 +-- .../test-pipeline-dns.log-expected.json | 15 +- .../data_stream/dns/sample_event.json | 49 ++-- ...st-pipeline-dns-firewall.log-expected.json | 14 +- .../dns_firewall/sample_event.json | 52 ++-- ...il-security-alerts-event.log-expected.json | 92 ++++++- .../email_security_alerts/sample_event.json | 80 ++++-- ...-pipeline-firewall-event.log-expected.json | 10 +- .../firewall_event/sample_event.json | 56 ++-- ...est-pipeline-gateway-dns.log-expected.json | 87 +++--- .../data_stream/gateway_dns/sample_event.json | 95 ++++--- ...st-pipeline-gateway-http.log-expected.json | 90 +++++-- .../gateway_http/sample_event.json | 90 ++++--- ...pipeline-gateway-network.log-expected.json | 36 +-- .../gateway_network/sample_event.json | 70 ++--- ...st-pipeline-http-request.log-expected.json | 147 ++++++++-- .../http_request/sample_event.json | 32 ++- .../test-pipeline-magic-ids.log-expected.json | 4 +- .../data_stream/magic_ids/sample_event.json | 52 ++-- ...test-pipeline-nel-report.log-expected.json | 6 +- .../data_stream/nel_report/sample_event.json | 45 ++-- ...peline-network-analytics.log-expected.json | 61 ++++- .../network_analytics/sample_event.json | 74 ++--- ...pipeline-network-session.log-expected.json | 30 ++- .../network_session/sample_event.json | 74 ++--- ...eline-page-shield-events.log-expected.json | 5 +- .../page_shield_events/sample_event.json | 46 ++-- ...t-pipeline-sinkhole-http.log-expected.json | 6 +- .../sinkhole_http/sample_event.json | 65 ++--- ...-pipeline-spectrum-event.log-expected.json | 43 ++- .../spectrum_event/sample_event.json | 61 ++--- ...t-pipeline-workers-trace.log-expected.json | 4 +- .../workers_trace/sample_event.json | 54 ++-- packages/cloudflare_logpush/docs/README.md | 252 +++++++++--------- 43 files changed, 1321 insertions(+), 903 deletions(-) diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log-expected.json b/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log-expected.json index 87d2b889b2f..1b96870f451 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/test/pipeline/test-pipeline-access-request.log-expected.json @@ -52,7 +52,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "login", @@ -142,7 +142,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "login", @@ -232,7 +232,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "login", diff --git a/packages/cloudflare_logpush/data_stream/access_request/sample_event.json b/packages/cloudflare_logpush/data_stream/access_request/sample_event.json index 95605b0e2b2..282a88b16f0 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/access_request/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2023-05-23T17:18:33.000Z", "agent": { - "ephemeral_id": "3c43110d-da0b-4e1b-adec-7031cdfb87a1", - "id": "57b2b3df-7f1f-49a9-8b35-90731f9b1b4e", - "name": "elastic-agent-50154", + "ephemeral_id": "801054ca-d5cc-4f9a-aa8a-4e4ee5b74814", + "id": "107ca55f-c937-46f3-9974-58527f82a166", + "name": "elastic-agent-58297", "type": "filebeat", "version": "8.17.1" }, @@ -22,15 +22,22 @@ }, "ip": "67.43.156.93" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "access_request": { + "action": "login", + "allowed": true, "app": { + "domain": "partner-zt-logs.cloudflareaccess.com/warp", "uuid": "123e4567-e89b-12d3-a456-426614174000" }, + "client": { + "ip": "67.43.156.93" + }, "connection": "onetimepin", + "country": "us", + "ray": { + "id": "00c0ffeeabc12345" + }, "request": { "prompt": "Please provide your reason for accessing the application.", "response": "I need to access the application for work purposes." @@ -41,19 +48,24 @@ "approver2@example.com" ], "duration": 7200 + }, + "timestamp": "2023-05-23T17:18:33.000Z", + "user": { + "email": "user@example.com", + "id": "166befbb-00e3-5e20-bd6e-27245333949f" } } }, "data_stream": { "dataset": "cloudflare_logpush.access_request", - "namespace": "33755", + "namespace": "85961", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "57b2b3df-7f1f-49a9-8b35-90731f9b1b4e", + "id": "107ca55f-c937-46f3-9974-58527f82a166", "snapshot": false, "version": "8.17.1" }, @@ -65,32 +77,16 @@ ], "dataset": "cloudflare_logpush.access_request", "id": "00c0ffeeabc12345", - "ingested": "2025-12-12T07:31:27Z", + "ingested": "2026-04-09T11:24:10Z", "kind": "event", + "original": "{\"Action\":\"login\",\"Allowed\":true,\"AppDomain\":\"partner-zt-logs.cloudflareaccess.com/warp\",\"AppUUID\":\"123e4567-e89b-12d3-a456-426614174000\",\"Connection\":\"onetimepin\",\"Country\":\"us\",\"CreatedAt\":1684862313000000000,\"Email\":\"user@example.com\",\"IPAddress\":\"67.43.156.93\",\"PurposeJustificationPrompt\":\"Please provide your reason for accessing the application.\",\"PurposeJustificationResponse\":\"I need to access the application for work purposes.\",\"RayID\":\"00c0ffeeabc12345\",\"TemporaryAccessApprovers\":[\"approver1@example.com\",\"approver2@example.com\"],\"TemporaryAccessDuration\":7200,\"UserUID\":\"166befbb-00e3-5e20-bd6e-27245333949f\"}", "type": [ "access", "allowed" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "access_request.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/access_request.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "ip": [ @@ -104,6 +100,8 @@ ] }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-access_request" ], diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/test/pipeline/test-pipeline-audit.log-expected.json b/packages/cloudflare_logpush/data_stream/audit/_dev/test/pipeline/test-pipeline-audit.log-expected.json index 9be556586be..c147668ada0 100644 --- a/packages/cloudflare_logpush/data_stream/audit/_dev/test/pipeline/test-pipeline-audit.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/test/pipeline/test-pipeline-audit.log-expected.json @@ -39,7 +39,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "token_create", @@ -60,7 +60,8 @@ "81.2.69.142" ], "user": [ - "enl3j9du8rnx2swwd9l32qots7l54t9s" + "enl3j9du8rnx2swwd9l32qots7l54t9s", + "user@example.com" ] }, "source": { @@ -114,7 +115,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "token_create", @@ -135,7 +136,8 @@ "81.2.69.142" ], "user": [ - "enl3j9du8rnx2swwd9l32qots7l54t9s" + "enl3j9du8rnx2swwd9l32qots7l54t9s", + "user@example.com" ] }, "source": { @@ -189,7 +191,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "token_create", @@ -210,7 +212,8 @@ "81.2.69.142" ], "user": [ - "enl3j9du8rnx2swwd9l32qots7l54t9s" + "enl3j9du8rnx2swwd9l32qots7l54t9s", + "user@example.com" ] }, "source": { diff --git a/packages/cloudflare_logpush/data_stream/audit/sample_event.json b/packages/cloudflare_logpush/data_stream/audit/sample_event.json index 84a04e1f297..4ea393a0c81 100644 --- a/packages/cloudflare_logpush/data_stream/audit/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/audit/sample_event.json @@ -1,20 +1,26 @@ { "@timestamp": "2021-11-30T20:19:48.000Z", "agent": { - "ephemeral_id": "f6aa0050-d066-4e9f-ad0e-44e29655cb0a", - "id": "c88b6a8a-5b51-4f19-9386-d141ad8d5fd7", - "name": "elastic-agent-42667", + "ephemeral_id": "a0211120-06cd-4e63-8802-1e13ef915aad", + "id": "9c1a4701-f21a-4bf1-839e-0176c1bae24b", + "name": "elastic-agent-38547", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "audit": { + "action": { + "result": "success", + "type": "token_create" + }, "actor": { + "email": "user@example.com", + "id": "enl3j9du8rnx2swwd9l32qots7l54t9s", + "ip": "81.2.69.142", "type": "user" }, + "id": "73fd39ed-5aab-4a2a-b93c-c9a4abf0c425", + "interface": "UI", "metadata": { "token_name": "test", "token_tag": "b7261c49a793a82678d12285f0bc1401" @@ -33,19 +39,20 @@ "resource": { "id": "enl3j9du8rnx2swwd9l32qots7l54t9s", "type": "account" - } + }, + "timestamp": "2021-11-30T20:19:48.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.audit", - "namespace": "27343", + "namespace": "29929", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "c88b6a8a-5b51-4f19-9386-d141ad8d5fd7", + "id": "9c1a4701-f21a-4bf1-839e-0176c1bae24b", "snapshot": false, "version": "8.17.1" }, @@ -57,46 +64,33 @@ ], "dataset": "cloudflare_logpush.audit", "id": "73fd39ed-5aab-4a2a-b93c-c9a4abf0c425", - "ingested": "2025-12-12T07:37:27Z", + "ingested": "2026-04-09T11:25:10Z", "kind": "event", + "original": "{\"ActionResult\":true,\"ActionType\":\"token_create\",\"ActorEmail\":\"user@example.com\",\"ActorID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ActorIP\":\"81.2.69.142\",\"ActorType\":\"user\",\"ID\":\"73fd39ed-5aab-4a2a-b93c-c9a4abf0c425\",\"Interface\":\"UI\",\"Metadata\":{\"token_name\":\"test\",\"token_tag\":\"b7261c49a793a82678d12285f0bc1401\"},\"NewValue\":{\"key1\":\"value1\",\"key2\":\"value2\"},\"OldValue\":{\"key3\":\"value4\",\"key4\":\"value4\"},\"OwnerID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ResourceID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ResourceType\":\"account\",\"When\":\"2021-11-30T20:19:48Z\"}", "outcome": "success", "provider": "UI", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "audit.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/audit.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "ip": [ "81.2.69.142" ], "user": [ - "enl3j9du8rnx2swwd9l32qots7l54t9s" + "enl3j9du8rnx2swwd9l32qots7l54t9s", + "user@example.com" ] }, "source": { "ip": "81.2.69.142" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-audit" ], diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log-expected.json b/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log-expected.json index 71f7f5e30cc..f082e5e1691 100644 --- a/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/test/pipeline/test-pipeline-casb.log-expected.json @@ -75,7 +75,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -175,7 +175,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -275,7 +275,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ diff --git a/packages/cloudflare_logpush/data_stream/casb/sample_event.json b/packages/cloudflare_logpush/data_stream/casb/sample_event.json index bbd9d1d0ffb..4ad0d729f80 100644 --- a/packages/cloudflare_logpush/data_stream/casb/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/casb/sample_event.json @@ -1,15 +1,12 @@ { "@timestamp": "2023-05-16T10:00:00.000Z", "agent": { - "ephemeral_id": "941f6da3-676d-466a-817c-3d2b60cb0da8", - "id": "082d25a2-4d08-4619-9273-1bbf8ecda1ac", - "name": "elastic-agent-85424", + "ephemeral_id": "8ec02fce-f2f2-4839-97c3-1c76a7b02c7b", + "id": "12524cd6-a1de-484d-a9a6-e70ddd09c755", + "name": "elastic-agent-24633", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "casb": { "asset": { @@ -63,31 +60,35 @@ "url": "/services/data/userID" } }, - "name": "John Doe" + "name": "John Doe", + "url": "https://example.com/resource" }, "finding": { + "id": "6b187be4-2dd5-42c5-a37b-111111111111", "type": { "id": "a2790c4f-03f5-449f-b209-5f4447f417aa", - "name": "Salesforce User Sending Email with Different Email Address" + "name": "Salesforce User Sending Email with Different Email Address", + "severity": "Medium" } }, "integration": { "id": "c772678d-5cf1-4c73-bf3f-111111111111", "name": "Salesforce Testing", "policy_vendor": "Salesforce Connection" - } + }, + "timestamp": "2023-05-16T10:00:00.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.casb", - "namespace": "79671", + "namespace": "16545", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "082d25a2-4d08-4619-9273-1bbf8ecda1ac", + "id": "12524cd6-a1de-484d-a9a6-e70ddd09c755", "snapshot": false, "version": "8.17.1" }, @@ -98,34 +99,20 @@ ], "dataset": "cloudflare_logpush.casb", "id": "6b187be4-2dd5-42c5-a37b-111111111111", - "ingested": "2025-12-12T11:49:37Z", + "ingested": "2026-04-09T11:26:09Z", "kind": "event", + "original": "{\"AssetDisplayName\":\"John Doe\",\"AssetExternalID\":\"0051N000004mG2LAAA\",\"AssetLink\":\"https://example.com/resource\",\"AssetMetadata\":{\"AccountId\":null,\"Address\":{\"city\":\"Singapore\",\"country\":\"Singapore\",\"countryCode\":\"SG\",\"geocodeAccuracy\":null,\"latitude\":null,\"longitude\":null,\"postalCode\":null,\"state\":null,\"stateCode\":null,\"street\":null},\"Alias\":\"JDoe\",\"BadgeText\":\"\",\"BannerPhotoUrl\":\"/profilephoto/001\",\"CallCenterId\":null,\"CommunityNickname\":\"Doe.John\",\"CompanyName\":\"MyCompany\",\"ContactId\":null,\"DefaultGroupNotificationFrequency\":\"N\",\"Department\":\"521\",\"DigestFrequency\":\"D\",\"Division\":null,\"Email\":\"user@example.com\",\"EmailEncodingKey\":\"UTF-8\",\"EmailPreferencesAutoBcc\":true,\"EmployeeNumber\":\"18124\",\"Extension\":null,\"Fax\":null,\"FederationIdentifier\":null,\"FirstName\":\"John\",\"ForecastEnabled\":false,\"FullPhotoUrl\":\"https://photos.com/profilephoto/001\",\"Id\":\"0051N000004mG2LAAA\",\"IsActive\":false,\"IsProfilePhotoActive\":false,\"LanguageLocaleKey\":\"en_US\",\"LastLoginDate\":\"2021-10-06T06:32:09.000+0000\",\"LastName\":\"Doe\",\"LastReferencedDate\":null,\"LastViewedDate\":null,\"LocaleSidKey\":\"en_SG\",\"MediumBannerPhotoUrl\":\"/profilephoto/001/E\",\"MobilePhone\":null,\"Name\":\"John Doe\",\"OfflineTrialExpirationDate\":null,\"Phone\":\"+3460000000\",\"ReceivesAdminInfoEmails\":true,\"ReceivesInfoEmails\":true,\"SenderEmail\":\"sender@example.com\",\"SenderName\":null,\"Signature\":null,\"SmallBannerPhotoUrl\":\"/profilephoto/001/D\",\"SmallPhotoUrl\":\"https://photos.com/photo/001\",\"TimeZoneSidKey\":\"Asia/Singapore\",\"Title\":\"Customer Solutions Engineer\",\"UserPermissionsCallCenterAutoLogin\":false,\"UserPermissionsInteractionUser\":true,\"UserPermissionsMarketingUser\":false,\"UserPermissionsOfflineUser\":false,\"UserPermissionsSupportUser\":false,\"UserRoleId\":\"00E2G000001E\",\"UserType\":\"Standard\",\"attributes\":{\"type\":\"User\",\"url\":\"/services/data/userID\"}},\"DetectedTimestamp\":\"2023-05-16T10:00:00Z\",\"FindingTypeDisplayName\":\"Salesforce User Sending Email with Different Email Address\",\"FindingTypeID\":\"a2790c4f-03f5-449f-b209-5f4447f417aa\",\"FindingTypeSeverity\":\"Medium\",\"InstanceID\":\"6b187be4-2dd5-42c5-a37b-111111111111\",\"IntegrationDisplayName\":\"Salesforce Testing\",\"IntegrationID\":\"c772678d-5cf1-4c73-bf3f-111111111111\",\"IntegrationPolicyVendor\":\"Salesforce Connection\"}", "severity": 2, "type": [ "access" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "casb.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/casb.log" - }, - "offset": 0 + "type": "http_endpoint" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-casb" ], diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log-expected.json b/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log-expected.json index 495c2fe7046..d7c79cddb3e 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/test/pipeline/test-pipeline-device-posture.log-expected.json @@ -43,8 +43,17 @@ "version": "2023.3.258" } }, + "device": { + "id": "083a8354-d56c-11ed-9771-111111111", + "manufacturer": "Google Compute Engine", + "model": { + "name": "Google Compute Engine" + }, + "serial_number": "GoogleCloud-ABCD1234567890", + "type": "linux" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -135,8 +144,17 @@ "version": "2023.3.258" } }, + "device": { + "id": "083a8354-d56c-11ed-9771-111111111", + "manufacturer": "Google Compute Engine", + "model": { + "name": "Google Compute Engine" + }, + "serial_number": "GoogleCloud-ABCD1234567890", + "type": "linux" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -227,8 +245,17 @@ "version": "2023.3.258" } }, + "device": { + "id": "083a8354-d56c-11ed-9771-111111111", + "manufacturer": "Google Compute Engine", + "model": { + "name": "Google Compute Engine" + }, + "serial_number": "GoogleCloud-ABCD1234567890", + "type": "linux" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ diff --git a/packages/cloudflare_logpush/data_stream/device_posture/sample_event.json b/packages/cloudflare_logpush/data_stream/device_posture/sample_event.json index aa588710d82..8914d82b35a 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/device_posture/sample_event.json @@ -1,26 +1,12 @@ { "@timestamp": "2023-05-17T12:00:00.000Z", "agent": { - "ephemeral_id": "edffd861-c70c-4011-9b8e-4bb37d718964", - "id": "c1433c6f-9bf4-4f42-b22e-10ef701c8969", - "name": "elastic-agent-95387", + "ephemeral_id": "e08f9602-22ca-4bbd-88b6-9a0b0811783b", + "id": "beb39b36-63df-4209-9ab6-fcdfe0ba4b5d", + "name": "elastic-agent-12848", "type": "filebeat", "version": "8.17.1" }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-device-posture-bucket-31257", - "name": "elastic-package-device-posture-bucket-31257" - }, - "object": { - "key": "test-device-posture.log" - } - } - }, - "cloud": { - "region": "us-east-1" - }, "cloudflare_logpush": { "device_posture": { "eval": { @@ -39,22 +25,48 @@ "result": true }, "host": { + "id": "083a8354-d56c-11ed-9771-111111111", "manufacturer": "Google Compute Engine", "model": "Google Compute Engine", + "name": "zt-test-vm1", + "os": { + "family": "linux", + "version": "5.15.0" + }, "serial": "GoogleCloud-ABCD1234567890" - } + }, + "rule": { + "category": "os_version", + "id": "policy-abcdefgh", + "name": "Ubuntu" + }, + "timestamp": "2023-05-17T12:00:00.000Z", + "user": { + "email": "user@example.com", + "id": "user-abcdefgh" + }, + "version": "2023.3.258" } }, "data_stream": { "dataset": "cloudflare_logpush.device_posture", - "namespace": "33464", + "namespace": "48116", "type": "logs" }, + "device": { + "id": "083a8354-d56c-11ed-9771-111111111", + "manufacturer": "Google Compute Engine", + "model": { + "name": "Google Compute Engine" + }, + "serial_number": "GoogleCloud-ABCD1234567890", + "type": "linux" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "c1433c6f-9bf4-4f42-b22e-10ef701c8969", + "id": "beb39b36-63df-4209-9ab6-fcdfe0ba4b5d", "snapshot": false, "version": "8.17.1" }, @@ -64,9 +76,9 @@ "host" ], "dataset": "cloudflare_logpush.device_posture", - "ingested": "2025-12-12T08:15:20Z", + "ingested": "2026-04-09T11:27:07Z", "kind": "event", - "original": "{\"ClientVersion\":\"2023.3.258\",\"DeviceID\":\"083a8354-d56c-11ed-9771-111111111\",\"DeviceManufacturer\":\"Google Compute Engine\",\"DeviceModel\":\"Google Compute Engine\",\"DeviceName\":\"zt-test-vm1\",\"DeviceSerialNumber\":\"GoogleCloud-ABCD1234567890\",\"DeviceType\":\"linux\",\"Email\":\"user@example.com\",\"OSVersion\":\"5.15.0\",\"PolicyID\":\"policy-abcdefgh\",\"PostureCheckName\":\"Ubuntu\",\"PostureCheckType\":\"os_version\",\"PostureEvaluatedResult\":true,\"PostureExpectedJSON\":{\"version\":\"5.15.0-1025-gcp\",\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\"},\"PostureReceivedJSON\":{\"version\":\"5.15.0-1025-gcp\",\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\"},\"Timestamp\":\"2023-05-17T12:00:00Z\",\"UserUID\":\"user-abcdefgh\"}", + "original": "{\"ClientVersion\":\"2023.3.258\",\"DeviceID\":\"083a8354-d56c-11ed-9771-111111111\",\"DeviceManufacturer\":\"Google Compute Engine\",\"DeviceModel\":\"Google Compute Engine\",\"DeviceName\":\"zt-test-vm1\",\"DeviceSerialNumber\":\"GoogleCloud-ABCD1234567890\",\"DeviceType\":\"linux\",\"Email\":\"user@example.com\",\"OSVersion\":\"5.15.0\",\"PolicyID\":\"policy-abcdefgh\",\"PostureCheckName\":\"Ubuntu\",\"PostureCheckType\":\"os_version\",\"PostureEvaluatedResult\":true,\"PostureExpectedJSON\":{\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\",\"version\":\"5.15.0-1025-gcp\"},\"PostureReceivedJSON\":{\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\",\"version\":\"5.15.0-1025-gcp\"},\"Timestamp\":\"2023-05-17T12:00:00Z\",\"UserUID\":\"user-abcdefgh\"}", "outcome": "success", "type": [ "info" @@ -81,13 +93,7 @@ } }, "input": { - "type": "aws-s3" - }, - "log": { - "file": { - "path": "https://elastic-package-device-posture-bucket-31257.s3.us-east-1.amazonaws.com/test-device-posture.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hosts": [ @@ -105,8 +111,8 @@ "name": "Ubuntu" }, "tags": [ - "collect_sqs_logs", "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-device_posture" ], diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/test/pipeline/test-pipeline-dlp-forensic-copies.log-expected.json b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/test/pipeline/test-pipeline-dlp-forensic-copies.log-expected.json index 24194953830..d133115769d 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/test/pipeline/test-pipeline-dlp-forensic-copies.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/test/pipeline/test-pipeline-dlp-forensic-copies.log-expected.json @@ -18,7 +18,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json index 825707a39cf..956e173d437 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json @@ -1,26 +1,12 @@ { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "ac5a4097-8089-4aaa-86b6-7a2807d34abf", - "id": "f5cd4a36-a09b-49fe-8b0c-0148d4e98428", - "name": "elastic-agent-81829", + "ephemeral_id": "208956c5-24de-474a-b20f-ff5ad6cbac48", + "id": "fafd216b-168c-4f9e-9e00-8481e2e330b9", + "name": "elastic-agent-41362", "type": "filebeat", "version": "8.17.1" }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-dlp-forensic-copies-bucket-19014", - "name": "elastic-package-dlp-forensic-copies-bucket-19014" - }, - "object": { - "key": "test-dlp-forensic-copies.log" - } - } - }, - "cloud": { - "region": "us-east-1" - }, "cloudflare_logpush": { "dlp_forensic_copies": { "account_id": "acc-id", @@ -38,14 +24,14 @@ }, "data_stream": { "dataset": "cloudflare_logpush.dlp_forensic_copies", - "namespace": "64194", + "namespace": "30202", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "f5cd4a36-a09b-49fe-8b0c-0148d4e98428", + "id": "fafd216b-168c-4f9e-9e00-8481e2e330b9", "snapshot": false, "version": "8.17.1" }, @@ -55,28 +41,22 @@ "network" ], "dataset": "cloudflare_logpush.dlp_forensic_copies", - "ingested": "2025-12-12T08:39:29Z", + "ingested": "2026-04-09T12:07:18Z", "kind": "event", - "original": "{\"AccountID\":\"acc-id\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"}}", + "original": "{\"AccountID\":\"acc-id\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\"}", "type": [ "info" ] }, "input": { - "type": "aws-s3" - }, - "log": { - "file": { - "path": "https://elastic-package-dlp-forensic-copies-bucket-19014.s3.us-east-1.amazonaws.com/test-dlp-forensic-copies.log" - }, - "offset": 0 + "type": "http_endpoint" }, "rule": { "id": "9" }, "tags": [ - "collect_sqs_logs", "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-dlp_forensic_copies" ] diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log-expected.json b/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log-expected.json index 0ba343d760c..bc005a2a713 100644 --- a/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/test/pipeline/test-pipeline-dns.log-expected.json @@ -28,10 +28,11 @@ "dns": { "question": { "name": "example.com" - } + }, + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -85,10 +86,11 @@ "dns": { "question": { "name": "example.com" - } + }, + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -142,10 +144,11 @@ "dns": { "question": { "name": "example.com" - } + }, + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ diff --git a/packages/cloudflare_logpush/data_stream/dns/sample_event.json b/packages/cloudflare_logpush/data_stream/dns/sample_event.json index c90114c5173..cef24d60db2 100644 --- a/packages/cloudflare_logpush/data_stream/dns/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/dns/sample_event.json @@ -1,15 +1,12 @@ { "@timestamp": "2022-05-26T09:23:54.000Z", "agent": { - "ephemeral_id": "28ac7554-1d13-49b2-a05e-c28fff86aed2", - "id": "90e30034-7fa9-4654-9018-4c3f3605c19e", - "name": "elastic-agent-61553", + "ephemeral_id": "6e8cd3c5-88ea-41d7-8778-fff4144c7e24", + "id": "3d115c8b-f50d-487c-ae2d-0be7ab8d2307", + "name": "elastic-agent-66764", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "dns": { "colo": { @@ -20,29 +17,35 @@ "subnet_length": 0 }, "query": { + "name": "example.com", "type": 65535 }, "response": { "cached": false, "code": 0 - } + }, + "source": { + "ip": "175.16.199.0" + }, + "timestamp": "2022-05-26T09:23:54.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.dns", - "namespace": "24004", + "namespace": "42607", "type": "logs" }, "dns": { "question": { "name": "example.com" - } + }, + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "90e30034-7fa9-4654-9018-4c3f3605c19e", + "id": "3d115c8b-f50d-487c-ae2d-0be7ab8d2307", "snapshot": false, "version": "8.17.1" }, @@ -52,31 +55,15 @@ "network" ], "dataset": "cloudflare_logpush.dns", - "ingested": "2025-12-12T08:46:14Z", + "ingested": "2026-04-09T12:08:16Z", "kind": "event", + "original": "{\"ColoCode\":\"MRS\",\"EDNSSubnet\":\"1.128.0.0\",\"EDNSSubnetLength\":0,\"QueryName\":\"example.com\",\"QueryType\":65535,\"ResponseCached\":false,\"ResponseCode\":0,\"SourceIP\":\"175.16.199.0\",\"Timestamp\":\"2022-05-26T09:23:54Z\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "dns.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/dns.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "ip": [ @@ -88,6 +75,8 @@ "ip": "175.16.199.0" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-dns" ] diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/test/pipeline/test-pipeline-dns-firewall.log-expected.json b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/test/pipeline/test-pipeline-dns-firewall.log-expected.json index 9c6b1c84df7..8fb731e7730 100644 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/test/pipeline/test-pipeline-dns-firewall.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/test/pipeline/test-pipeline-dns-firewall.log-expected.json @@ -38,12 +38,13 @@ }, "dns": { "question": { - "name": "example.com" + "name": "example.com", + "type": "A" }, - "response_code": "0" + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -123,12 +124,13 @@ }, "dns": { "question": { - "name": "nonexistentdomain.xyz" + "name": "nonexistentdomain.xyz", + "type": "AAAA" }, - "response_code": "3" + "response_code": "NXDomain" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/sample_event.json b/packages/cloudflare_logpush/data_stream/dns_firewall/sample_event.json index 2947096c4d1..2a710575aaa 100644 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/sample_event.json @@ -1,15 +1,12 @@ { "@timestamp": "2023-09-19T12:30:00.000Z", "agent": { - "ephemeral_id": "2f6c2025-b827-4b56-b9d7-5515c0e8ec22", - "id": "9cf56aad-c149-4ec4-b610-b508a6f3812d", - "name": "elastic-agent-80366", + "ephemeral_id": "1701442d-5ccf-456d-9a7c-65110dae2a77", + "id": "46fd2c9a-f97e-4c91-9ee6-2ee13dfd5c62", + "name": "elastic-agent-60076", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "dns_firewall": { "cluster_id": "CLUSTER-001", @@ -22,6 +19,7 @@ }, "question": { "dnssec_ok": true, + "name": "example.com", "recursion_desired": true, "size": 60, "tcp": false, @@ -29,8 +27,13 @@ }, "response": { "cached": true, - "cached_stale": false + "cached_stale": false, + "code": "0" + }, + "source": { + "ip": "67.43.156.2" }, + "timestamp": "2023-09-19T12:30:00.000Z", "upstream": { "ip": "81.2.69.144", "response_code": "0", @@ -40,20 +43,21 @@ }, "data_stream": { "dataset": "cloudflare_logpush.dns_firewall", - "namespace": "97058", + "namespace": "48306", "type": "logs" }, "dns": { "question": { - "name": "example.com" + "name": "example.com", + "type": "A" }, - "response_code": "0" + "response_code": "NoError" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "9cf56aad-c149-4ec4-b610-b508a6f3812d", + "id": "46fd2c9a-f97e-4c91-9ee6-2ee13dfd5c62", "snapshot": false, "version": "8.17.1" }, @@ -63,31 +67,15 @@ "network" ], "dataset": "cloudflare_logpush.dns_firewall", - "ingested": "2025-12-12T08:52:06Z", + "ingested": "2026-04-09T12:09:16Z", "kind": "event", + "original": "{\"ClientResponseCode\":0,\"ClusterID\":\"CLUSTER-001\",\"ColoCode\":\"SFO\",\"EDNSSubnet\":\"67.43.156.0\",\"EDNSSubnetLength\":24,\"QueryDO\":true,\"QueryName\":\"example.com\",\"QueryRD\":true,\"QuerySize\":60,\"QueryTCP\":false,\"QueryType\":1,\"ResponseCached\":true,\"ResponseCachedStale\":false,\"SourceIP\":\"67.43.156.2\",\"Timestamp\":\"2023-09-19T12:30:00Z\",\"UpstreamIP\":\"81.2.69.144\",\"UpstreamResponseCode\":0,\"UpstreamResponseTimeMs\":30}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "dns_firewall.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/dns_firewall.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "udp" @@ -115,6 +103,8 @@ "ip": "67.43.156.2" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-dns_firewall" ] diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/test/pipeline/test-pipeline-email-security-alerts-event.log-expected.json b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/test/pipeline/test-pipeline-email-security-alerts-event.log-expected.json index 9a1d3122fa5..ee19fc38fe9 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/test/pipeline/test-pipeline-email-security-alerts-event.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/test/pipeline/test-pipeline-email-security-alerts-event.log-expected.json @@ -64,7 +64,48 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" + }, + "email": { + "attachments": [ + { + "file": { + "hash": { + "md5": "91f073bd208689ddbd248e8989ecae90", + "sha1": "62b77e14e2c43049c45b5725018e78d0f9986930", + "sha256": "3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06", + "ssdeep": "McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3" + }, + "mime_type": "application/x-msi", + "name": "attachment.gif" + } + } + ], + "cc": { + "address": [ + "firstlast+cc@cloudflare.com" + ] + }, + "from": { + "address": [ + "firstlast+from@cloudflare.com" + ] + }, + "message_id": "", + "reply_to": { + "address": [ + "firstlast+reply@cloudflare.com" + ] + }, + "sender": { + "address": "firstlast+env_from@cloudflare.com" + }, + "subject": "innocuous message: please read", + "to": { + "address": [ + "firstlast+to@cloudflare.com" + ] + } }, "event": { "category": [ @@ -108,6 +149,9 @@ "firstlast+from@cloudflare.com", "First Last (from)", "firstlast+env_from@cloudflare.com", + "firstlast+origin@cloudflare.com", + "firstlast+reply@cloudflare.com", + "First Last (reply)", "firstlast+to@cloudflare.com", "firstlast+env_to@cloudflare.com", "First Last (to)", @@ -201,7 +245,48 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" + }, + "email": { + "attachments": [ + { + "file": { + "hash": { + "md5": "91f073bd208689ddbd248e8989ecae90", + "sha1": "62b77e14e2c43049c45b5725018e78d0f9986930", + "sha256": "3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06", + "ssdeep": "McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3" + }, + "mime_type": "application/x-msi", + "name": "attachment.gif" + } + } + ], + "cc": { + "address": [ + "firstlast+cc@cloudflare.com" + ] + }, + "from": { + "address": [ + "firstlast+from@cloudflare.com" + ] + }, + "message_id": "", + "reply_to": { + "address": [ + "firstlast+reply@cloudflare.com" + ] + }, + "sender": { + "address": "firstlast+env_from@cloudflare.com" + }, + "subject": "innocuous message: please read", + "to": { + "address": [ + "firstlast+to@cloudflare.com" + ] + } }, "event": { "category": [ @@ -245,6 +330,9 @@ "firstlast+from@cloudflare.com", "First Last (from)", "firstlast+env_from@cloudflare.com", + "firstlast+origin@cloudflare.com", + "firstlast+reply@cloudflare.com", + "First Last (reply)", "firstlast+to@cloudflare.com", "firstlast+env_to@cloudflare.com", "First Last (to)", diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/sample_event.json b/packages/cloudflare_logpush/data_stream/email_security_alerts/sample_event.json index dc8c31c5bd3..f8d51196b05 100644 --- a/packages/cloudflare_logpush/data_stream/email_security_alerts/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/sample_event.json @@ -1,15 +1,12 @@ { "@timestamp": "2024-08-28T15:32:35.000Z", "agent": { - "ephemeral_id": "e482a27d-3e24-4212-8a06-620ac597d5af", - "id": "7bc49be7-d33e-43a3-bfec-1d4c34fbf105", - "name": "elastic-agent-95627", + "ephemeral_id": "8329703a-0e11-43c7-8a5a-28d2bbd62dfc", + "id": "6960421c-0684-4f16-ad7c-ed15de6943cd", + "name": "elastic-agent-55230", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "email_security_alerts": { "alert_id": "4WtWkr6nlBz9sNH-2024-08-28T15:32:35", @@ -52,6 +49,7 @@ "smtp_envelope_to": [ "firstlast+env_to@cloudflare.com" ], + "smtp_helo_server_ip": "81.2.69.144", "smtp_helo_server_ip_as_name": "asn", "smtp_helo_server_ip_as_number": "42", "smtp_helo_server_ip_geo": "US/NV/Las Vegas", @@ -61,23 +59,60 @@ "CredentialHarvester", "Dropper" ], + "timestamp": "2024-08-28T15:32:35.000Z", "to": "firstlast+to@cloudflare.com", "to_name": "First Last (to)" } }, "data_stream": { "dataset": "cloudflare_logpush.email_security_alerts", - "namespace": "91726", + "namespace": "46333", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "7bc49be7-d33e-43a3-bfec-1d4c34fbf105", + "id": "6960421c-0684-4f16-ad7c-ed15de6943cd", "snapshot": false, "version": "8.17.1" }, + "email": { + "attachments": [ + { + "file": { + "hash": { + "md5": "91f073bd208689ddbd248e8989ecae90", + "sha1": "62b77e14e2c43049c45b5725018e78d0f9986930", + "sha256": "3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06", + "ssdeep": "McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3" + }, + "mime_type": "application/x-msi", + "name": "attachment.gif" + } + } + ], + "cc": { + "address": [ + "firstlast+cc@cloudflare.com" + ] + }, + "from": { + "address": [ + "firstlast+from@cloudflare.com" + ] + }, + "message_id": "", + "reply_to": { + "address": [ + "firstlast+reply@cloudflare.com" + ] + }, + "sender": { + "address": "firstlast+env_from@cloudflare.com" + }, + "subject": "innocuous message: please read" + }, "event": { "agent_id_status": "verified", "category": [ @@ -85,8 +120,9 @@ "network" ], "dataset": "cloudflare_logpush.email_security_alerts", - "ingested": "2025-12-12T08:58:27Z", + "ingested": "2026-04-09T12:10:18Z", "kind": "alert", + "original": "{\"AlertID\":\"4WtWkr6nlBz9sNH-2024-08-28T15:32:35\",\"AlertReasons\":[\"because\",\"said-so\"],\"Attachments\":[{\"ContentTypeComputed\":\"application/x-msi\",\"ContentTypeProvided\":\"image/gif\",\"Decrypted\":true,\"Encrypted\":true,\"Md5\":\"91f073bd208689ddbd248e8989ecae90\",\"Name\":\"attachment.gif\",\"Sha1\":\"62b77e14e2c43049c45b5725018e78d0f9986930\",\"Sha256\":\"3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06\",\"Ssdeep\":\"McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3\"}],\"CC\":[\"firstlast+cc@cloudflare.com\"],\"CCName\":[\"First Last (cc)\"],\"FinalDisposition\":\"malicious\",\"From\":\"firstlast+from@cloudflare.com\",\"FromName\":\"First Last (from)\",\"Links\":[\"https://example.com\"],\"MessageDeliveryMode\":\"unset\",\"MessageID\":\"\\u003cMessage-ID\\u003e\",\"Origin\":\"unset\",\"OriginalSender\":\"firstlast+origin@cloudflare.com\",\"ReplyTo\":\"firstlast+reply@cloudflare.com\",\"ReplyToName\":\"First Last (reply)\",\"SMTPEnvelopeFrom\":\"firstlast+env_from@cloudflare.com\",\"SMTPEnvelopeTo\":[\"firstlast+env_to@cloudflare.com\"],\"SMTPHeloServerIP\":\"81.2.69.144\",\"SMTPHeloServerIPAsName\":\"asn\",\"SMTPHeloServerIPAsNumber\":\"42\",\"SMTPHeloServerIPGeo\":\"US/NV/Las Vegas\",\"SMTPHeloServerName\":\"servername\",\"Subject\":\"innocuous message: please read\",\"ThreatCategories\":[\"CredentialHarvester\",\"Dropper\"],\"Timestamp\":\"2024-08-28T15:32:35Z\",\"To\":\"firstlast+to@cloudflare.com\",\"ToName\":\"First Last (to)\"}", "type": [ "info" ] @@ -103,25 +139,8 @@ "name": "attachment.gif" } ], - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "email_security_alerts.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/email_security_alerts.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hash": [ @@ -142,6 +161,9 @@ "firstlast+from@cloudflare.com", "First Last (from)", "firstlast+env_from@cloudflare.com", + "firstlast+origin@cloudflare.com", + "firstlast+reply@cloudflare.com", + "First Last (reply)", "firstlast+env_to@cloudflare.com", "firstlast+cc@cloudflare.com", "First Last (cc)" @@ -165,6 +187,8 @@ "ip": "81.2.69.144" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-email_security_alerts" ] diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json index 1d3463d2072..384fb0b01f1 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json @@ -64,7 +64,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", @@ -199,7 +199,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", @@ -334,7 +334,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", @@ -446,7 +446,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -574,7 +574,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/sample_event.json b/packages/cloudflare_logpush/data_stream/firewall_event/sample_event.json index f2d60752f19..b4533d1eb07 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/firewall_event/sample_event.json @@ -1,32 +1,22 @@ { "@timestamp": "2022-05-31T05:23:43.000Z", "agent": { - "ephemeral_id": "37b1d591-989b-4113-b104-1f4212137e5a", - "id": "c39dd230-1d6b-4fa2-a12b-ae61fb5e7f5f", - "name": "elastic-agent-20484", + "ephemeral_id": "e85ec68a-f705-433b-8974-b945f4f35fbd", + "id": "837b9eb7-7faa-42d4-bd20-6e9e76fb1876", + "name": "elastic-agent-61647", "type": "filebeat", "version": "8.17.1" }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-firewall-event-bucket-13780", - "name": "elastic-package-firewall-event-bucket-13780" - }, - "object": { - "key": "test-firewall-event.log" - } - } - }, - "cloud": { - "region": "us-east-1" - }, "cloudflare_logpush": { "firewall_event": { + "action": "block", "client": { "asn": { - "description": "CLOUDFLARENET" + "description": "CLOUDFLARENET", + "value": 15169 }, + "country": "us", + "ip": "175.16.199.0", "ip_class": "searchEngine", "referer": { "host": "abc.example.com", @@ -36,6 +26,7 @@ }, "request": { "host": "xyz.example.com", + "method": "GET", "path": "/abc/checkout", "protocol": "HTTP/1.1", "query": "?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))", @@ -48,6 +39,9 @@ "edge": { "colo": { "code": "IAD" + }, + "response": { + "status": 403 } }, "kind": "firewall", @@ -67,19 +61,23 @@ "ray": { "id": "713d477539b55c29" }, - "source": "firewallrules" + "rule": { + "id": "7dc666e026974dab84884c73b3e2afe1" + }, + "source": "firewallrules", + "timestamp": "2022-05-31T05:23:43.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.firewall_event", - "namespace": "63366", + "namespace": "50135", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "c39dd230-1d6b-4fa2-a12b-ae61fb5e7f5f", + "id": "837b9eb7-7faa-42d4-bd20-6e9e76fb1876", "snapshot": false, "version": "8.17.1" }, @@ -91,9 +89,9 @@ ], "dataset": "cloudflare_logpush.firewall_event", "id": "713d477539b55c29", - "ingested": "2025-12-12T09:03:48Z", + "ingested": "2026-04-09T12:11:17Z", "kind": "event", - "original": "{\"ClientRequestScheme\":\"https\",\"MatchIndex\":1,\"ClientRefererHost\":\"abc.example.com\",\"Source\":\"firewallrules\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\",\"ClientRefererPath\":\"/abc/checkout\",\"Metadata\":{\"filter\":\"1ced07e066a34abf8b14f2a99593bc8d\",\"type\":\"customer\"},\"EdgeResponseStatus\":403,\"ClientRequestProtocol\":\"HTTP/1.1\",\"OriginatorRayID\":\"00\",\"RayID\":\"713d477539b55c29\",\"ClientRequestMethod\":\"GET\",\"ClientIP\":\"175.16.199.0\",\"ClientRequestPath\":\"/abc/checkout\",\"Action\":\"block\",\"Kind\":\"firewall\",\"RuleID\":\"7dc666e026974dab84884c73b3e2afe1\",\"ClientIPClass\":\"searchEngine\",\"ClientASNDescription\":\"CLOUDFLARENET\",\"ClientCountry\":\"us\",\"ClientRefererQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRequestQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"OriginResponseStatus\":0,\"EdgeColoCode\":\"IAD\",\"ClientRefererScheme\":\"referer URL scheme\",\"Datetime\":\"2022-05-31T05:23:43Z\",\"ClientRequestHost\":\"xyz.example.com\",\"ClientASN\":15169}", + "original": "{\"Action\":\"block\",\"ClientASN\":15169,\"ClientASNDescription\":\"CLOUDFLARENET\",\"ClientCountry\":\"us\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"searchEngine\",\"ClientRefererHost\":\"abc.example.com\",\"ClientRefererPath\":\"/abc/checkout\",\"ClientRefererQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))\\u0026timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRefererScheme\":\"referer URL scheme\",\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"GET\",\"ClientRequestPath\":\"/abc/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))\\u0026timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRequestScheme\":\"https\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\",\"Datetime\":\"2022-05-31T05:23:43Z\",\"EdgeColoCode\":\"IAD\",\"EdgeResponseStatus\":403,\"Kind\":\"firewall\",\"MatchIndex\":1,\"Metadata\":{\"filter\":\"1ced07e066a34abf8b14f2a99593bc8d\",\"type\":\"customer\"},\"OriginResponseStatus\":0,\"OriginatorRayID\":\"00\",\"RayID\":\"713d477539b55c29\",\"RuleID\":\"7dc666e026974dab84884c73b3e2afe1\",\"Source\":\"firewallrules\"}", "type": [ "info" ] @@ -108,13 +106,7 @@ "version": "1.1" }, "input": { - "type": "aws-s3" - }, - "log": { - "file": { - "path": "https://elastic-package-firewall-event-bucket-13780.s3.us-east-1.amazonaws.com/test-firewall-event.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "protocol": "http" @@ -141,8 +133,8 @@ "ip": "175.16.199.0" }, "tags": [ - "collect_sqs_logs", "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-firewall_event" ], diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json index fbbd247e365..a3da0799e66 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json @@ -69,11 +69,13 @@ "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -111,6 +113,9 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b111aaa" + }, "dns": { "answers": [ { @@ -138,7 +143,7 @@ "response_code": "0" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -165,12 +170,14 @@ "zt-test-vm1" ], "ip": [ + "89.160.20.129", "67.43.156.2", - "89.160.20.129" + "67.43.156.1", + "67.43.156.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245000000", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245000000" ] }, "source": { @@ -267,11 +274,13 @@ "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -309,6 +318,9 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b111aaa" + }, "dns": { "answers": [ { @@ -336,7 +348,7 @@ "response_code": "0" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -363,12 +375,14 @@ "zt-test-vm1" ], "ip": [ + "89.160.20.129", "67.43.156.2", - "89.160.20.129" + "67.43.156.1", + "67.43.156.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245000000", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245000000" ] }, "source": { @@ -465,11 +479,13 @@ "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -507,6 +523,9 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b111aaa" + }, "dns": { "answers": [ { @@ -534,7 +553,7 @@ "response_code": "0" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -561,12 +580,14 @@ "zt-test-vm1" ], "ip": [ + "89.160.20.129", "67.43.156.2", - "89.160.20.129" + "67.43.156.1", + "67.43.156.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245000000", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245000000" ] }, "source": { @@ -677,7 +698,7 @@ "response_code": "0" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -704,8 +725,8 @@ "81.2.69.144" ], "user": [ - "10d3bda2088ec9b9d71caa6d7f3a6657", - "me@example.com" + "me@example.com", + "10d3bda2088ec9b9d71caa6d7f3a6657" ] }, "source": { @@ -803,7 +824,7 @@ "response_code": "0" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -830,8 +851,8 @@ "81.2.69.144" ], "user": [ - "10d3bda2088ec9b9d71caa6d7f3a6657", - "me@example.com" + "me@example.com", + "10d3bda2088ec9b9d71caa6d7f3a6657" ] }, "source": { diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json b/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json index 812e5ecb7de..703cd07ee52 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json @@ -1,22 +1,41 @@ { "@timestamp": "2023-05-02T22:49:53.000Z", "agent": { - "ephemeral_id": "4b6b3e0f-122c-4c43-bc90-343c619d6b41", - "id": "57130e12-3aa9-4de1-8bab-eed6847b485d", - "name": "elastic-agent-76347", + "ephemeral_id": "def51ce2-dfc0-4efd-97a1-babdfb9f533e", + "id": "b186845d-8f10-4c72-9deb-85a958642d54", + "name": "elastic-agent-38138", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "gateway_dns": { + "answers": [ + { + "data": "CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=", + "type": "1" + }, + { + "data": "CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=", + "type": "1" + }, + { + "data": "CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=", + "type": "1" + } + ], "application_id": 0, "colo": { "code": "ORD", "id": 14 }, + "destination": { + "ip": "89.160.20.129", + "port": 443 + }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b111aaa", + "name": "zt-test-vm1" + }, "location": { "id": "f233bd67-78c7-4050-9aff-ad63cce25732", "name": "GCP default" @@ -37,6 +56,7 @@ "id": "1412", "name": "7bdc7a9c-81d3-4816-8e56-de1acad3dec5" }, + "protocol": "https", "question": { "category": { "ids": [ @@ -48,17 +68,37 @@ "Technology" ] }, + "name": "security.ubuntu.com", "reversed": "com.ubuntu.security", "size": 48, + "type": "A", "type_id": 1 }, + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", - "timezone_inferred_method": "fromLocalTime" + "response_code": "0", + "source": { + "ip": "67.43.156.2", + "port": 0 + }, + "timestamp": "2023-05-02T22:49:53.000Z", + "timezone": "UTC", + "timezone_inferred_method": "fromLocalTime", + "user": { + "email": "user@test.com", + "id": "166befbb-00e3-5e20-bd6e-27245000000" + } } }, "data_stream": { "dataset": "cloudflare_logpush.gateway_dns", - "namespace": "69154", + "namespace": "96111", "type": "logs" }, "destination": { @@ -83,6 +123,9 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b111aaa" + }, "dns": { "answers": [ { @@ -110,10 +153,10 @@ "response_code": "0" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "57130e12-3aa9-4de1-8bab-eed6847b485d", + "id": "b186845d-8f10-4c72-9deb-85a958642d54", "snapshot": false, "version": "8.17.1" }, @@ -123,37 +166,21 @@ "network" ], "dataset": "cloudflare_logpush.gateway_dns", - "ingested": "2025-12-12T09:10:08Z", + "ingested": "2026-04-09T12:12:16Z", "kind": "event", + "original": "{\"ApplicationID\":0,\"ColoCode\":\"ORD\",\"ColoID\":14,\"Datetime\":\"2023-05-02T22:49:53Z\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b111aaa\",\"DeviceName\":\"zt-test-vm1\",\"DstIP\":\"89.160.20.129\",\"DstPort\":443,\"Email\":\"user@test.com\",\"Location\":\"GCP default\",\"LocationID\":\"f233bd67-78c7-4050-9aff-ad63cce25732\",\"MatchedCategoryIDs\":[7,163],\"MatchedCategoryNames\":[\"Photography\",\"Weather\"],\"Policy\":\"7bdc7a9c-81d3-4816-8e56-de1acad3dec5\",\"PolicyID\":\"1412\",\"Protocol\":\"https\",\"QueryCategoryIDs\":[26,155],\"QueryCategoryNames\":[\"Technology\",\"Technology\"],\"QueryName\":\"security.ubuntu.com\",\"QueryNameReversed\":\"com.ubuntu.security\",\"QuerySize\":48,\"QueryType\":1,\"QueryTypeName\":\"A\",\"RCode\":0,\"RData\":[{\"data\":\"CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=\",\"type\":\"1\"},{\"data\":\"CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=\",\"type\":\"1\"},{\"data\":\"CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=\",\"type\":\"1\"}],\"ResolvedIPs\":[\"67.43.156.1\",\"67.43.156.2\",\"67.43.156.3\"],\"ResolverDecision\":\"allowedOnNoPolicyMatch\",\"SrcIP\":\"67.43.156.2\",\"SrcPort\":0,\"TimeZone\":\"UTC\",\"TimeZoneInferredMethod\":\"fromLocalTime\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245000000\"}", "outcome": "success", "timezone": "UTC", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "gateway_dns.log" - } - } - }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b111aaa", "name": "zt-test-vm1" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/gateway_dns.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "protocol": "https" @@ -164,12 +191,14 @@ "zt-test-vm1" ], "ip": [ + "89.160.20.129", "67.43.156.2", - "89.160.20.129" + "67.43.156.1", + "67.43.156.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245000000", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245000000" ] }, "source": { @@ -189,6 +218,8 @@ "port": 0 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-gateway_dns" ], diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json index aeaf9838def..82f1ad31897 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json @@ -93,8 +93,11 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", @@ -108,6 +111,14 @@ "denied" ] }, + "file": { + "extension": "bin", + "hash": { + "sha256": "91dc1db739a705105e1c763bfdbdaa84c0de8" + }, + "name": "downloaded_test", + "size": 43 + }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", "name": "zt-test-vm1" @@ -123,18 +134,22 @@ "version": "HTTP/2" }, "related": { + "hash": [ + "91dc1db739a705105e1c763bfdbdaa84c0de8" + ], "hosts": [ "083a8354-d56c-11ed-9771-6a842b100cff", - "zt-test-vm1" + "zt-test-vm1", + "guce.yahoo.com" ], "ip": [ - "67.43.156.2", "89.160.20.129", + "67.43.156.2", "192.168.1.123" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@example.com" + "user@example.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -263,8 +278,11 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", @@ -278,6 +296,14 @@ "denied" ] }, + "file": { + "extension": "bin", + "hash": { + "sha256": "91dc1db739a705105e1c763bfdbdaa84c0de8" + }, + "name": "downloaded_test", + "size": 43 + }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", "name": "zt-test-vm1" @@ -293,18 +319,22 @@ "version": "HTTP/2" }, "related": { + "hash": [ + "91dc1db739a705105e1c763bfdbdaa84c0de8" + ], "hosts": [ "083a8354-d56c-11ed-9771-6a842b100cff", - "zt-test-vm1" + "zt-test-vm1", + "guce.yahoo.com" ], "ip": [ - "67.43.156.2", "89.160.20.129", + "67.43.156.2", "192.168.1.123" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@example.com" + "user@example.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -433,8 +463,11 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", @@ -448,6 +481,14 @@ "denied" ] }, + "file": { + "extension": "bin", + "hash": { + "sha256": "91dc1db739a705105e1c763bfdbdaa84c0de8" + }, + "name": "downloaded_test", + "size": 43 + }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", "name": "zt-test-vm1" @@ -463,18 +504,22 @@ "version": "HTTP/2" }, "related": { + "hash": [ + "91dc1db739a705105e1c763bfdbdaa84c0de8" + ], "hosts": [ "083a8354-d56c-11ed-9771-6a842b100cff", - "zt-test-vm1" + "zt-test-vm1", + "guce.yahoo.com" ], "ip": [ - "67.43.156.2", "89.160.20.129", + "67.43.156.2", "192.168.1.123" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@example.com" + "user@example.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -607,8 +652,11 @@ "ip": "81.2.69.144", "port": 443 }, + "device": { + "id": "8a6c498e329785ebbc823215bad5818d" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "allow", @@ -622,6 +670,9 @@ "allowed" ] }, + "file": { + "size": 0 + }, "host": { "id": "8a6c498e329785ebbc823215bad5818d", "name": "device_name" @@ -639,14 +690,15 @@ "related": { "hosts": [ "8a6c498e329785ebbc823215bad5818d", - "device_name" + "device_name", + "example.com" ], "ip": [ "81.2.69.144" ], "user": [ - "10d3bda2088ec9b9d71caa6d7f3a6657", - "me@example.com" + "me@example.com", + "10d3bda2088ec9b9d71caa6d7f3a6657" ] }, "source": { diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json b/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json index 5c2d5b5fef9..78dbd0b93fd 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json @@ -1,18 +1,16 @@ { "@timestamp": "2023-05-03T20:55:05.000Z", "agent": { - "ephemeral_id": "48652a2c-a69f-44aa-8c1c-cadb9863f0c9", - "id": "48f0311d-16dc-41e5-b2cf-40ae0743d746", - "name": "elastic-agent-46882", + "ephemeral_id": "495cc7d6-75fe-4ee0-9344-ab1fef520b1b", + "id": "4fc671b9-3bfd-441b-982f-72e6ae663247", + "name": "elastic-agent-68727", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "gateway_http": { "account_id": "e1836771179f98aabb828da5ea69a348", + "action": "block", "blocked_file": { "hash": "91dc1db739a705105e1c763bfdbdaa84c0de8", "name": "downloaded_test", @@ -20,6 +18,10 @@ "size": 43, "type": "bin" }, + "destination": { + "ip": "89.160.20.129", + "port": 443 + }, "downloaded_files": [ "downloaded_file", "downloaded_test" @@ -36,28 +38,47 @@ } ] }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff", + "name": "zt-test-vm1" + }, "isolated": false, "policy": { "id": "85063bec-74cb-4546-85a3-e0cde2cdfda2", "name": "Block Yahoo" }, "request": { - "host": "guce.yahoo.com" + "host": "guce.yahoo.com", + "method": "GET", + "referrer": "https://www.example.com/", + "version": "HTTP/2" }, "request_id": "1884fec9b600007fb06a299400000001", + "response": { + "status_code": 302 + }, "source": { - "internal_ip": "192.168.1.123" + "internal_ip": "192.168.1.123", + "ip": "67.43.156.2", + "port": 47924 }, + "timestamp": "2023-05-03T20:55:05.000Z", "untrusted_certificate_action": "none", "uploaded_files": [ "uploaded_file", "uploaded_test" - ] + ], + "url": "https://test.com", + "user": { + "email": "user@example.com", + "id": "166befbb-00e3-5e20-bd6e-27245723949f" + }, + "user_agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0" } }, "data_stream": { "dataset": "cloudflare_logpush.gateway_http", - "namespace": "27028", + "namespace": "36007", "type": "logs" }, "destination": { @@ -82,11 +103,14 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "48f0311d-16dc-41e5-b2cf-40ae0743d746", + "id": "4fc671b9-3bfd-441b-982f-72e6ae663247", "snapshot": false, "version": "8.17.1" }, @@ -97,23 +121,21 @@ "network" ], "dataset": "cloudflare_logpush.gateway_http", - "ingested": "2025-12-12T09:16:06Z", + "ingested": "2026-04-09T12:13:16Z", "kind": "event", + "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a348\",\"Action\":\"block\",\"BlockedFileHash\":\"91dc1db739a705105e1c763bfdbdaa84c0de8\",\"BlockedFileName\":\"downloaded_test\",\"BlockedFileReason\":\"malware\",\"BlockedFileSize\":43,\"BlockedFileType\":\"bin\",\"Datetime\":\"2023-05-03T20:55:05Z\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"DownloadedFileNames\":[\"downloaded_file\",\"downloaded_test\"],\"Email\":\"user@example.com\",\"FileInfo\":{\"files\":[{\"name\":\"downloaded_file\",\"size\":43},{\"name\":\"downloaded_test\",\"size\":341}]},\"HTTPHost\":\"guce.yahoo.com\",\"HTTPMethod\":\"GET\",\"HTTPStatusCode\":302,\"HTTPVersion\":\"HTTP/2\",\"IsIsolated\":false,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"Block Yahoo\",\"Referer\":\"https://www.example.com/\",\"RequestID\":\"1884fec9b600007fb06a299400000001\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.123\",\"SourcePort\":47924,\"URL\":\"https://test.com\",\"UntrustedCertificateAction\":\"none\",\"UploadedFileNames\":[\"uploaded_file\",\"uploaded_test\"],\"UserAgent\":\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ "info", "denied" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "gateway_http.log" - } - } + "file": { + "extension": "bin", + "hash": { + "sha256": "91dc1db739a705105e1c763bfdbdaa84c0de8" + }, + "name": "downloaded_test", + "size": 43 }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", @@ -130,27 +152,25 @@ "version": "HTTP/2" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/gateway_http.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { + "hash": [ + "91dc1db739a705105e1c763bfdbdaa84c0de8" + ], "hosts": [ "083a8354-d56c-11ed-9771-6a842b100cff", - "zt-test-vm1" + "zt-test-vm1", + "guce.yahoo.com" ], "ip": [ - "67.43.156.2", "89.160.20.129", + "67.43.156.2", "192.168.1.123" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@example.com" + "user@example.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -170,6 +190,8 @@ "port": 47924 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-gateway_http" ], diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log-expected.json b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log-expected.json index 3b3ff241395..2cb716b64e7 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log-expected.json @@ -60,8 +60,11 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "allowedOnNoRuleMatch", @@ -84,19 +87,18 @@ }, "related": { "hosts": [ - "www.elastic.co", "083a8354-d56c-11ed-9771-6a842b100cff", "zt-test-vm1" ], "ip": [ - "67.43.156.2", "89.160.20.129", "175.16.199.4", + "67.43.156.2", "192.168.1.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -189,8 +191,11 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "allowedOnNoRuleMatch", @@ -213,19 +218,18 @@ }, "related": { "hosts": [ - "www.elastic.co", "083a8354-d56c-11ed-9771-6a842b100cff", "zt-test-vm1" ], "ip": [ - "67.43.156.2", "89.160.20.129", "175.16.199.4", + "67.43.156.2", "192.168.1.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -318,8 +322,11 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "allowedOnNoRuleMatch", @@ -342,19 +349,18 @@ }, "related": { "hosts": [ - "www.elastic.co", "083a8354-d56c-11ed-9771-6a842b100cff", "zt-test-vm1" ], "ip": [ - "67.43.156.2", "89.160.20.129", "175.16.199.4", + "67.43.156.2", "192.168.1.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json b/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json index e627dac9db8..961e09b62ef 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json @@ -1,18 +1,24 @@ { "@timestamp": "2023-05-18T21:12:57.058Z", "agent": { - "ephemeral_id": "e97e2537-55fc-411f-9d94-ee80a16b9840", - "id": "b7375e7f-8998-462f-8a35-45412fd644da", - "name": "elastic-agent-36424", + "ephemeral_id": "391df046-6d35-4ecc-966f-6897acd12cb2", + "id": "96ff21c9-75c6-443e-851b-0a3ef8bed874", + "name": "elastic-agent-78284", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "gateway_network": { "account_id": "e1836771179f98aabb828da5ea69a111", + "action": "allowedOnNoRuleMatch", + "destination": { + "ip": "89.160.20.129", + "port": 443 + }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff", + "name": "zt-test-vm1" + }, "override": { "ip": "175.16.199.4", "port": 8080 @@ -21,14 +27,24 @@ "id": "85063bec-74cb-4546-85a3-e0cde2cdfda2", "name": "My policy" }, + "session_id": "5f2d04be-3512-11e8-b467-0ed5f89f718b", + "sni": "www.elastic.co", "source": { - "internal_ip": "192.168.1.3" + "internal_ip": "192.168.1.3", + "ip": "67.43.156.2", + "port": 47924 + }, + "timestamp": "2023-05-18T21:12:57.058Z", + "transport": "tcp", + "user": { + "email": "user@test.com", + "id": "166befbb-00e3-5e20-bd6e-27245723949f" } } }, "data_stream": { "dataset": "cloudflare_logpush.gateway_network", - "namespace": "25906", + "namespace": "57876", "type": "logs" }, "destination": { @@ -54,11 +70,14 @@ "ip": "89.160.20.129", "port": 443 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "b7375e7f-8998-462f-8a35-45412fd644da", + "id": "96ff21c9-75c6-443e-851b-0a3ef8bed874", "snapshot": false, "version": "8.17.1" }, @@ -70,54 +89,37 @@ ], "dataset": "cloudflare_logpush.gateway_network", "id": "5f2d04be-3512-11e8-b467-0ed5f89f718b", - "ingested": "2025-12-12T09:23:26Z", + "ingested": "2026-04-09T12:14:17Z", "kind": "event", + "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"Action\":\"allowedOnNoRuleMatch\",\"Datetime\":1684444377058000000,\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"Email\":\"user@test.com\",\"OverrideIP\":\"175.16.199.4\",\"OverridePort\":8080,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"My policy\",\"SNI\":\"www.elastic.co\",\"SessionID\":\"5f2d04be-3512-11e8-b467-0ed5f89f718b\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.3\",\"SourcePort\":47924,\"Transport\":\"tcp\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "gateway_network.log" - } - } - }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", "name": "zt-test-vm1" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/gateway_network.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "tcp" }, "related": { "hosts": [ - "www.elastic.co", "083a8354-d56c-11ed-9771-6a842b100cff", "zt-test-vm1" ], "ip": [ - "67.43.156.2", "89.160.20.129", "175.16.199.4", + "67.43.156.2", "192.168.1.3" ], "user": [ - "166befbb-00e3-5e20-bd6e-27245723949f", - "user@test.com" + "user@test.com", + "166befbb-00e3-5e20-bd6e-27245723949f" ] }, "source": { @@ -137,6 +139,8 @@ "port": 47924 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-gateway_network" ], diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json b/packages/cloudflare_logpush/data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json index 9c4161d01c1..3dc2911e655 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json @@ -180,8 +180,11 @@ "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -196,9 +199,11 @@ }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -208,9 +213,17 @@ "protocol": "http" }, "related": { + "hash": [ + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -220,7 +233,8 @@ "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", @@ -430,8 +444,11 @@ "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -446,9 +463,11 @@ }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -458,9 +477,17 @@ "protocol": "http" }, "related": { + "hash": [ + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -470,7 +497,8 @@ "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", @@ -676,8 +704,11 @@ "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -692,9 +723,11 @@ }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -704,9 +737,17 @@ "protocol": "http" }, "related": { + "hash": [ + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -716,7 +757,8 @@ "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", @@ -941,8 +983,11 @@ "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -957,9 +1002,11 @@ }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -969,9 +1016,17 @@ "protocol": "http" }, "related": { + "hash": [ + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -981,7 +1036,8 @@ "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", @@ -1219,8 +1275,11 @@ "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -1235,9 +1294,11 @@ }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -1247,9 +1308,18 @@ "protocol": "http" }, "related": { + "hash": [ + "t13d1516h2_8daaf6152771_02713d6af862", + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -1259,7 +1329,8 @@ "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", @@ -1319,7 +1390,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -1341,6 +1412,9 @@ } }, "related": { + "hosts": [ + "example.com" + ], "ip": [ "81.2.69.144" ] @@ -1541,8 +1615,11 @@ "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -1557,9 +1634,11 @@ }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -1569,9 +1648,17 @@ "protocol": "http" }, "related": { + "hash": [ + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -1581,7 +1668,8 @@ "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", @@ -1690,6 +1778,7 @@ "reqs_rank_1h": 16, "uas_rank_1h": 20 }, + "js_detection_passed": "Missing", "origin": { "ip": "1.128.0.0", "response": { @@ -1727,8 +1816,11 @@ "destination": { "ip": "1.128.0.0" }, + "device": { + "type": "mobile" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -1743,9 +1835,11 @@ }, "http": { "request": { + "bytes": 2716, "method": "GET" }, "response": { + "bytes": 781, "status_code": 404 }, "version": "1.1" @@ -1755,7 +1849,12 @@ }, "related": { "hash": [ - "9b02ebd3a43b62d825e1ac605b621dc8" + "9b02ebd3a43b62d825e1ac605b621dc8", + "t13d1713h1_5b57614c22b0_eca864cca44a" + ], + "hosts": [ + "example.test.gr", + "..gr" ], "ip": [ "1.128.0.0" @@ -1768,7 +1867,8 @@ "geo": { "country_iso_code": "gr" }, - "ip": "1.128.0.0" + "ip": "1.128.0.0", + "port": 48176 }, "tags": [ "preserve_original_event", @@ -1776,6 +1876,9 @@ ], "tls": { "cipher": "AEAD-AES128-GCM-SHA256", + "client": { + "ja3": "9b02ebd3a43b62d825e1ac605b621dc8" + }, "version": "1.3", "version_protocol": "tls" }, diff --git a/packages/cloudflare_logpush/data_stream/http_request/sample_event.json b/packages/cloudflare_logpush/data_stream/http_request/sample_event.json index ade62054c76..8756178afe8 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/http_request/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2022-05-25T13:25:26.000Z", "agent": { - "ephemeral_id": "3cd010e7-a0bc-4a53-aff5-3af3c380387f", - "id": "3ce9b73c-faf9-42ad-a254-eb64a83a369c", - "name": "elastic-agent-80039", + "ephemeral_id": "a3407cc1-f3c0-44c6-9dac-2e4dc34ac934", + "id": "538a4934-da8a-4c15-9793-42ad5bbd4554", + "name": "elastic-agent-42454", "type": "filebeat", "version": "8.17.1" }, @@ -191,17 +191,20 @@ }, "data_stream": { "dataset": "cloudflare_logpush.http_request", - "namespace": "73428", + "namespace": "48349", "type": "logs" }, "destination": { "ip": "67.43.156.0" }, + "device": { + "type": "desktop" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "3ce9b73c-faf9-42ad-a254-eb64a83a369c", + "id": "538a4934-da8a-4c15-9793-42ad5bbd4554", "snapshot": false, "version": "8.17.1" }, @@ -212,7 +215,7 @@ ], "dataset": "cloudflare_logpush.http_request", "id": "710e98d9367f357d", - "ingested": "2026-01-20T11:31:07Z", + "ingested": "2026-04-09T12:15:17Z", "kind": "event", "original": "{\"BotDetectionIDs\":[7,8,9],\"BotScore\":20,\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":[\"bing\",\"api\"],\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityAction\":\"unknown\",\"SecurityLevel\":\"off\",\"SecurityRuleDescription\":\"matchad variable message\",\"SecurityRuleID\":\"98d93d5\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAttackScore\":50,\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRCEAttackScore\":1,\"WAFSQLiAttackScore\":99,\"WAFXSSAttackScore\":90,\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}", "type": [ @@ -221,9 +224,11 @@ }, "http": { "request": { + "bytes": 5800, "method": "POST" }, "response": { + "bytes": 981308, "mime_type": "application/json", "status_code": 200 }, @@ -236,9 +241,17 @@ "protocol": "http" }, "related": { + "hash": [ + "Fingerprint" + ], + "hosts": [ + "xyz.example.com", + "abc.example.com" + ], "ip": [ "175.16.199.0", - "67.43.156.0" + "67.43.156.0", + "1.128.0.0" ] }, "source": { @@ -248,7 +261,8 @@ "geo": { "country_iso_code": "sa" }, - "ip": "175.16.199.0" + "ip": "175.16.199.0", + "port": 0 }, "tags": [ "preserve_original_event", diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/test/pipeline/test-pipeline-magic-ids.log-expected.json b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/test/pipeline/test-pipeline-magic-ids.log-expected.json index 1658cd5993e..33a85d9b52b 100644 --- a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/test/pipeline/test-pipeline-magic-ids.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/test/pipeline/test-pipeline-magic-ids.log-expected.json @@ -49,7 +49,7 @@ "port": 80 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "pass", @@ -143,7 +143,7 @@ "port": 39889 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "block", diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/sample_event.json b/packages/cloudflare_logpush/data_stream/magic_ids/sample_event.json index a3eb8c63b67..7e4835f9a8c 100644 --- a/packages/cloudflare_logpush/data_stream/magic_ids/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/magic_ids/sample_event.json @@ -1,31 +1,39 @@ { "@timestamp": "2023-09-11T03:02:57.000Z", "agent": { - "ephemeral_id": "b4087b6e-9b59-4f1e-b4a1-8be9334146fb", - "id": "01cff4ad-3e8d-4819-a226-ad295d9fcb79", - "name": "elastic-agent-65074", + "ephemeral_id": "182cd4f0-d10d-43ec-87ed-b73ec15c9c94", + "id": "f8c28412-5c22-4d91-92d9-09f412249ad4", + "name": "elastic-agent-80952", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "magic_ids": { + "action": "pass", "colo": { "city": "Tokyo", "code": "NRT" }, + "destination": { + "ip": "89.160.20.129", + "port": 80 + }, "signature": { "id": 2031296, "message": "ET CURRENT_EVENTS [Fireeye] POSSIBLE HackTool.TCP.Rubeus.[User32LogonProcesss]", "revision": 1 - } + }, + "source": { + "ip": "67.43.156.2", + "port": 44667 + }, + "timestamp": "2023-09-11T03:02:57.000Z", + "transport": "tcp" } }, "data_stream": { "dataset": "cloudflare_logpush.magic_ids", - "namespace": "22953", + "namespace": "98324", "type": "logs" }, "destination": { @@ -51,10 +59,10 @@ "port": 80 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "01cff4ad-3e8d-4819-a226-ad295d9fcb79", + "id": "f8c28412-5c22-4d91-92d9-09f412249ad4", "snapshot": false, "version": "8.17.1" }, @@ -66,32 +74,16 @@ "intrusion_detection" ], "dataset": "cloudflare_logpush.magic_ids", - "ingested": "2025-12-12T09:39:17Z", + "ingested": "2026-04-09T12:16:18Z", "kind": "event", + "original": "{\"Action\":\"pass\",\"ColoCity\":\"Tokyo\",\"ColoCode\":\"NRT\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":80,\"Protocol\":\"tcp\",\"SignatureID\":2031296,\"SignatureMessage\":\"ET CURRENT_EVENTS [Fireeye] POSSIBLE HackTool.TCP.Rubeus.[User32LogonProcesss]\",\"SignatureRevision\":1,\"SourceIP\":\"67.43.156.2\",\"SourcePort\":44667,\"Timestamp\":\"2023-09-11T03:02:57Z\"}", "type": [ "info", "allowed" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "magic_ids.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/magic_ids.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "tcp" @@ -119,6 +111,8 @@ "port": 44667 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-magic_ids" ] diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log-expected.json b/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log-expected.json index 6939994b8f8..755c5e561b7 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/test/pipeline/test-pipeline-nel-report.log-expected.json @@ -26,7 +26,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "error": { "type": "network-error" @@ -72,7 +72,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "error": { "type": "network-error" @@ -118,7 +118,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "error": { "type": "network-error" diff --git a/packages/cloudflare_logpush/data_stream/nel_report/sample_event.json b/packages/cloudflare_logpush/data_stream/nel_report/sample_event.json index ed48ca0d2a0..0a9a7b62261 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/nel_report/sample_event.json @@ -1,15 +1,12 @@ { "@timestamp": "2021-07-27T00:01:07.000Z", "agent": { - "ephemeral_id": "268e4658-b07c-4510-b437-dcbb422583b5", - "id": "7ccbc26d-a497-4fcf-8a84-93b0b1f3c120", - "name": "elastic-agent-51944", + "ephemeral_id": "f47b4167-4005-4e5a-91a9-6bc194922f12", + "id": "46580a00-78be-4479-86d4-fe77270b9f1d", + "name": "elastic-agent-89164", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "nel_report": { "client": { @@ -21,24 +18,28 @@ "country": "US" } }, + "error": { + "type": "network-error" + }, "last_known_good": { "colo": { "code": "SJC" } }, - "phase": "connection" + "phase": "connection", + "timestamp": "2021-07-27T00:01:07.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.nel_report", - "namespace": "32139", + "namespace": "40594", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "7ccbc26d-a497-4fcf-8a84-93b0b1f3c120", + "id": "46580a00-78be-4479-86d4-fe77270b9f1d", "snapshot": false, "version": "8.17.1" }, @@ -51,33 +52,19 @@ "network" ], "dataset": "cloudflare_logpush.nel_report", - "ingested": "2025-12-12T09:45:27Z", + "ingested": "2026-04-09T12:17:17Z", "kind": "event", + "original": "{\"ClientIPASN\":\"13335\",\"ClientIPASNDescription\":\"CLOUDFLARENET\",\"ClientIPCountry\":\"US\",\"LastKnownGoodColoCode\":\"SJC\",\"Phase\":\"connection\",\"Timestamp\":\"2021-07-27T00:01:07Z\",\"Type\":\"network-error\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "nel_report.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/nel_report.log" - }, - "offset": 0 + "type": "http_endpoint" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-nel_report" ] diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/test/pipeline/test-pipeline-network-analytics.log-expected.json b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/test/pipeline/test-pipeline-network-analytics.log-expected.json index c44fba8ca28..94e7ace7517 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/test/pipeline/test-pipeline-network-analytics.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/test/pipeline/test-pipeline-network-analytics.log-expected.json @@ -169,7 +169,7 @@ "port": 5678 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -191,6 +191,9 @@ "hash": [ "gbuun" ], + "hosts": [ + "SJC" + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -209,7 +212,17 @@ "tags": [ "preserve_original_event", "preserve_duplicate_custom_fields" - ] + ], + "threat": { + "group": { + "id": "xyz987" + }, + "indicator": { + "id": [ + "abc777" + ] + } + } }, { "@timestamp": "2021-07-27T00:01:07.000Z", @@ -380,7 +393,7 @@ "port": 5678 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -402,6 +415,9 @@ "hash": [ "gbuun" ], + "hosts": [ + "SJC" + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -420,7 +436,17 @@ "tags": [ "preserve_original_event", "preserve_duplicate_custom_fields" - ] + ], + "threat": { + "group": { + "id": "xyz987" + }, + "indicator": { + "id": [ + "abc777" + ] + } + } }, { "@timestamp": "2021-07-27T00:01:07.000Z", @@ -591,7 +617,7 @@ "port": 5678 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -613,6 +639,9 @@ "hash": [ "gbuun" ], + "hosts": [ + "SJC" + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -631,7 +660,17 @@ "tags": [ "preserve_original_event", "preserve_duplicate_custom_fields" - ] + ], + "threat": { + "group": { + "id": "xyz987" + }, + "indicator": { + "id": [ + "abc777" + ] + } + } }, { "@timestamp": "2025-05-30T19:16:28.000Z", @@ -781,7 +820,7 @@ "port": 27015 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -805,6 +844,9 @@ "9ydqy025w0qn", "u0yhd7hbvm5r" ], + "hosts": [ + "fra16" + ], "ip": [ "81.2.69.144", "81.2.69.142" @@ -979,7 +1021,7 @@ "port": 80 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -1003,6 +1045,9 @@ "9q8yyf3hhvek", "9q9k93whbm56" ], + "hosts": [ + "sjc06" + ], "ip": [ "81.2.69.144", "81.2.69.142" diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/sample_event.json b/packages/cloudflare_logpush/data_stream/network_analytics/sample_event.json index ba6773d3598..563909f1cbf 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/network_analytics/sample_event.json @@ -1,15 +1,12 @@ { "@timestamp": "2021-07-27T00:01:07.000Z", "agent": { - "ephemeral_id": "0ed44bac-a86d-417a-83f6-06250aca48c5", - "id": "47e353e4-8ed7-44c4-b167-ad9581cbdeaf", - "name": "elastic-agent-13604", + "ephemeral_id": "0d28a777-a74f-445a-91af-dbe3dc1a8911", + "id": "46d299f3-9ad2-490b-98c4-f04527c6974d", + "name": "elastic-agent-32051", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "network_analytics": { "attack": { @@ -31,10 +28,14 @@ "description": "asn description" } }, + "asn": 1900, "country": "AD", "geo_hash": "gbuun", - "geo_location": "gbuun" + "geo_location": "gbuun", + "ip": "175.16.199.0", + "port": 1234 }, + "direction": "ingress", "gre": { "checksum": 10, "ether": { @@ -68,6 +69,7 @@ "fragments": 1480 }, "protocol": { + "name": "tcp", "value": 6 }, "source": { @@ -104,8 +106,10 @@ "scope": "local", "system": "flowtrackd" }, + "outcome": "success", "protocol_state": "OPEN", "rule": { + "id": "rule1", "set": { "id": "3b64149bfa6e4220bbbc2bd6db589552", "override": { @@ -120,9 +124,12 @@ "description": "Source ASN Description" } }, + "asn": 1500, "country": "AD", "geo_hash": "gbuun", - "geo_location": "gbuun" + "geo_location": "gbuun", + "ip": "67.43.156.0", + "port": 5678 }, "tcp": { "acknowledgement_number": 1000, @@ -151,6 +158,7 @@ "size": 10 } }, + "timestamp": "2021-07-27T00:01:07.000Z", "udp": { "checksum": 10, "payload_length": 10 @@ -160,7 +168,7 @@ }, "data_stream": { "dataset": "cloudflare_logpush.network_analytics", - "namespace": "16796", + "namespace": "34147", "type": "logs" }, "destination": { @@ -168,13 +176,13 @@ "number": 1900 }, "ip": "175.16.199.0", - "port": 0 + "port": 1234 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "47e353e4-8ed7-44c4-b167-ad9581cbdeaf", + "id": "46d299f3-9ad2-490b-98c4-f04527c6974d", "snapshot": false, "version": "8.17.1" }, @@ -184,34 +192,19 @@ "network" ], "dataset": "cloudflare_logpush.network_analytics", - "ingested": "2025-12-12T09:51:57Z", + "ingested": "2026-04-09T12:45:42Z", "kind": "event", + "original": "{\"AttackCampaignID\":\"xyz987\",\"AttackID\":\"abc777\",\"ColoCountry\":\"AD\",\"ColoGeoHash\":\"gbuun\",\"ColoID\":46,\"ColoName\":\"SJC\",\"Datetime\":\"2021-07-27T00:01:07Z\",\"DestinationASN\":1900,\"DestinationASNDescription\":\"asn description\",\"DestinationCountry\":\"AD\",\"DestinationGeoHash\":\"gbuun\",\"DestinationPort\":1234,\"Direction\":\"ingress\",\"GREChecksum\":10,\"GREEthertype\":10,\"GREHeaderLength\":1024,\"GREKey\":10,\"GRESequenceNumber\":10,\"GREVersion\":10,\"ICMPChecksum\":10,\"ICMPCode\":10,\"ICMPType\":10,\"IPDestinationAddress\":\"175.16.199.0\",\"IPDestinationSubnet\":\"/24\",\"IPFragmentOffset\":1480,\"IPHeaderLength\":20,\"IPMoreFragments\":1480,\"IPProtocol\":6,\"IPProtocolName\":\"tcp\",\"IPSourceAddress\":\"67.43.156.0\",\"IPSourceSubnet\":\"/24\",\"IPTotalLength\":1024,\"IPTotalLengthBuckets\":10,\"IPTtl\":240,\"IPTtlBuckets\":2,\"IPv4Checksum\":0,\"IPv4DontFragment\":0,\"IPv4Dscp\":46,\"IPv4Ecn\":1,\"IPv4Identification\":1,\"IPv4Options\":1,\"IPv6Dscp\":46,\"IPv6Ecn\":1,\"IPv6ExtensionHeaders\":\"header\",\"IPv6FlowLabel\":1,\"IPv6Identification\":1,\"MitigationReason\":\"BLOCKED\",\"MitigationScope\":\"local\",\"MitigationSystem\":\"flowtrackd\",\"Outcome\":\"pass\",\"ProtocolState\":\"OPEN\",\"RuleID\":\"rule1\",\"RulesetID\":\"3b64149bfa6e4220bbbc2bd6db589552\",\"RulesetOverrideID\":\"id1\",\"SampleInterval\":1,\"SourceASN\":1500,\"SourceASNDescription\":\"Source ASN Description\",\"SourceCountry\":\"AD\",\"SourceGeoHash\":\"gbuun\",\"SourcePort\":5678,\"TCPAcknowledgementNumber\":1000,\"TCPChecksum\":10,\"TCPDataOffset\":0,\"TCPFlags\":1,\"TCPFlagsString\":\"Human-readable flags string\",\"TCPMss\":512,\"TCPOptions\":\"mss\",\"TCPSackBlocks\":1,\"TCPSacksPermitted\":1,\"TCPSequenceNumber\":100,\"TCPTimestampEcr\":100,\"TCPTimestampValue\":100,\"TCPUrgentPointer\":10,\"TCPWindowScale\":10,\"TCPWindowSize\":10,\"UDPChecksum\":10,\"UDPPayloadLength\":10,\"Verdict\":\"pass\"}", "outcome": "success", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "network_analytics.log" - } - } - }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/network_analytics.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { + "community_id": "1:FRJ1/ev5eXNHmcCgyev/H8HrvH4=", "direction": "ingress", "transport": "tcp" }, @@ -219,6 +212,9 @@ "hash": [ "gbuun" ], + "hosts": [ + "SJC" + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -232,10 +228,22 @@ "number": 1500 }, "ip": "67.43.156.0", - "port": 0 + "port": 5678 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-network_analytics" - ] + ], + "threat": { + "group": { + "id": "xyz987" + }, + "indicator": { + "id": [ + "abc777" + ] + } + } } diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log-expected.json b/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log-expected.json index a191c60a83c..a167cb80e4d 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/test/pipeline/test-pipeline-network-session.log-expected.json @@ -101,14 +101,18 @@ "ip": "89.160.20.129", "port": 80 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ "network", "session" ], + "duration": 0, "end": "2023-05-04T11:29:14.000Z", "id": "18881f179300007fb0d06d6400000001", "kind": "event", @@ -136,7 +140,8 @@ "ip": [ "67.43.156.2", "89.160.20.129", - "2a02:cf40::23" + "2a02:cf40::23", + "1.128.0.1" ], "user": [ "166befbb-00e3-5e20-bd6e-27245723949f", @@ -165,6 +170,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "TLS_AES_128_GCM_SHA256", "server": { "issuer": "DigiCert Inc" } @@ -275,14 +281,18 @@ "ip": "89.160.20.129", "port": 80 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ "network", "session" ], + "duration": 0, "end": "2023-05-04T11:29:14.000Z", "id": "18881f179300007fb0d06d6400000001", "kind": "event", @@ -310,7 +320,8 @@ "ip": [ "67.43.156.2", "89.160.20.129", - "2a02:cf40::23" + "2a02:cf40::23", + "1.128.0.1" ], "user": [ "166befbb-00e3-5e20-bd6e-27245723949f", @@ -339,6 +350,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "TLS_AES_128_GCM_SHA256", "server": { "issuer": "DigiCert Inc" } @@ -449,14 +461,18 @@ "ip": "89.160.20.129", "port": 80 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ "network", "session" ], + "duration": 0, "end": "2023-05-04T11:29:14.000Z", "id": "18881f179300007fb0d06d6400000001", "kind": "event", @@ -484,7 +500,8 @@ "ip": [ "67.43.156.2", "89.160.20.129", - "2a02:cf40::23" + "2a02:cf40::23", + "1.128.0.1" ], "user": [ "166befbb-00e3-5e20-bd6e-27245723949f", @@ -513,6 +530,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "TLS_AES_128_GCM_SHA256", "server": { "issuer": "DigiCert Inc" } diff --git a/packages/cloudflare_logpush/data_stream/network_session/sample_event.json b/packages/cloudflare_logpush/data_stream/network_session/sample_event.json index 0e93b33da6b..663ab4c4b1a 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/network_session/sample_event.json @@ -1,19 +1,19 @@ { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "ba4a058c-c545-42a7-a7fa-e33b4ff32fda", - "id": "70c5474a-4b38-40f2-a354-bf18bcdc45c9", - "name": "elastic-agent-55862", + "ephemeral_id": "d785638d-aca3-4b02-b48d-7b37080e572e", + "id": "3d3e0b91-99a0-45b6-a856-fbe5ecc636f8", + "name": "elastic-agent-24801", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "network_session": { "account_id": "e1836771179f98aabb828da5ea69a111", "destination": { + "bytes": 679, + "ip": "89.160.20.129", + "port": 80, "tunnel_id": "00000000-0000-0000-0000-000000000000" }, "egress": { @@ -25,6 +25,10 @@ "name": "Egress Rule 1" } }, + "host": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff", + "name": "zt-test-vm1" + }, "ingress": { "colo_name": "ORD" }, @@ -32,8 +36,16 @@ "rule_evaluation": { "time_ms": 10 }, + "session": { + "end": "2023-05-04T11:29:14.000Z", + "id": "18881f179300007fb0d06d6400000001", + "start": "2023-05-04T11:29:14.000Z" + }, "source": { - "internal_ip": "1.128.0.1" + "bytes": 2333, + "internal_ip": "1.128.0.1", + "ip": "67.43.156.2", + "port": 52994 }, "tcp": { "client": { @@ -44,6 +56,7 @@ "reuse": false } }, + "timestamp": "2023-05-04T11:29:14.000Z", "tls": { "client": { "cipher": "TLS_AES_128_GCM_SHA256", @@ -52,18 +65,27 @@ }, "server": { "certificate": { + "issuer": "DigiCert Inc", "validation_result": "VALID" }, "cipher": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "handshake_time_ms": 130, "version": "TLS 1.2" } + }, + "transport": "TCP", + "user": { + "email": "user@test.com", + "id": "166befbb-00e3-5e20-bd6e-27245723949f" + }, + "vlan": { + "id": "0ce99869-63d3-4d5d-bdaf-d4f33df964aa" } } }, "data_stream": { "dataset": "cloudflare_logpush.network_session", - "namespace": "60780", + "namespace": "34281", "type": "logs" }, "destination": { @@ -89,11 +111,14 @@ "ip": "89.160.20.129", "port": 80 }, + "device": { + "id": "083a8354-d56c-11ed-9771-6a842b100cff" + }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "70c5474a-4b38-40f2-a354-bf18bcdc45c9", + "id": "3d3e0b91-99a0-45b6-a856-fbe5ecc636f8", "snapshot": false, "version": "8.17.1" }, @@ -104,38 +129,23 @@ "session" ], "dataset": "cloudflare_logpush.network_session", + "duration": 0, "end": "2023-05-04T11:29:14.000Z", "id": "18881f179300007fb0d06d6400000001", - "ingested": "2025-12-12T09:58:37Z", + "ingested": "2026-04-09T12:46:40Z", "kind": "event", + "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"BytesReceived\":679,\"BytesSent\":2333,\"ClientTCPHandshakeDurationMs\":12,\"ClientTLSCipher\":\"TLS_AES_128_GCM_SHA256\",\"ClientTLSHandshakeDurationMs\":125,\"ClientTLSVersion\":\"TLS 1.3\",\"ConnectionCloseReason\":\"CLIENT_CLOSED\",\"ConnectionReuse\":false,\"DestinationTunnelID\":\"00000000-0000-0000-0000-000000000000\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"EgressColoName\":\"ORD\",\"EgressIP\":\"2a02:cf40::23\",\"EgressPort\":41052,\"EgressRuleID\":\"00000000-0000-0000-0000-000000000000\",\"EgressRuleName\":\"Egress Rule 1\",\"Email\":\"user@test.com\",\"IngressColoName\":\"ORD\",\"Offramp\":\"INTERNET\",\"OriginIP\":\"89.160.20.129\",\"OriginPort\":80,\"OriginTLSCertificateIssuer\":\"DigiCert Inc\",\"OriginTLSCertificateValidationResult\":\"VALID\",\"OriginTLSCipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"OriginTLSHandshakeDurationMs\":130,\"OriginTLSVersion\":\"TLS 1.2\",\"Protocol\":\"TCP\",\"RuleEvaluationDurationMs\":10,\"SessionEndTime\":\"2023-05-04T11:29:14Z\",\"SessionID\":\"18881f179300007fb0d06d6400000001\",\"SessionStartTime\":\"2023-05-04T11:29:14Z\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"1.128.0.1\",\"SourcePort\":52994,\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\",\"VirtualNetworkID\":\"0ce99869-63d3-4d5d-bdaf-d4f33df964aa\"}", "start": "2023-05-04T11:29:14.000Z", "type": [ "connection" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "network_session.log" - } - } - }, "host": { "id": "083a8354-d56c-11ed-9771-6a842b100cff", "name": "zt-test-vm1" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/network_session.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "transport": "TCP", @@ -151,7 +161,8 @@ "ip": [ "67.43.156.2", "89.160.20.129", - "2a02:cf40::23" + "2a02:cf40::23", + "1.128.0.1" ], "user": [ "166befbb-00e3-5e20-bd6e-27245723949f", @@ -176,10 +187,13 @@ "port": 52994 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-network_session" ], "tls": { + "cipher": "TLS_AES_128_GCM_SHA256", "server": { "issuer": "DigiCert Inc" } diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/test/pipeline/test-pipeline-page-shield-events.log-expected.json b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/test/pipeline/test-pipeline-page-shield-events.log-expected.json index d7cf7810f6e..947a2213e34 100644 --- a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/test/pipeline/test-pipeline-page-shield-events.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/test/pipeline/test-pipeline-page-shield-events.log-expected.json @@ -17,7 +17,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "log", @@ -35,7 +35,8 @@ }, "related": { "hosts": [ - "hostymchost.face" + "hostymchost.face", + "example.com" ] }, "tags": [ diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/sample_event.json b/packages/cloudflare_logpush/data_stream/page_shield_events/sample_event.json index f296b8a3165..c0da15a32ec 100644 --- a/packages/cloudflare_logpush/data_stream/page_shield_events/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/sample_event.json @@ -1,35 +1,36 @@ { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "1dfb269c-1b6e-443c-8dd2-2cc980f787ff", - "id": "a717a65c-0d5d-44b7-ad3e-8459415f563c", - "name": "elastic-agent-55379", + "ephemeral_id": "08459ce8-c654-46ff-b461-08814b63ccb4", + "id": "a0cd7b50-1ec2-4305-bc0c-fca2957c4807", + "name": "elastic-agent-76009", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "page_shield_events": { + "action": "log", "csp_directive": "directive", + "host": "hostymchost.face", "page_url": "http://example.com/?query=42", "policy_id": "9", "resource_type": "other", + "timestamp": "2023-05-04T11:29:14.000Z", + "url": "https://example.com/?query=hog", "url_contains_cdn_cgi_path": true, "url_host": "example.com" } }, "data_stream": { "dataset": "cloudflare_logpush.page_shield_events", - "namespace": "48764", + "namespace": "64890", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "a717a65c-0d5d-44b7-ad3e-8459415f563c", + "id": "a0cd7b50-1ec2-4305-bc0c-fca2957c4807", "snapshot": false, "version": "8.17.1" }, @@ -40,41 +41,28 @@ "network" ], "dataset": "cloudflare_logpush.page_shield_events", - "ingested": "2025-12-12T10:04:57Z", + "ingested": "2026-04-09T12:47:37Z", "kind": "event", + "original": "{\"Action\":\"log\",\"CSPDirective\":\"directive\",\"Host\":\"hostymchost.face\",\"PageURL\":\"http://example.com/?query=42\",\"PolicyID\":\"9\",\"ResourceType\":\"other\",\"Timestamp\":\"2023-05-04T11:29:14Z\",\"URL\":\"https://example.com/?query=hog\",\"URLContainsCDNCGIPath\":true,\"URLHost\":\"example.com\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "page_shield_events.log" - } - } - }, "host": { "name": "hostymchost.face" }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/page_shield_events.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hosts": [ - "hostymchost.face" + "hostymchost.face", + "example.com" ] }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-page_shield_events" ], diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/test/pipeline/test-pipeline-sinkhole-http.log-expected.json b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/test/pipeline/test-pipeline-sinkhole-http.log-expected.json index 10e233b1984..54e0f21829d 100644 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/test/pipeline/test-pipeline-sinkhole-http.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/test/pipeline/test-pipeline-sinkhole-http.log-expected.json @@ -61,7 +61,7 @@ "ip": "89.160.20.129" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -190,7 +190,7 @@ "ip": "89.160.20.129" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ @@ -319,7 +319,7 @@ "ip": "89.160.20.129" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "category": [ diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/sample_event.json b/packages/cloudflare_logpush/data_stream/sinkhole_http/sample_event.json index c8ce41ffc74..8bd4a6289f1 100644 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/sample_event.json @@ -1,33 +1,52 @@ { "@timestamp": "2023-09-19T12:00:00.000Z", "agent": { - "ephemeral_id": "fe6f8be2-37c0-4f74-83b6-d6061fff286d", - "id": "97a57f01-e27c-4ad1-87e6-ef4b6d5e1e0e", - "name": "elastic-agent-25658", + "ephemeral_id": "19317b78-9e9c-4e24-ac35-c2433bb62b3e", + "id": "60337292-fcdb-4c92-89b4-f0902b077688", + "name": "elastic-agent-61800", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "sinkhole_http": { "account_id": "AC123456", + "destination": { + "ip": "89.160.20.129" + }, + "host": { + "name": "example.com" + }, "request": { + "body": { + "bytes": 39, + "content": "{\"action\": \"login\", \"user\": \"john_doe\"}" + }, "headers": [ "Host: example.com", "User-Agent: Mozilla/5.0", "Accept: */*", "Connection: keep-alive" ], - "password": "password123" + "method": "POST", + "password": "password123", + "referrer": "https://searchengine.com/", + "uri": "/api/v1/login", + "url": "https://example.com/api/v1/login" }, - "sinkhole_id": "SH001" + "sinkhole_id": "SH001", + "source": { + "ip": "67.43.156.2" + }, + "timestamp": "2023-09-19T12:00:00.000Z", + "user": { + "name": "john_doe" + }, + "user_agent": "Mozilla/5.0" } }, "data_stream": { "dataset": "cloudflare_logpush.sinkhole_http", - "namespace": "95284", + "namespace": "11606", "type": "logs" }, "destination": { @@ -52,10 +71,10 @@ "ip": "89.160.20.129" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "97a57f01-e27c-4ad1-87e6-ef4b6d5e1e0e", + "id": "60337292-fcdb-4c92-89b4-f0902b077688", "snapshot": false, "version": "8.17.1" }, @@ -65,23 +84,13 @@ "network" ], "dataset": "cloudflare_logpush.sinkhole_http", - "ingested": "2025-12-12T10:10:56Z", + "ingested": "2026-04-09T12:48:38Z", "kind": "event", + "original": "{\"AccountID\":\"AC123456\",\"Body\":\"{\\\"action\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\"}\",\"BodyLength\":39,\"DestAddr\":\"89.160.20.129\",\"Headers\":\"Host: example.com\\nUser-Agent: Mozilla/5.0\\nAccept: */*\\nConnection: keep-alive\",\"Host\":\"example.com\",\"Method\":\"POST\",\"Password\":\"password123\",\"R2Path\":\"\",\"Referrer\":\"https://searchengine.com/\",\"SinkholeID\":\"SH001\",\"SrcAddr\":\"67.43.156.2\",\"Timestamp\":\"2023-09-19T12:00:00Z\",\"URI\":\"/api/v1/login\",\"URL\":\"https://example.com/api/v1/login\",\"UserAgent\":\"Mozilla/5.0\",\"Username\":\"john_doe\"}", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "sinkhole_http.log" - } - } - }, "host": { "name": "example.com" }, @@ -96,13 +105,7 @@ } }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/sinkhole_http.log" - }, - "offset": 0 + "type": "http_endpoint" }, "related": { "hosts": [ @@ -132,6 +135,8 @@ "ip": "67.43.156.2" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-sinkhole_http" ], diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log-expected.json b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log-expected.json index ff07519457c..9c76da2319d 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/pipeline/test-pipeline-spectrum-event.log-expected.json @@ -29,7 +29,7 @@ "time": "2022-05-26T09:24:00.000Z" }, "disconnect": { - "time": "1970-01-01T00:00:00.000Z" + "time": "2022-06-01T00:00:00.000Z" }, "ip_firewall": false, "origin": { @@ -59,17 +59,18 @@ "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "connect", "category": [ "network" ], - "end": "1970-01-01T00:00:00.000Z", + "duration": 484560000000000, + "end": "2022-06-01T00:00:00.000Z", "id": "7ef659a2f8ef4810a9bade96fdad7c75", "kind": "event", - "original": "{\"Application\":\"7ef659a2f8ef4810a9bade96fdad7c75\",\"ClientAsn\":200391,\"ClientBytes\":0,\"ClientCountry\":\"bg\",\"ClientIP\":\"67.43.156.0\",\"ClientMatchedIpFirewall\":\"UNKNOWN\",\"ClientPort\":40456,\"ClientProto\":\"tcp\",\"ClientTcpRtt\":0,\"ClientTlsCipher\":\"UNK\",\"ClientTlsClientHelloServerName\":\"server name\",\"ClientTlsProtocol\":\"TLSv1.2\",\"ClientTlsStatus\":\"UNKNOWN\",\"ColoCode\":\"SOF\",\"ConnectTimestamp\":\"2022-05-26T09:24:00Z\",\"DisconnectTimestamp\":\"1970-01-01T00:00:00Z\",\"Event\":\"connect\",\"IpFirewall\":false,\"OriginBytes\":0,\"OriginIP\":\"175.16.199.0\",\"OriginPort\":3389,\"OriginProto\":\"tcp\",\"OriginTcpRtt\":0,\"OriginTlsCipher\":\"UNK\",\"OriginTlsFingerprint\":\"0000000000000000000000000000000000000000000000000000000000000000.\",\"OriginTlsMode\":\"off\",\"OriginTlsProtocol\":\"unknown\",\"OriginTlsStatus\":\"UNKNOWN\",\"ProxyProtocol\":\"off\",\"Status\":0,\"Timestamp\":\"2022-05-26T09:24:00Z\"}", + "original": "{\"Application\":\"7ef659a2f8ef4810a9bade96fdad7c75\",\"ClientAsn\":200391,\"ClientBytes\":0,\"ClientCountry\":\"bg\",\"ClientIP\":\"67.43.156.0\",\"ClientMatchedIpFirewall\":\"UNKNOWN\",\"ClientPort\":40456,\"ClientProto\":\"tcp\",\"ClientTcpRtt\":0,\"ClientTlsCipher\":\"UNK\",\"ClientTlsClientHelloServerName\":\"server name\",\"ClientTlsProtocol\":\"TLSv1.2\",\"ClientTlsStatus\":\"UNKNOWN\",\"ColoCode\":\"SOF\",\"ConnectTimestamp\":\"2022-05-26T09:24:00Z\",\"DisconnectTimestamp\":\"2022-06-01T00:00:00Z\",\"Event\":\"connect\",\"IpFirewall\":false,\"OriginBytes\":0,\"OriginIP\":\"175.16.199.0\",\"OriginPort\":3389,\"OriginProto\":\"tcp\",\"OriginTcpRtt\":0,\"OriginTlsCipher\":\"UNK\",\"OriginTlsFingerprint\":\"0000000000000000000000000000000000000000000000000000000000000000.\",\"OriginTlsMode\":\"off\",\"OriginTlsProtocol\":\"unknown\",\"OriginTlsStatus\":\"UNKNOWN\",\"ProxyProtocol\":\"off\",\"Status\":0,\"Timestamp\":\"2022-05-26T09:24:00Z\"}", "start": "2022-05-26T09:24:00.000Z", "type": [ "info" @@ -85,6 +86,9 @@ "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -106,6 +110,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "UNK", "version": "1.2", "version_protocol": "tls" } @@ -166,7 +171,7 @@ "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "connect", @@ -191,6 +196,9 @@ "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -212,6 +220,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "UNK", "version": "1.2", "version_protocol": "tls" } @@ -272,7 +281,7 @@ "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "connect", @@ -297,6 +306,9 @@ "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -318,6 +330,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "UNK", "version": "1.2", "version_protocol": "tls" } @@ -374,7 +387,7 @@ "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "connect", @@ -398,6 +411,9 @@ "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -419,6 +435,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "UNK", "version": "1.2", "version_protocol": "tls" } @@ -475,7 +492,7 @@ "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "connect", @@ -499,6 +516,9 @@ "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -520,6 +540,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "UNK", "version": "1.2", "version_protocol": "tls" } @@ -576,7 +597,7 @@ "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "connect", @@ -600,6 +621,9 @@ "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -621,6 +645,7 @@ "preserve_duplicate_custom_fields" ], "tls": { + "cipher": "UNK", "version": "1.2", "version_protocol": "tls" } diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/sample_event.json b/packages/cloudflare_logpush/data_stream/spectrum_event/sample_event.json index 1937111302a..1be8bef72ac 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/sample_event.json @@ -1,20 +1,23 @@ { "@timestamp": "2022-05-26T09:24:00.000Z", "agent": { - "ephemeral_id": "1e145e40-ba54-4666-86af-b8bbda38bfa6", - "id": "e7a01d20-2fff-4feb-bb44-7361b6bc648a", - "name": "elastic-agent-97143", + "ephemeral_id": "6c15f29d-826a-465a-ae6d-4944d6ae3294", + "id": "ee31129a-42a2-4a73-b55f-27553245fac9", + "name": "elastic-agent-45871", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "spectrum_event": { "action": "connect", + "application": "7ef659a2f8ef4810a9bade96fdad7c75", "client": { + "asn": 200391, + "bytes": 0, + "country": "bg", + "ip": "67.43.156.0", "matched_ip_firewall": "UNKNOWN", + "port": 40456, "protocol": "tcp", "tcp_rtt": 0, "tls": { @@ -35,6 +38,9 @@ }, "ip_firewall": false, "origin": { + "bytes": 0, + "ip": "175.16.199.0", + "port": 3389, "protocol": "tcp", "tcp_rtt": 0, "tls": { @@ -47,12 +53,14 @@ }, "proxy": { "protocol": "off" - } + }, + "status": 0, + "timestamp": "2022-05-26T09:24:00.000Z" } }, "data_stream": { "dataset": "cloudflare_logpush.spectrum_event", - "namespace": "19575", + "namespace": "72798", "type": "logs" }, "destination": { @@ -61,10 +69,10 @@ "port": 3389 }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "e7a01d20-2fff-4feb-bb44-7361b6bc648a", + "id": "ee31129a-42a2-4a73-b55f-27553245fac9", "snapshot": false, "version": "8.17.1" }, @@ -75,45 +83,33 @@ "network" ], "dataset": "cloudflare_logpush.spectrum_event", + "duration": -1653557040000000000, "end": "1970-01-01T00:00:00.000Z", "id": "7ef659a2f8ef4810a9bade96fdad7c75", - "ingested": "2025-12-12T10:17:07Z", + "ingested": "2026-04-09T12:49:39Z", "kind": "event", + "original": "{\"Application\":\"7ef659a2f8ef4810a9bade96fdad7c75\",\"ClientAsn\":200391,\"ClientBytes\":0,\"ClientCountry\":\"bg\",\"ClientIP\":\"67.43.156.0\",\"ClientMatchedIpFirewall\":\"UNKNOWN\",\"ClientPort\":40456,\"ClientProto\":\"tcp\",\"ClientTcpRtt\":0,\"ClientTlsCipher\":\"UNK\",\"ClientTlsClientHelloServerName\":\"server name\",\"ClientTlsProtocol\":\"unknown\",\"ClientTlsStatus\":\"UNKNOWN\",\"ColoCode\":\"SOF\",\"ConnectTimestamp\":\"2022-05-26T09:24:00Z\",\"DisconnectTimestamp\":\"1970-01-01T00:00:00Z\",\"Event\":\"connect\",\"IpFirewall\":false,\"OriginBytes\":0,\"OriginIP\":\"175.16.199.0\",\"OriginPort\":3389,\"OriginProto\":\"tcp\",\"OriginTcpRtt\":0,\"OriginTlsCipher\":\"UNK\",\"OriginTlsFingerprint\":\"0000000000000000000000000000000000000000000000000000000000000000.\",\"OriginTlsMode\":\"off\",\"OriginTlsProtocol\":\"unknown\",\"OriginTlsStatus\":\"UNKNOWN\",\"ProxyProtocol\":\"off\",\"Status\":0,\"Timestamp\":\"2022-05-26T09:24:00Z\"}", "start": "2022-05-26T09:24:00.000Z", "type": [ "info" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "spectrum_event.log" - } - } - }, "http": { "response": { "status_code": 0 } }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/spectrum_event.log" - }, - "offset": 0 + "type": "http_endpoint" }, "network": { "community_id": "1:X7lywUVKlduqRq5SyCRaBj4hLP0=", "transport": "tcp" }, "related": { + "hash": [ + "0000000000000000000000000000000000000000000000000000000000000000." + ], "ip": [ "67.43.156.0", "175.16.199.0" @@ -131,7 +127,12 @@ "port": 40456 }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-spectrum_event" - ] + ], + "tls": { + "cipher": "UNK" + } } diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/test/pipeline/test-pipeline-workers-trace.log-expected.json b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/test/pipeline/test-pipeline-workers-trace.log-expected.json index 36ad9607a45..aa33d903016 100644 --- a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/test/pipeline/test-pipeline-workers-trace.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/test/pipeline/test-pipeline-workers-trace.log-expected.json @@ -44,7 +44,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "fetch", @@ -106,7 +106,7 @@ } }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "event": { "action": "fetch", diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/sample_event.json b/packages/cloudflare_logpush/data_stream/workers_trace/sample_event.json index 85b75f73216..80309b8d08c 100644 --- a/packages/cloudflare_logpush/data_stream/workers_trace/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/workers_trace/sample_event.json @@ -1,18 +1,25 @@ { "@timestamp": "2023-07-20T11:35:46.804Z", "agent": { - "ephemeral_id": "7be35875-0716-4185-a715-56f4262e6677", - "id": "5de705c9-500f-4f23-b85e-94dfb165569e", - "name": "elastic-agent-35180", + "ephemeral_id": "ad4fc167-31f0-412e-b1ee-d96d6dfb5f0f", + "id": "5cf9216c-17ec-48fc-b595-875f0002f34e", + "name": "elastic-agent-46737", "type": "filebeat", "version": "8.17.1" }, - "cloud": { - "provider": "google cloud" - }, "cloudflare_logpush": { "workers_trace": { "dispatch_namespace": "my-worker-dispatch", + "event": { + "ray_id": "7e9ae7157ac0c33a", + "request": { + "method": "GET", + "url": "http://chat-gpt-little-butterfly-0c3d.example.workers.dev/v2/_catalog" + }, + "response": { + "status": 404 + } + }, "exceptions": [ { "message": "Uncaught TypeError: Cannot read property 'x' of undefined", @@ -29,25 +36,28 @@ "message": "Something went wrong" } ], + "outcome": "exception", "script": { "name": "chat-gpt-little-butterfly-0c3d", "tags": [ "api", "chatgpt" ] - } + }, + "timestamp": "2023-07-20T11:35:46.804Z", + "type": "fetch" } }, "data_stream": { "dataset": "cloudflare_logpush.workers_trace", - "namespace": "22099", + "namespace": "96709", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "9.3.0" }, "elastic_agent": { - "id": "5de705c9-500f-4f23-b85e-94dfb165569e", + "id": "5cf9216c-17ec-48fc-b595-875f0002f34e", "snapshot": false, "version": "8.17.1" }, @@ -59,25 +69,15 @@ ], "dataset": "cloudflare_logpush.workers_trace", "id": "7e9ae7157ac0c33a", - "ingested": "2025-12-12T10:23:03Z", + "ingested": "2026-04-09T12:50:39Z", "kind": "event", + "original": "{\"DispatchNamespace\":\"my-worker-dispatch\",\"Event\":{\"RayID\":\"7e9ae7157ac0c33a\",\"Request\":{\"Method\":\"GET\",\"URL\":\"http://chat-gpt-little-butterfly-0c3d.example.workers.dev/v2/_catalog\"},\"Response\":{\"Status\":404}},\"EventTimestampMs\":1689852946804,\"EventType\":\"fetch\",\"Exceptions\":[{\"Message\":\"Uncaught TypeError: Cannot read property 'x' of undefined\",\"Stack\":\"TypeError: Cannot read property 'x' of undefined\\n at fetchHandler (/workers/script.js:12:27)\\n at handleRequest (/workers/script.js:6:13)\"}],\"Logs\":[{\"level\":\"info\",\"message\":\"Request received for /api/data\"},{\"level\":\"error\",\"message\":\"Something went wrong\"}],\"Outcome\":\"exception\",\"ScriptName\":\"chat-gpt-little-butterfly-0c3d\",\"ScriptTags\":[\"api\",\"chatgpt\"]}", "outcome": "failure", "type": [ "info", "error" ] }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/json", - "name": "workers_trace.log" - } - } - }, "http": { "request": { "method": "GET" @@ -87,15 +87,11 @@ } }, "input": { - "type": "gcs" - }, - "log": { - "file": { - "path": "gs://testbucket/workers_trace.log" - }, - "offset": 0 + "type": "http_endpoint" }, "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields", "forwarded", "cloudflare_logpush-workers_trace" ], diff --git a/packages/cloudflare_logpush/docs/README.md b/packages/cloudflare_logpush/docs/README.md index 6622394d961..04e5fcb3fe6 100644 --- a/packages/cloudflare_logpush/docs/README.md +++ b/packages/cloudflare_logpush/docs/README.md @@ -207,9 +207,9 @@ An example event for `access_request` looks as following: { "@timestamp": "2023-05-23T17:18:33.000Z", "agent": { - "ephemeral_id": "a4b006f5-ea6d-4027-8aa3-2c81ff622f0d", - "id": "39bb77e3-2cdc-4e93-9302-147cd7c45658", - "name": "elastic-agent-84026", + "ephemeral_id": "801054ca-d5cc-4f9a-aa8a-4e4ee5b74814", + "id": "107ca55f-c937-46f3-9974-58527f82a166", + "name": "elastic-agent-58297", "type": "filebeat", "version": "8.17.1" }, @@ -264,14 +264,14 @@ An example event for `access_request` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.access_request", - "namespace": "88021", + "namespace": "85961", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "39bb77e3-2cdc-4e93-9302-147cd7c45658", + "id": "107ca55f-c937-46f3-9974-58527f82a166", "snapshot": false, "version": "8.17.1" }, @@ -283,7 +283,7 @@ An example event for `access_request` looks as following: ], "dataset": "cloudflare_logpush.access_request", "id": "00c0ffeeabc12345", - "ingested": "2026-04-07T10:14:21Z", + "ingested": "2026-04-09T11:24:10Z", "kind": "event", "original": "{\"Action\":\"login\",\"Allowed\":true,\"AppDomain\":\"partner-zt-logs.cloudflareaccess.com/warp\",\"AppUUID\":\"123e4567-e89b-12d3-a456-426614174000\",\"Connection\":\"onetimepin\",\"Country\":\"us\",\"CreatedAt\":1684862313000000000,\"Email\":\"user@example.com\",\"IPAddress\":\"67.43.156.93\",\"PurposeJustificationPrompt\":\"Please provide your reason for accessing the application.\",\"PurposeJustificationResponse\":\"I need to access the application for work purposes.\",\"RayID\":\"00c0ffeeabc12345\",\"TemporaryAccessApprovers\":[\"approver1@example.com\",\"approver2@example.com\"],\"TemporaryAccessDuration\":7200,\"UserUID\":\"166befbb-00e3-5e20-bd6e-27245333949f\"}", "type": [ @@ -374,9 +374,9 @@ An example event for `audit` looks as following: { "@timestamp": "2021-11-30T20:19:48.000Z", "agent": { - "ephemeral_id": "a95e5d2e-311a-4aae-ae73-f9e157308c0e", - "id": "8992a634-8664-45e9-9581-cd467d2590e0", - "name": "elastic-agent-87538", + "ephemeral_id": "a0211120-06cd-4e63-8802-1e13ef915aad", + "id": "9c1a4701-f21a-4bf1-839e-0176c1bae24b", + "name": "elastic-agent-38547", "type": "filebeat", "version": "8.17.1" }, @@ -418,14 +418,14 @@ An example event for `audit` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.audit", - "namespace": "77347", + "namespace": "29929", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "8992a634-8664-45e9-9581-cd467d2590e0", + "id": "9c1a4701-f21a-4bf1-839e-0176c1bae24b", "snapshot": false, "version": "8.17.1" }, @@ -437,7 +437,7 @@ An example event for `audit` looks as following: ], "dataset": "cloudflare_logpush.audit", "id": "73fd39ed-5aab-4a2a-b93c-c9a4abf0c425", - "ingested": "2026-04-07T10:15:20Z", + "ingested": "2026-04-09T11:25:10Z", "kind": "event", "original": "{\"ActionResult\":true,\"ActionType\":\"token_create\",\"ActorEmail\":\"user@example.com\",\"ActorID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ActorIP\":\"81.2.69.142\",\"ActorType\":\"user\",\"ID\":\"73fd39ed-5aab-4a2a-b93c-c9a4abf0c425\",\"Interface\":\"UI\",\"Metadata\":{\"token_name\":\"test\",\"token_tag\":\"b7261c49a793a82678d12285f0bc1401\"},\"NewValue\":{\"key1\":\"value1\",\"key2\":\"value2\"},\"OldValue\":{\"key3\":\"value4\",\"key4\":\"value4\"},\"OwnerID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ResourceID\":\"enl3j9du8rnx2swwd9l32qots7l54t9s\",\"ResourceType\":\"account\",\"When\":\"2021-11-30T20:19:48Z\"}", "outcome": "success", @@ -526,9 +526,9 @@ An example event for `casb` looks as following: { "@timestamp": "2023-05-16T10:00:00.000Z", "agent": { - "ephemeral_id": "e6196296-ea3c-4d44-8154-c7c5bc5bd548", - "id": "17cbd457-fcca-44a8-9448-2fdec84f262b", - "name": "elastic-agent-21002", + "ephemeral_id": "8ec02fce-f2f2-4839-97c3-1c76a7b02c7b", + "id": "12524cd6-a1de-484d-a9a6-e70ddd09c755", + "name": "elastic-agent-24633", "type": "filebeat", "version": "8.17.1" }, @@ -606,14 +606,14 @@ An example event for `casb` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.casb", - "namespace": "21142", + "namespace": "16545", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "17cbd457-fcca-44a8-9448-2fdec84f262b", + "id": "12524cd6-a1de-484d-a9a6-e70ddd09c755", "snapshot": false, "version": "8.17.1" }, @@ -624,7 +624,7 @@ An example event for `casb` looks as following: ], "dataset": "cloudflare_logpush.casb", "id": "6b187be4-2dd5-42c5-a37b-111111111111", - "ingested": "2026-04-07T10:16:20Z", + "ingested": "2026-04-09T11:26:09Z", "kind": "event", "original": "{\"AssetDisplayName\":\"John Doe\",\"AssetExternalID\":\"0051N000004mG2LAAA\",\"AssetLink\":\"https://example.com/resource\",\"AssetMetadata\":{\"AccountId\":null,\"Address\":{\"city\":\"Singapore\",\"country\":\"Singapore\",\"countryCode\":\"SG\",\"geocodeAccuracy\":null,\"latitude\":null,\"longitude\":null,\"postalCode\":null,\"state\":null,\"stateCode\":null,\"street\":null},\"Alias\":\"JDoe\",\"BadgeText\":\"\",\"BannerPhotoUrl\":\"/profilephoto/001\",\"CallCenterId\":null,\"CommunityNickname\":\"Doe.John\",\"CompanyName\":\"MyCompany\",\"ContactId\":null,\"DefaultGroupNotificationFrequency\":\"N\",\"Department\":\"521\",\"DigestFrequency\":\"D\",\"Division\":null,\"Email\":\"user@example.com\",\"EmailEncodingKey\":\"UTF-8\",\"EmailPreferencesAutoBcc\":true,\"EmployeeNumber\":\"18124\",\"Extension\":null,\"Fax\":null,\"FederationIdentifier\":null,\"FirstName\":\"John\",\"ForecastEnabled\":false,\"FullPhotoUrl\":\"https://photos.com/profilephoto/001\",\"Id\":\"0051N000004mG2LAAA\",\"IsActive\":false,\"IsProfilePhotoActive\":false,\"LanguageLocaleKey\":\"en_US\",\"LastLoginDate\":\"2021-10-06T06:32:09.000+0000\",\"LastName\":\"Doe\",\"LastReferencedDate\":null,\"LastViewedDate\":null,\"LocaleSidKey\":\"en_SG\",\"MediumBannerPhotoUrl\":\"/profilephoto/001/E\",\"MobilePhone\":null,\"Name\":\"John Doe\",\"OfflineTrialExpirationDate\":null,\"Phone\":\"+3460000000\",\"ReceivesAdminInfoEmails\":true,\"ReceivesInfoEmails\":true,\"SenderEmail\":\"sender@example.com\",\"SenderName\":null,\"Signature\":null,\"SmallBannerPhotoUrl\":\"/profilephoto/001/D\",\"SmallPhotoUrl\":\"https://photos.com/photo/001\",\"TimeZoneSidKey\":\"Asia/Singapore\",\"Title\":\"Customer Solutions Engineer\",\"UserPermissionsCallCenterAutoLogin\":false,\"UserPermissionsInteractionUser\":true,\"UserPermissionsMarketingUser\":false,\"UserPermissionsOfflineUser\":false,\"UserPermissionsSupportUser\":false,\"UserRoleId\":\"00E2G000001E\",\"UserType\":\"Standard\",\"attributes\":{\"type\":\"User\",\"url\":\"/services/data/userID\"}},\"DetectedTimestamp\":\"2023-05-16T10:00:00Z\",\"FindingTypeDisplayName\":\"Salesforce User Sending Email with Different Email Address\",\"FindingTypeID\":\"a2790c4f-03f5-449f-b209-5f4447f417aa\",\"FindingTypeSeverity\":\"Medium\",\"InstanceID\":\"6b187be4-2dd5-42c5-a37b-111111111111\",\"IntegrationDisplayName\":\"Salesforce Testing\",\"IntegrationID\":\"c772678d-5cf1-4c73-bf3f-111111111111\",\"IntegrationPolicyVendor\":\"Salesforce Connection\"}", "severity": 2, @@ -699,9 +699,9 @@ An example event for `device_posture` looks as following: { "@timestamp": "2023-05-17T12:00:00.000Z", "agent": { - "ephemeral_id": "697666b7-2c61-4b7e-8a91-8582b66162ab", - "id": "3452c3cf-0cc4-408d-a6b0-f55c7f306781", - "name": "elastic-agent-71222", + "ephemeral_id": "e08f9602-22ca-4bbd-88b6-9a0b0811783b", + "id": "beb39b36-63df-4209-9ab6-fcdfe0ba4b5d", + "name": "elastic-agent-12848", "type": "filebeat", "version": "8.17.1" }, @@ -748,7 +748,7 @@ An example event for `device_posture` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.device_posture", - "namespace": "60433", + "namespace": "48116", "type": "logs" }, "device": { @@ -764,7 +764,7 @@ An example event for `device_posture` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "3452c3cf-0cc4-408d-a6b0-f55c7f306781", + "id": "beb39b36-63df-4209-9ab6-fcdfe0ba4b5d", "snapshot": false, "version": "8.17.1" }, @@ -774,7 +774,7 @@ An example event for `device_posture` looks as following: "host" ], "dataset": "cloudflare_logpush.device_posture", - "ingested": "2026-04-07T10:17:21Z", + "ingested": "2026-04-09T11:27:07Z", "kind": "event", "original": "{\"ClientVersion\":\"2023.3.258\",\"DeviceID\":\"083a8354-d56c-11ed-9771-111111111\",\"DeviceManufacturer\":\"Google Compute Engine\",\"DeviceModel\":\"Google Compute Engine\",\"DeviceName\":\"zt-test-vm1\",\"DeviceSerialNumber\":\"GoogleCloud-ABCD1234567890\",\"DeviceType\":\"linux\",\"Email\":\"user@example.com\",\"OSVersion\":\"5.15.0\",\"PolicyID\":\"policy-abcdefgh\",\"PostureCheckName\":\"Ubuntu\",\"PostureCheckType\":\"os_version\",\"PostureEvaluatedResult\":true,\"PostureExpectedJSON\":{\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\",\"version\":\"5.15.0-1025-gcp\"},\"PostureReceivedJSON\":{\"operator\":\"==\",\"os_distro_name\":\"ubuntu\",\"os_distro_revision\":\"20.04\",\"version\":\"5.15.0-1025-gcp\"},\"Timestamp\":\"2023-05-17T12:00:00Z\",\"UserUID\":\"user-abcdefgh\"}", "outcome": "success", @@ -879,9 +879,9 @@ An example event for `dlp_forensic_copies` looks as following: { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "5eca1e85-a7a9-4c98-920e-c688eddd8dfa", - "id": "c938d3c0-71e7-4d2d-9f16-6c3193069652", - "name": "elastic-agent-17032", + "ephemeral_id": "208956c5-24de-474a-b20f-ff5ad6cbac48", + "id": "fafd216b-168c-4f9e-9e00-8481e2e330b9", + "name": "elastic-agent-41362", "type": "filebeat", "version": "8.17.1" }, @@ -902,14 +902,14 @@ An example event for `dlp_forensic_copies` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.dlp_forensic_copies", - "namespace": "36828", + "namespace": "30202", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "c938d3c0-71e7-4d2d-9f16-6c3193069652", + "id": "fafd216b-168c-4f9e-9e00-8481e2e330b9", "snapshot": false, "version": "8.17.1" }, @@ -919,7 +919,7 @@ An example event for `dlp_forensic_copies` looks as following: "network" ], "dataset": "cloudflare_logpush.dlp_forensic_copies", - "ingested": "2026-04-07T10:18:20Z", + "ingested": "2026-04-09T12:07:18Z", "kind": "event", "original": "{\"AccountID\":\"acc-id\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\"}", "type": [ @@ -986,9 +986,9 @@ An example event for `dns` looks as following: { "@timestamp": "2022-05-26T09:23:54.000Z", "agent": { - "ephemeral_id": "52bd7d3d-8a51-4486-b6c3-bdc863b853d2", - "id": "29ff4554-a2a6-43c4-a7c6-a69bdd20aad3", - "name": "elastic-agent-20485", + "ephemeral_id": "6e8cd3c5-88ea-41d7-8778-fff4144c7e24", + "id": "3d115c8b-f50d-487c-ae2d-0be7ab8d2307", + "name": "elastic-agent-66764", "type": "filebeat", "version": "8.17.1" }, @@ -1017,7 +1017,7 @@ An example event for `dns` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.dns", - "namespace": "29720", + "namespace": "42607", "type": "logs" }, "dns": { @@ -1030,7 +1030,7 @@ An example event for `dns` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "29ff4554-a2a6-43c4-a7c6-a69bdd20aad3", + "id": "3d115c8b-f50d-487c-ae2d-0be7ab8d2307", "snapshot": false, "version": "8.17.1" }, @@ -1040,7 +1040,7 @@ An example event for `dns` looks as following: "network" ], "dataset": "cloudflare_logpush.dns", - "ingested": "2026-04-07T10:19:17Z", + "ingested": "2026-04-09T12:08:16Z", "kind": "event", "original": "{\"ColoCode\":\"MRS\",\"EDNSSubnet\":\"1.128.0.0\",\"EDNSSubnetLength\":0,\"QueryName\":\"example.com\",\"QueryType\":65535,\"ResponseCached\":false,\"ResponseCode\":0,\"SourceIP\":\"175.16.199.0\",\"Timestamp\":\"2022-05-26T09:23:54Z\"}", "type": [ @@ -1114,9 +1114,9 @@ An example event for `dns_firewall` looks as following: { "@timestamp": "2023-09-19T12:30:00.000Z", "agent": { - "ephemeral_id": "0bae3f0e-8910-457f-815b-c9c7c798fded", - "id": "71e660c4-04ce-40ed-b44d-69ba6c7bcf60", - "name": "elastic-agent-48112", + "ephemeral_id": "1701442d-5ccf-456d-9a7c-65110dae2a77", + "id": "46fd2c9a-f97e-4c91-9ee6-2ee13dfd5c62", + "name": "elastic-agent-60076", "type": "filebeat", "version": "8.17.1" }, @@ -1156,7 +1156,7 @@ An example event for `dns_firewall` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.dns_firewall", - "namespace": "87794", + "namespace": "48306", "type": "logs" }, "dns": { @@ -1170,7 +1170,7 @@ An example event for `dns_firewall` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "71e660c4-04ce-40ed-b44d-69ba6c7bcf60", + "id": "46fd2c9a-f97e-4c91-9ee6-2ee13dfd5c62", "snapshot": false, "version": "8.17.1" }, @@ -1180,7 +1180,7 @@ An example event for `dns_firewall` looks as following: "network" ], "dataset": "cloudflare_logpush.dns_firewall", - "ingested": "2026-04-07T10:20:19Z", + "ingested": "2026-04-09T12:09:16Z", "kind": "event", "original": "{\"ClientResponseCode\":0,\"ClusterID\":\"CLUSTER-001\",\"ColoCode\":\"SFO\",\"EDNSSubnet\":\"67.43.156.0\",\"EDNSSubnetLength\":24,\"QueryDO\":true,\"QueryName\":\"example.com\",\"QueryRD\":true,\"QuerySize\":60,\"QueryTCP\":false,\"QueryType\":1,\"ResponseCached\":true,\"ResponseCachedStale\":false,\"SourceIP\":\"67.43.156.2\",\"Timestamp\":\"2023-09-19T12:30:00Z\",\"UpstreamIP\":\"81.2.69.144\",\"UpstreamResponseCode\":0,\"UpstreamResponseTimeMs\":30}", "type": [ @@ -1280,9 +1280,9 @@ An example event for `email_security_alerts` looks as following: { "@timestamp": "2024-08-28T15:32:35.000Z", "agent": { - "ephemeral_id": "c0ef4c22-6924-4e42-86cf-e213e8c61ba4", - "id": "e991eee8-5ba2-4777-bbe2-f414cda793b8", - "name": "elastic-agent-79774", + "ephemeral_id": "8329703a-0e11-43c7-8a5a-28d2bbd62dfc", + "id": "6960421c-0684-4f16-ad7c-ed15de6943cd", + "name": "elastic-agent-55230", "type": "filebeat", "version": "8.17.1" }, @@ -1345,14 +1345,14 @@ An example event for `email_security_alerts` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.email_security_alerts", - "namespace": "89350", + "namespace": "46333", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "e991eee8-5ba2-4777-bbe2-f414cda793b8", + "id": "6960421c-0684-4f16-ad7c-ed15de6943cd", "snapshot": false, "version": "8.17.1" }, @@ -1399,7 +1399,7 @@ An example event for `email_security_alerts` looks as following: "network" ], "dataset": "cloudflare_logpush.email_security_alerts", - "ingested": "2026-04-07T10:21:21Z", + "ingested": "2026-04-09T12:10:18Z", "kind": "alert", "original": "{\"AlertID\":\"4WtWkr6nlBz9sNH-2024-08-28T15:32:35\",\"AlertReasons\":[\"because\",\"said-so\"],\"Attachments\":[{\"ContentTypeComputed\":\"application/x-msi\",\"ContentTypeProvided\":\"image/gif\",\"Decrypted\":true,\"Encrypted\":true,\"Md5\":\"91f073bd208689ddbd248e8989ecae90\",\"Name\":\"attachment.gif\",\"Sha1\":\"62b77e14e2c43049c45b5725018e78d0f9986930\",\"Sha256\":\"3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06\",\"Ssdeep\":\"McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3\"}],\"CC\":[\"firstlast+cc@cloudflare.com\"],\"CCName\":[\"First Last (cc)\"],\"FinalDisposition\":\"malicious\",\"From\":\"firstlast+from@cloudflare.com\",\"FromName\":\"First Last (from)\",\"Links\":[\"https://example.com\"],\"MessageDeliveryMode\":\"unset\",\"MessageID\":\"\\u003cMessage-ID\\u003e\",\"Origin\":\"unset\",\"OriginalSender\":\"firstlast+origin@cloudflare.com\",\"ReplyTo\":\"firstlast+reply@cloudflare.com\",\"ReplyToName\":\"First Last (reply)\",\"SMTPEnvelopeFrom\":\"firstlast+env_from@cloudflare.com\",\"SMTPEnvelopeTo\":[\"firstlast+env_to@cloudflare.com\"],\"SMTPHeloServerIP\":\"81.2.69.144\",\"SMTPHeloServerIPAsName\":\"asn\",\"SMTPHeloServerIPAsNumber\":\"42\",\"SMTPHeloServerIPGeo\":\"US/NV/Las Vegas\",\"SMTPHeloServerName\":\"servername\",\"Subject\":\"innocuous message: please read\",\"ThreatCategories\":[\"CredentialHarvester\",\"Dropper\"],\"Timestamp\":\"2024-08-28T15:32:35Z\",\"To\":\"firstlast+to@cloudflare.com\",\"ToName\":\"First Last (to)\"}", "type": [ @@ -1540,9 +1540,9 @@ An example event for `firewall_event` looks as following: { "@timestamp": "2022-05-31T05:23:43.000Z", "agent": { - "ephemeral_id": "4a6896a5-f0b1-4275-9bf9-e6e92c06a166", - "id": "dc6bddca-eefa-4068-adc7-79eda79dc040", - "name": "elastic-agent-39254", + "ephemeral_id": "e85ec68a-f705-433b-8974-b945f4f35fbd", + "id": "837b9eb7-7faa-42d4-bd20-6e9e76fb1876", + "name": "elastic-agent-61647", "type": "filebeat", "version": "8.17.1" }, @@ -1609,14 +1609,14 @@ An example event for `firewall_event` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.firewall_event", - "namespace": "93638", + "namespace": "50135", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "dc6bddca-eefa-4068-adc7-79eda79dc040", + "id": "837b9eb7-7faa-42d4-bd20-6e9e76fb1876", "snapshot": false, "version": "8.17.1" }, @@ -1628,7 +1628,7 @@ An example event for `firewall_event` looks as following: ], "dataset": "cloudflare_logpush.firewall_event", "id": "713d477539b55c29", - "ingested": "2026-04-07T10:22:20Z", + "ingested": "2026-04-09T12:11:17Z", "kind": "event", "original": "{\"Action\":\"block\",\"ClientASN\":15169,\"ClientASNDescription\":\"CLOUDFLARENET\",\"ClientCountry\":\"us\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"searchEngine\",\"ClientRefererHost\":\"abc.example.com\",\"ClientRefererPath\":\"/abc/checkout\",\"ClientRefererQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))\\u0026timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRefererScheme\":\"referer URL scheme\",\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"GET\",\"ClientRequestPath\":\"/abc/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestQuery\":\"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))\\u0026timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))\",\"ClientRequestScheme\":\"https\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\",\"Datetime\":\"2022-05-31T05:23:43Z\",\"EdgeColoCode\":\"IAD\",\"EdgeResponseStatus\":403,\"Kind\":\"firewall\",\"MatchIndex\":1,\"Metadata\":{\"filter\":\"1ced07e066a34abf8b14f2a99593bc8d\",\"type\":\"customer\"},\"OriginResponseStatus\":0,\"OriginatorRayID\":\"00\",\"RayID\":\"713d477539b55c29\",\"RuleID\":\"7dc666e026974dab84884c73b3e2afe1\",\"Source\":\"firewallrules\"}", "type": [ @@ -1772,9 +1772,9 @@ An example event for `gateway_dns` looks as following: { "@timestamp": "2023-05-02T22:49:53.000Z", "agent": { - "ephemeral_id": "78dc937a-0f0c-4fb7-9945-2e49bbc091c6", - "id": "b69b3b71-ded8-4572-b93d-1ef026594275", - "name": "elastic-agent-96379", + "ephemeral_id": "def51ce2-dfc0-4efd-97a1-babdfb9f533e", + "id": "b186845d-8f10-4c72-9deb-85a958642d54", + "name": "elastic-agent-38138", "type": "filebeat", "version": "8.17.1" }, @@ -1869,7 +1869,7 @@ An example event for `gateway_dns` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.gateway_dns", - "namespace": "45109", + "namespace": "96111", "type": "logs" }, "destination": { @@ -1927,7 +1927,7 @@ An example event for `gateway_dns` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "b69b3b71-ded8-4572-b93d-1ef026594275", + "id": "b186845d-8f10-4c72-9deb-85a958642d54", "snapshot": false, "version": "8.17.1" }, @@ -1937,7 +1937,7 @@ An example event for `gateway_dns` looks as following: "network" ], "dataset": "cloudflare_logpush.gateway_dns", - "ingested": "2026-04-07T10:23:18Z", + "ingested": "2026-04-09T12:12:16Z", "kind": "event", "original": "{\"ApplicationID\":0,\"ColoCode\":\"ORD\",\"ColoID\":14,\"Datetime\":\"2023-05-02T22:49:53Z\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b111aaa\",\"DeviceName\":\"zt-test-vm1\",\"DstIP\":\"89.160.20.129\",\"DstPort\":443,\"Email\":\"user@test.com\",\"Location\":\"GCP default\",\"LocationID\":\"f233bd67-78c7-4050-9aff-ad63cce25732\",\"MatchedCategoryIDs\":[7,163],\"MatchedCategoryNames\":[\"Photography\",\"Weather\"],\"Policy\":\"7bdc7a9c-81d3-4816-8e56-de1acad3dec5\",\"PolicyID\":\"1412\",\"Protocol\":\"https\",\"QueryCategoryIDs\":[26,155],\"QueryCategoryNames\":[\"Technology\",\"Technology\"],\"QueryName\":\"security.ubuntu.com\",\"QueryNameReversed\":\"com.ubuntu.security\",\"QuerySize\":48,\"QueryType\":1,\"QueryTypeName\":\"A\",\"RCode\":0,\"RData\":[{\"data\":\"CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=\",\"type\":\"1\"},{\"data\":\"CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=\",\"type\":\"1\"},{\"data\":\"CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=\",\"type\":\"1\"}],\"ResolvedIPs\":[\"67.43.156.1\",\"67.43.156.2\",\"67.43.156.3\"],\"ResolverDecision\":\"allowedOnNoPolicyMatch\",\"SrcIP\":\"67.43.156.2\",\"SrcPort\":0,\"TimeZone\":\"UTC\",\"TimeZoneInferredMethod\":\"fromLocalTime\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245000000\"}", "outcome": "success", @@ -2115,9 +2115,9 @@ An example event for `gateway_http` looks as following: { "@timestamp": "2023-05-03T20:55:05.000Z", "agent": { - "ephemeral_id": "16e90d1c-6204-4d75-99a4-1b3952547172", - "id": "7871aadb-a497-48a4-a492-94f7e6e190e0", - "name": "elastic-agent-69130", + "ephemeral_id": "495cc7d6-75fe-4ee0-9344-ab1fef520b1b", + "id": "4fc671b9-3bfd-441b-982f-72e6ae663247", + "name": "elastic-agent-68727", "type": "filebeat", "version": "8.17.1" }, @@ -2192,7 +2192,7 @@ An example event for `gateway_http` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.gateway_http", - "namespace": "70767", + "namespace": "36007", "type": "logs" }, "destination": { @@ -2224,7 +2224,7 @@ An example event for `gateway_http` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "7871aadb-a497-48a4-a492-94f7e6e190e0", + "id": "4fc671b9-3bfd-441b-982f-72e6ae663247", "snapshot": false, "version": "8.17.1" }, @@ -2235,7 +2235,7 @@ An example event for `gateway_http` looks as following: "network" ], "dataset": "cloudflare_logpush.gateway_http", - "ingested": "2026-04-07T10:24:18Z", + "ingested": "2026-04-09T12:13:16Z", "kind": "event", "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a348\",\"Action\":\"block\",\"BlockedFileHash\":\"91dc1db739a705105e1c763bfdbdaa84c0de8\",\"BlockedFileName\":\"downloaded_test\",\"BlockedFileReason\":\"malware\",\"BlockedFileSize\":43,\"BlockedFileType\":\"bin\",\"Datetime\":\"2023-05-03T20:55:05Z\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"DownloadedFileNames\":[\"downloaded_file\",\"downloaded_test\"],\"Email\":\"user@example.com\",\"FileInfo\":{\"files\":[{\"name\":\"downloaded_file\",\"size\":43},{\"name\":\"downloaded_test\",\"size\":341}]},\"HTTPHost\":\"guce.yahoo.com\",\"HTTPMethod\":\"GET\",\"HTTPStatusCode\":302,\"HTTPVersion\":\"HTTP/2\",\"IsIsolated\":false,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"Block Yahoo\",\"Referer\":\"https://www.example.com/\",\"RequestID\":\"1884fec9b600007fb06a299400000001\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.123\",\"SourcePort\":47924,\"URL\":\"https://test.com\",\"UntrustedCertificateAction\":\"none\",\"UploadedFileNames\":[\"uploaded_file\",\"uploaded_test\"],\"UserAgent\":\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ @@ -2416,9 +2416,9 @@ An example event for `gateway_network` looks as following: { "@timestamp": "2023-05-18T21:12:57.058Z", "agent": { - "ephemeral_id": "84fcdf72-331c-4a7f-823e-539c5ea00ede", - "id": "39b947da-4405-47cb-9077-a8d58d0ff74b", - "name": "elastic-agent-16155", + "ephemeral_id": "391df046-6d35-4ecc-966f-6897acd12cb2", + "id": "96ff21c9-75c6-443e-851b-0a3ef8bed874", + "name": "elastic-agent-78284", "type": "filebeat", "version": "8.17.1" }, @@ -2459,7 +2459,7 @@ An example event for `gateway_network` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.gateway_network", - "namespace": "62524", + "namespace": "57876", "type": "logs" }, "destination": { @@ -2492,7 +2492,7 @@ An example event for `gateway_network` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "39b947da-4405-47cb-9077-a8d58d0ff74b", + "id": "96ff21c9-75c6-443e-851b-0a3ef8bed874", "snapshot": false, "version": "8.17.1" }, @@ -2504,7 +2504,7 @@ An example event for `gateway_network` looks as following: ], "dataset": "cloudflare_logpush.gateway_network", "id": "5f2d04be-3512-11e8-b467-0ed5f89f718b", - "ingested": "2026-04-07T10:25:18Z", + "ingested": "2026-04-09T12:14:17Z", "kind": "event", "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"Action\":\"allowedOnNoRuleMatch\",\"Datetime\":1684444377058000000,\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"Email\":\"user@test.com\",\"OverrideIP\":\"175.16.199.4\",\"OverridePort\":8080,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"My policy\",\"SNI\":\"www.elastic.co\",\"SessionID\":\"5f2d04be-3512-11e8-b467-0ed5f89f718b\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.3\",\"SourcePort\":47924,\"Transport\":\"tcp\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ @@ -2640,9 +2640,9 @@ An example event for `http_request` looks as following: { "@timestamp": "2022-05-25T13:25:26.000Z", "agent": { - "ephemeral_id": "63db57a5-406d-44bc-836c-da04e2acb85e", - "id": "1b015357-e819-4d54-9d04-e72849d9691b", - "name": "elastic-agent-32793", + "ephemeral_id": "a3407cc1-f3c0-44c6-9dac-2e4dc34ac934", + "id": "538a4934-da8a-4c15-9793-42ad5bbd4554", + "name": "elastic-agent-42454", "type": "filebeat", "version": "8.17.1" }, @@ -2830,7 +2830,7 @@ An example event for `http_request` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.http_request", - "namespace": "38039", + "namespace": "48349", "type": "logs" }, "destination": { @@ -2843,7 +2843,7 @@ An example event for `http_request` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "1b015357-e819-4d54-9d04-e72849d9691b", + "id": "538a4934-da8a-4c15-9793-42ad5bbd4554", "snapshot": false, "version": "8.17.1" }, @@ -2854,7 +2854,7 @@ An example event for `http_request` looks as following: ], "dataset": "cloudflare_logpush.http_request", "id": "710e98d9367f357d", - "ingested": "2026-04-07T10:26:19Z", + "ingested": "2026-04-09T12:15:17Z", "kind": "event", "original": "{\"BotDetectionIDs\":[7,8,9],\"BotScore\":20,\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":[\"bing\",\"api\"],\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityAction\":\"unknown\",\"SecurityLevel\":\"off\",\"SecurityRuleDescription\":\"matchad variable message\",\"SecurityRuleID\":\"98d93d5\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAttackScore\":50,\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRCEAttackScore\":1,\"WAFSQLiAttackScore\":99,\"WAFXSSAttackScore\":90,\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}", "type": [ @@ -3087,9 +3087,9 @@ An example event for `magic_ids` looks as following: { "@timestamp": "2023-09-11T03:02:57.000Z", "agent": { - "ephemeral_id": "59364f67-62d2-4044-970d-40ba6542ffdc", - "id": "33f5db7e-7e2e-4e82-8a91-c5694677f76d", - "name": "elastic-agent-64158", + "ephemeral_id": "182cd4f0-d10d-43ec-87ed-b73ec15c9c94", + "id": "f8c28412-5c22-4d91-92d9-09f412249ad4", + "name": "elastic-agent-80952", "type": "filebeat", "version": "8.17.1" }, @@ -3119,7 +3119,7 @@ An example event for `magic_ids` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.magic_ids", - "namespace": "77401", + "namespace": "98324", "type": "logs" }, "destination": { @@ -3148,7 +3148,7 @@ An example event for `magic_ids` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "33f5db7e-7e2e-4e82-8a91-c5694677f76d", + "id": "f8c28412-5c22-4d91-92d9-09f412249ad4", "snapshot": false, "version": "8.17.1" }, @@ -3160,7 +3160,7 @@ An example event for `magic_ids` looks as following: "intrusion_detection" ], "dataset": "cloudflare_logpush.magic_ids", - "ingested": "2026-04-07T10:27:20Z", + "ingested": "2026-04-09T12:16:18Z", "kind": "event", "original": "{\"Action\":\"pass\",\"ColoCity\":\"Tokyo\",\"ColoCode\":\"NRT\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":80,\"Protocol\":\"tcp\",\"SignatureID\":2031296,\"SignatureMessage\":\"ET CURRENT_EVENTS [Fireeye] POSSIBLE HackTool.TCP.Rubeus.[User32LogonProcesss]\",\"SignatureRevision\":1,\"SourceIP\":\"67.43.156.2\",\"SourcePort\":44667,\"Timestamp\":\"2023-09-11T03:02:57Z\"}", "type": [ @@ -3254,9 +3254,9 @@ An example event for `nel_report` looks as following: { "@timestamp": "2021-07-27T00:01:07.000Z", "agent": { - "ephemeral_id": "6d48242a-56b5-4837-b56e-33babdc5e374", - "id": "ecb544ed-5183-4910-8bf4-e1d26c69e8fd", - "name": "elastic-agent-98632", + "ephemeral_id": "f47b4167-4005-4e5a-91a9-6bc194922f12", + "id": "46580a00-78be-4479-86d4-fe77270b9f1d", + "name": "elastic-agent-89164", "type": "filebeat", "version": "8.17.1" }, @@ -3285,14 +3285,14 @@ An example event for `nel_report` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.nel_report", - "namespace": "90511", + "namespace": "40594", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "ecb544ed-5183-4910-8bf4-e1d26c69e8fd", + "id": "46580a00-78be-4479-86d4-fe77270b9f1d", "snapshot": false, "version": "8.17.1" }, @@ -3305,7 +3305,7 @@ An example event for `nel_report` looks as following: "network" ], "dataset": "cloudflare_logpush.nel_report", - "ingested": "2026-04-07T10:28:19Z", + "ingested": "2026-04-09T12:17:17Z", "kind": "event", "original": "{\"ClientIPASN\":\"13335\",\"ClientIPASNDescription\":\"CLOUDFLARENET\",\"ClientIPCountry\":\"US\",\"LastKnownGoodColoCode\":\"SJC\",\"Phase\":\"connection\",\"Timestamp\":\"2021-07-27T00:01:07Z\",\"Type\":\"network-error\"}", "type": [ @@ -3368,9 +3368,9 @@ An example event for `network_analytics` looks as following: { "@timestamp": "2021-07-27T00:01:07.000Z", "agent": { - "ephemeral_id": "5e8103cc-39cd-49c1-adb5-76502c3887c3", - "id": "841ed7ef-7586-4df8-980f-703f36dd251e", - "name": "elastic-agent-32455", + "ephemeral_id": "0d28a777-a74f-445a-91af-dbe3dc1a8911", + "id": "46d299f3-9ad2-490b-98c4-f04527c6974d", + "name": "elastic-agent-32051", "type": "filebeat", "version": "8.17.1" }, @@ -3535,7 +3535,7 @@ An example event for `network_analytics` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.network_analytics", - "namespace": "91714", + "namespace": "34147", "type": "logs" }, "destination": { @@ -3549,7 +3549,7 @@ An example event for `network_analytics` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "841ed7ef-7586-4df8-980f-703f36dd251e", + "id": "46d299f3-9ad2-490b-98c4-f04527c6974d", "snapshot": false, "version": "8.17.1" }, @@ -3559,7 +3559,7 @@ An example event for `network_analytics` looks as following: "network" ], "dataset": "cloudflare_logpush.network_analytics", - "ingested": "2026-04-07T10:44:19Z", + "ingested": "2026-04-09T12:45:42Z", "kind": "event", "original": "{\"AttackCampaignID\":\"xyz987\",\"AttackID\":\"abc777\",\"ColoCountry\":\"AD\",\"ColoGeoHash\":\"gbuun\",\"ColoID\":46,\"ColoName\":\"SJC\",\"Datetime\":\"2021-07-27T00:01:07Z\",\"DestinationASN\":1900,\"DestinationASNDescription\":\"asn description\",\"DestinationCountry\":\"AD\",\"DestinationGeoHash\":\"gbuun\",\"DestinationPort\":1234,\"Direction\":\"ingress\",\"GREChecksum\":10,\"GREEthertype\":10,\"GREHeaderLength\":1024,\"GREKey\":10,\"GRESequenceNumber\":10,\"GREVersion\":10,\"ICMPChecksum\":10,\"ICMPCode\":10,\"ICMPType\":10,\"IPDestinationAddress\":\"175.16.199.0\",\"IPDestinationSubnet\":\"/24\",\"IPFragmentOffset\":1480,\"IPHeaderLength\":20,\"IPMoreFragments\":1480,\"IPProtocol\":6,\"IPProtocolName\":\"tcp\",\"IPSourceAddress\":\"67.43.156.0\",\"IPSourceSubnet\":\"/24\",\"IPTotalLength\":1024,\"IPTotalLengthBuckets\":10,\"IPTtl\":240,\"IPTtlBuckets\":2,\"IPv4Checksum\":0,\"IPv4DontFragment\":0,\"IPv4Dscp\":46,\"IPv4Ecn\":1,\"IPv4Identification\":1,\"IPv4Options\":1,\"IPv6Dscp\":46,\"IPv6Ecn\":1,\"IPv6ExtensionHeaders\":\"header\",\"IPv6FlowLabel\":1,\"IPv6Identification\":1,\"MitigationReason\":\"BLOCKED\",\"MitigationScope\":\"local\",\"MitigationSystem\":\"flowtrackd\",\"Outcome\":\"pass\",\"ProtocolState\":\"OPEN\",\"RuleID\":\"rule1\",\"RulesetID\":\"3b64149bfa6e4220bbbc2bd6db589552\",\"RulesetOverrideID\":\"id1\",\"SampleInterval\":1,\"SourceASN\":1500,\"SourceASNDescription\":\"Source ASN Description\",\"SourceCountry\":\"AD\",\"SourceGeoHash\":\"gbuun\",\"SourcePort\":5678,\"TCPAcknowledgementNumber\":1000,\"TCPChecksum\":10,\"TCPDataOffset\":0,\"TCPFlags\":1,\"TCPFlagsString\":\"Human-readable flags string\",\"TCPMss\":512,\"TCPOptions\":\"mss\",\"TCPSackBlocks\":1,\"TCPSacksPermitted\":1,\"TCPSequenceNumber\":100,\"TCPTimestampEcr\":100,\"TCPTimestampValue\":100,\"TCPUrgentPointer\":10,\"TCPWindowScale\":10,\"TCPWindowSize\":10,\"UDPChecksum\":10,\"UDPPayloadLength\":10,\"Verdict\":\"pass\"}", "outcome": "success", @@ -3743,9 +3743,9 @@ An example event for `network_session` looks as following: { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "2944bb96-75f0-470e-b5e7-15a6cbce31e6", - "id": "5f315a59-b129-4ec3-9025-50eac9ca468f", - "name": "elastic-agent-59563", + "ephemeral_id": "d785638d-aca3-4b02-b48d-7b37080e572e", + "id": "3d3e0b91-99a0-45b6-a856-fbe5ecc636f8", + "name": "elastic-agent-24801", "type": "filebeat", "version": "8.17.1" }, @@ -3827,7 +3827,7 @@ An example event for `network_session` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.network_session", - "namespace": "16918", + "namespace": "34281", "type": "logs" }, "destination": { @@ -3860,7 +3860,7 @@ An example event for `network_session` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "5f315a59-b129-4ec3-9025-50eac9ca468f", + "id": "3d3e0b91-99a0-45b6-a856-fbe5ecc636f8", "snapshot": false, "version": "8.17.1" }, @@ -3874,7 +3874,7 @@ An example event for `network_session` looks as following: "duration": 0, "end": "2023-05-04T11:29:14.000Z", "id": "18881f179300007fb0d06d6400000001", - "ingested": "2026-04-07T10:30:21Z", + "ingested": "2026-04-09T12:46:40Z", "kind": "event", "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"BytesReceived\":679,\"BytesSent\":2333,\"ClientTCPHandshakeDurationMs\":12,\"ClientTLSCipher\":\"TLS_AES_128_GCM_SHA256\",\"ClientTLSHandshakeDurationMs\":125,\"ClientTLSVersion\":\"TLS 1.3\",\"ConnectionCloseReason\":\"CLIENT_CLOSED\",\"ConnectionReuse\":false,\"DestinationTunnelID\":\"00000000-0000-0000-0000-000000000000\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"EgressColoName\":\"ORD\",\"EgressIP\":\"2a02:cf40::23\",\"EgressPort\":41052,\"EgressRuleID\":\"00000000-0000-0000-0000-000000000000\",\"EgressRuleName\":\"Egress Rule 1\",\"Email\":\"user@test.com\",\"IngressColoName\":\"ORD\",\"Offramp\":\"INTERNET\",\"OriginIP\":\"89.160.20.129\",\"OriginPort\":80,\"OriginTLSCertificateIssuer\":\"DigiCert Inc\",\"OriginTLSCertificateValidationResult\":\"VALID\",\"OriginTLSCipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"OriginTLSHandshakeDurationMs\":130,\"OriginTLSVersion\":\"TLS 1.2\",\"Protocol\":\"TCP\",\"RuleEvaluationDurationMs\":10,\"SessionEndTime\":\"2023-05-04T11:29:14Z\",\"SessionID\":\"18881f179300007fb0d06d6400000001\",\"SessionStartTime\":\"2023-05-04T11:29:14Z\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"1.128.0.1\",\"SourcePort\":52994,\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\",\"VirtualNetworkID\":\"0ce99869-63d3-4d5d-bdaf-d4f33df964aa\"}", "start": "2023-05-04T11:29:14.000Z", @@ -4027,9 +4027,9 @@ An example event for `page_shield_events` looks as following: { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "ca737b17-cac2-420e-9d87-a75a0ba2f486", - "id": "e5093573-263c-47e2-a874-2d65a7a582ff", - "name": "elastic-agent-41800", + "ephemeral_id": "08459ce8-c654-46ff-b461-08814b63ccb4", + "id": "a0cd7b50-1ec2-4305-bc0c-fca2957c4807", + "name": "elastic-agent-76009", "type": "filebeat", "version": "8.17.1" }, @@ -4049,14 +4049,14 @@ An example event for `page_shield_events` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.page_shield_events", - "namespace": "91602", + "namespace": "64890", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "e5093573-263c-47e2-a874-2d65a7a582ff", + "id": "a0cd7b50-1ec2-4305-bc0c-fca2957c4807", "snapshot": false, "version": "8.17.1" }, @@ -4067,7 +4067,7 @@ An example event for `page_shield_events` looks as following: "network" ], "dataset": "cloudflare_logpush.page_shield_events", - "ingested": "2026-04-07T10:31:19Z", + "ingested": "2026-04-09T12:47:37Z", "kind": "event", "original": "{\"Action\":\"log\",\"CSPDirective\":\"directive\",\"Host\":\"hostymchost.face\",\"PageURL\":\"http://example.com/?query=42\",\"PolicyID\":\"9\",\"ResourceType\":\"other\",\"Timestamp\":\"2023-05-04T11:29:14Z\",\"URL\":\"https://example.com/?query=hog\",\"URLContainsCDNCGIPath\":true,\"URLHost\":\"example.com\"}", "type": [ @@ -4150,9 +4150,9 @@ An example event for `sinkhole_http` looks as following: { "@timestamp": "2023-09-19T12:00:00.000Z", "agent": { - "ephemeral_id": "6db1a86d-5bdd-4757-b7ec-ef46ba70549e", - "id": "334188ef-543b-4b19-a898-ff614abf72dc", - "name": "elastic-agent-88294", + "ephemeral_id": "19317b78-9e9c-4e24-ac35-c2433bb62b3e", + "id": "60337292-fcdb-4c92-89b4-f0902b077688", + "name": "elastic-agent-61800", "type": "filebeat", "version": "8.17.1" }, @@ -4195,7 +4195,7 @@ An example event for `sinkhole_http` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.sinkhole_http", - "namespace": "31136", + "namespace": "11606", "type": "logs" }, "destination": { @@ -4223,7 +4223,7 @@ An example event for `sinkhole_http` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "334188ef-543b-4b19-a898-ff614abf72dc", + "id": "60337292-fcdb-4c92-89b4-f0902b077688", "snapshot": false, "version": "8.17.1" }, @@ -4233,7 +4233,7 @@ An example event for `sinkhole_http` looks as following: "network" ], "dataset": "cloudflare_logpush.sinkhole_http", - "ingested": "2026-04-07T10:32:21Z", + "ingested": "2026-04-09T12:48:38Z", "kind": "event", "original": "{\"AccountID\":\"AC123456\",\"Body\":\"{\\\"action\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\"}\",\"BodyLength\":39,\"DestAddr\":\"89.160.20.129\",\"Headers\":\"Host: example.com\\nUser-Agent: Mozilla/5.0\\nAccept: */*\\nConnection: keep-alive\",\"Host\":\"example.com\",\"Method\":\"POST\",\"Password\":\"password123\",\"R2Path\":\"\",\"Referrer\":\"https://searchengine.com/\",\"SinkholeID\":\"SH001\",\"SrcAddr\":\"67.43.156.2\",\"Timestamp\":\"2023-09-19T12:00:00Z\",\"URI\":\"/api/v1/login\",\"URL\":\"https://example.com/api/v1/login\",\"UserAgent\":\"Mozilla/5.0\",\"Username\":\"john_doe\"}", "type": [ @@ -4362,9 +4362,9 @@ An example event for `spectrum_event` looks as following: { "@timestamp": "2022-05-26T09:24:00.000Z", "agent": { - "ephemeral_id": "e7abae94-0044-4900-9c4b-48c2505ae84c", - "id": "2738c1bb-9aa7-4f2f-a704-fb666fbd0dc1", - "name": "elastic-agent-87759", + "ephemeral_id": "6c15f29d-826a-465a-ae6d-4944d6ae3294", + "id": "ee31129a-42a2-4a73-b55f-27553245fac9", + "name": "elastic-agent-45871", "type": "filebeat", "version": "8.17.1" }, @@ -4421,7 +4421,7 @@ An example event for `spectrum_event` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.spectrum_event", - "namespace": "32034", + "namespace": "72798", "type": "logs" }, "destination": { @@ -4433,7 +4433,7 @@ An example event for `spectrum_event` looks as following: "version": "9.3.0" }, "elastic_agent": { - "id": "2738c1bb-9aa7-4f2f-a704-fb666fbd0dc1", + "id": "ee31129a-42a2-4a73-b55f-27553245fac9", "snapshot": false, "version": "8.17.1" }, @@ -4447,7 +4447,7 @@ An example event for `spectrum_event` looks as following: "duration": -1653557040000000000, "end": "1970-01-01T00:00:00.000Z", "id": "7ef659a2f8ef4810a9bade96fdad7c75", - "ingested": "2026-04-07T10:33:21Z", + "ingested": "2026-04-09T12:49:39Z", "kind": "event", "original": "{\"Application\":\"7ef659a2f8ef4810a9bade96fdad7c75\",\"ClientAsn\":200391,\"ClientBytes\":0,\"ClientCountry\":\"bg\",\"ClientIP\":\"67.43.156.0\",\"ClientMatchedIpFirewall\":\"UNKNOWN\",\"ClientPort\":40456,\"ClientProto\":\"tcp\",\"ClientTcpRtt\":0,\"ClientTlsCipher\":\"UNK\",\"ClientTlsClientHelloServerName\":\"server name\",\"ClientTlsProtocol\":\"unknown\",\"ClientTlsStatus\":\"UNKNOWN\",\"ColoCode\":\"SOF\",\"ConnectTimestamp\":\"2022-05-26T09:24:00Z\",\"DisconnectTimestamp\":\"1970-01-01T00:00:00Z\",\"Event\":\"connect\",\"IpFirewall\":false,\"OriginBytes\":0,\"OriginIP\":\"175.16.199.0\",\"OriginPort\":3389,\"OriginProto\":\"tcp\",\"OriginTcpRtt\":0,\"OriginTlsCipher\":\"UNK\",\"OriginTlsFingerprint\":\"0000000000000000000000000000000000000000000000000000000000000000.\",\"OriginTlsMode\":\"off\",\"OriginTlsProtocol\":\"unknown\",\"OriginTlsStatus\":\"UNKNOWN\",\"ProxyProtocol\":\"off\",\"Status\":0,\"Timestamp\":\"2022-05-26T09:24:00Z\"}", "start": "2022-05-26T09:24:00.000Z", @@ -4567,9 +4567,9 @@ An example event for `workers_trace` looks as following: { "@timestamp": "2023-07-20T11:35:46.804Z", "agent": { - "ephemeral_id": "a01887ca-67a7-4a33-819e-cc0cb6df6fbb", - "id": "6016b130-b61b-4830-aef9-3d270a4d8f09", - "name": "elastic-agent-70540", + "ephemeral_id": "ad4fc167-31f0-412e-b1ee-d96d6dfb5f0f", + "id": "5cf9216c-17ec-48fc-b595-875f0002f34e", + "name": "elastic-agent-46737", "type": "filebeat", "version": "8.17.1" }, @@ -4616,14 +4616,14 @@ An example event for `workers_trace` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.workers_trace", - "namespace": "66050", + "namespace": "96709", "type": "logs" }, "ecs": { "version": "9.3.0" }, "elastic_agent": { - "id": "6016b130-b61b-4830-aef9-3d270a4d8f09", + "id": "5cf9216c-17ec-48fc-b595-875f0002f34e", "snapshot": false, "version": "8.17.1" }, @@ -4635,7 +4635,7 @@ An example event for `workers_trace` looks as following: ], "dataset": "cloudflare_logpush.workers_trace", "id": "7e9ae7157ac0c33a", - "ingested": "2026-04-07T10:34:20Z", + "ingested": "2026-04-09T12:50:39Z", "kind": "event", "original": "{\"DispatchNamespace\":\"my-worker-dispatch\",\"Event\":{\"RayID\":\"7e9ae7157ac0c33a\",\"Request\":{\"Method\":\"GET\",\"URL\":\"http://chat-gpt-little-butterfly-0c3d.example.workers.dev/v2/_catalog\"},\"Response\":{\"Status\":404}},\"EventTimestampMs\":1689852946804,\"EventType\":\"fetch\",\"Exceptions\":[{\"Message\":\"Uncaught TypeError: Cannot read property 'x' of undefined\",\"Stack\":\"TypeError: Cannot read property 'x' of undefined\\n at fetchHandler (/workers/script.js:12:27)\\n at handleRequest (/workers/script.js:6:13)\"}],\"Logs\":[{\"level\":\"info\",\"message\":\"Request received for /api/data\"},{\"level\":\"error\",\"message\":\"Something went wrong\"}],\"Outcome\":\"exception\",\"ScriptName\":\"chat-gpt-little-butterfly-0c3d\",\"ScriptTags\":[\"api\",\"chatgpt\"]}", "outcome": "failure",