diff --git a/packages/azure_frontdoor/_dev/build/docs/README.md b/packages/azure_frontdoor/_dev/build/docs/README.md index 8fb57e8ac9f..3498714e9d1 100644 --- a/packages/azure_frontdoor/_dev/build/docs/README.md +++ b/packages/azure_frontdoor/_dev/build/docs/README.md @@ -11,15 +11,17 @@ The Azure Frontdoor logs integration retrieves the following types of log data f - **Access Logs**: Logs categorized as `FrontDoorAccessLog`. - **Web Application Firewall (WAF) Logs**: Logs categorized as `FrontDoorWebApplicationFirewallLog`. +- **Health Probe Logs**: Logs categorized as `FrontDoorHealthProbeLog`. -Currently, the integration does not support **Activity Logs** or **Health Probe logs**. +Currently, the integration does not support **Activity Logs**. ## Data streams -This integration collects two types of data streams: +This integration collects three types of data streams: - access log - waf logs +- health probe logs ## Requirements @@ -141,3 +143,7 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th ## WAF Logs {{fields "waf"}} + +## Health Probe Logs + +{{fields "health_probe"}} \ No newline at end of file diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml index 2b6841c3bbf..226ddfd3320 100644 --- a/packages/azure_frontdoor/changelog.yml +++ b/packages/azure_frontdoor/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.4.0" + changes: + - description: Add Health Probe dataset + type: enhancement + link: https://github.com/elastic/integrations/pull/20016 - version: "2.3.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..95f3801cbb1 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,4 @@ +fields: + tags: + - preserve_original_event + - azure-frontdoor-health-probe diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log new file mode 100644 index 00000000000..10af0358072 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log @@ -0,0 +1,3 @@ +{"time":"2026-07-07T09:31:41Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"259ffc3865154b8d8ca2c82692b60cf1","pop":"BL","httpVerb":"HEAD","result":"OriginError","httpStatusCode":"404","probeUrl":"http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/","originName":"test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io","originIP":"216.160.83.56","totalLatencyMilliseconds":"142","connectionLatencyMilliseconds":"71","DNSLatencyMicroseconds":"2000"}} +{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"216.160.83.56:443","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"n/a"}} +{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"[2001:db8::1]:8080","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"n/a"}} diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json new file mode 100644 index 00000000000..ba6f5b8ecb5 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json @@ -0,0 +1,239 @@ +{ + "expected": [ + { + "@timestamp": "2026-07-07T09:31:41.000Z", + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 71, + "dns_latency_microseconds": 2000, + "health_probe_id": "259ffc3865154b8d8ca2c82692b60cf1", + "origin_name": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "pop": "BL", + "result": "OriginError", + "total_latency_milliseconds": 142 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "destination": { + "address": "216.160.83.56", + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.56" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "original": "{\"time\":\"2026-07-07T09:31:41Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"259ffc3865154b8d8ca2c82692b60cf1\",\"pop\":\"BL\",\"httpVerb\":\"HEAD\",\"result\":\"OriginError\",\"httpStatusCode\":\"404\",\"probeUrl\":\"http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/\",\"originName\":\"test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io\",\"originIP\":\"216.160.83.56\",\"totalLatencyMilliseconds\":\"142\",\"connectionLatencyMilliseconds\":\"71\",\"DNSLatencyMicroseconds\":\"2000\"}}", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "HEAD" + }, + "response": { + "status_code": 404 + } + }, + "related": { + "ip": [ + "216.160.83.56" + ] + }, + "tags": [ + "preserve_original_event", + "azure-frontdoor-health-probe" + ], + "url": { + "domain": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "full": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "original": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "path": "/", + "port": 80, + "scheme": "http" + } + }, + { + "@timestamp": "2026-07-07T10:00:00.000Z", + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 10, + "health_probe_id": "abc123def456", + "origin_name": "backend.example.com", + "pop": "AMS", + "result": "Success", + "total_latency_milliseconds": 25 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "destination": { + "address": "216.160.83.56:443", + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.56", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"216.160.83.56:443\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"n/a\"}}", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "status_code": 200 + } + }, + "related": { + "ip": [ + "216.160.83.56" + ] + }, + "tags": [ + "preserve_original_event", + "azure-frontdoor-health-probe" + ], + "url": { + "domain": "backend.example.com", + "full": "https://backend.example.com/health", + "original": "https://backend.example.com/health", + "path": "/health", + "scheme": "https" + } + }, + { + "@timestamp": "2026-07-07T10:00:00.000Z", + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 10, + "health_probe_id": "abc123def456", + "origin_name": "backend.example.com", + "pop": "AMS", + "result": "Success", + "total_latency_milliseconds": 25 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "destination": { + "address": "[2001:db8::1]:8080", + "as": { + "number": 65551, + "organization": { + "name": "Documentation ASN" + } + }, + "geo": { + "city_name": "Greenwich", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.47687, + "lon": -4.1E-4 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "2001:db8::1", + "port": 8080 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"[2001:db8::1]:8080\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"n/a\"}}", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "status_code": 200 + } + }, + "related": { + "ip": [ + "2001:db8::1" + ] + }, + "tags": [ + "preserve_original_event", + "azure-frontdoor-health-probe" + ], + "url": { + "domain": "backend.example.com", + "full": "https://backend.example.com/health", + "original": "https://backend.example.com/health", + "path": "/health", + "scheme": "https" + } + } + ] +} diff --git a/packages/azure_frontdoor/data_stream/health_probe/agent/stream/azure-eventhub.yml.hbs b/packages/azure_frontdoor/data_stream/health_probe/agent/stream/azure-eventhub.yml.hbs new file mode 100644 index 00000000000..fb00f02724c --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/agent/stream/azure-eventhub.yml.hbs @@ -0,0 +1,59 @@ +{{#if auth_type}} +auth_type: {{auth_type}} +{{/if}} +{{#if connection_string}} +connection_string: {{connection_string}} +{{/if}} +{{#if eventhub_namespace}} +eventhub_namespace: {{eventhub_namespace}} +{{/if}} +{{#if tenant_id}} +tenant_id: {{tenant_id}} +{{/if}} +{{#if client_id}} +client_id: {{client_id}} +{{/if}} +{{#if client_secret}} +client_secret: {{client_secret}} +{{/if}} +{{#contains "client_secret" auth_type}} +{{#if authority_host}} +authority_host: {{authority_host}} +{{/if}} +{{/contains}} +{{#if storage_account_container }} +storage_account_container: {{storage_account_container}} +{{else}} +{{#if eventhub}} +storage_account_container: frontdoor-health-probe-{{eventhub}} +{{/if}} +{{/if}} +{{#if eventhub}} +eventhub: {{eventhub}} +{{/if}} +{{#if consumer_group}} +consumer_group: {{consumer_group}} +{{/if}} +{{#if storage_account}} +storage_account: {{storage_account}} +{{/if}} +{{#if storage_account_key}} +storage_account_key: {{storage_account_key}} +{{/if}} +{{#if resource_manager_endpoint}} +resource_manager_endpoint: {{resource_manager_endpoint}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..a160e58dcf2 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,235 @@ +--- +description: Pipeline for processing azure frontdoor health probe logs +processors: + - set: + field: ecs.version + value: "8.11.0" + - set: + field: event.category + value: [network] + - set: + field: event.type + value: [connection] + - rename: + field: message + target_field: event.original + ignore_missing: true + if: ctx.event?.original == null + - set: + field: cloud.provider + value: azure + - drop: + description: Drop if invalid json + if: ctx.event?.original != null && ctx.event.original.contains('"records"') + - json: + field: event.original + target_field: azure.frontdoor.health_probe + - script: + description: Drops fields with empty or N/A values recursively + tag: script_to_drop_invalid_values + lang: painless + source: | + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || v == 'N/A' || v == 'n/a' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || v == 'N/A' || v == 'n/a' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - rename: + field: azure.frontdoor.health_probe.resourceId + target_field: azure.frontdoor.resource_id + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.operationName + target_field: azure.frontdoor.operation_name + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.category + target_field: azure.frontdoor.category + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.healthProbeId + target_field: azure.frontdoor.health_probe.health_probe_id + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.pop + target_field: azure.frontdoor.health_probe.pop + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.httpVerb + target_field: http.request.method + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.result + target_field: azure.frontdoor.health_probe.result + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.httpStatusCode + target_field: http.response.status_code + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.probeUrl + target_field: url.original + ignore_missing: true + - uri_parts: + field: url.original + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: url.full + copy_from: url.original + ignore_empty_value: true + - rename: + field: azure.frontdoor.health_probe.properties.originName + target_field: azure.frontdoor.health_probe.origin_name + ignore_missing: true + - convert: + field: azure.frontdoor.health_probe.properties.totalLatencyMilliseconds + target_field: azure.frontdoor.health_probe.total_latency_milliseconds + type: long + tag: convert_total_latency + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.health_probe.properties.totalLatencyMilliseconds + ignore_missing: true + - convert: + field: azure.frontdoor.health_probe.properties.connectionLatencyMilliseconds + target_field: azure.frontdoor.health_probe.connection_latency_milliseconds + type: long + tag: convert_connection_latency + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.health_probe.properties.connectionLatencyMilliseconds + ignore_missing: true + - convert: + field: azure.frontdoor.health_probe.properties.DNSLatencyMicroseconds + target_field: azure.frontdoor.health_probe.dns_latency_microseconds + type: long + tag: convert_dns_latency + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.health_probe.properties.DNSLatencyMicroseconds + ignore_missing: true + - convert: + field: http.response.status_code + type: long + ignore_missing: true + - date: + field: azure.frontdoor.health_probe.time + target_field: "@timestamp" + formats: + - ISO8601 + - rename: + field: azure.frontdoor.health_probe.properties.originIP + target_field: destination.address + ignore_missing: true + - remove: + field: + - azure.frontdoor.health_probe.time + - azure.frontdoor.health_probe.properties + ignore_missing: true + - convert: + description: Convert destination.address to ip type, if it contains a port, extract the IP part + tag: convert_destination_address + field: destination.address + target_field: destination.ip + type: ip + if: ctx.destination?.address != null + on_failure: + - grok: + description: Extract destination.port from destination.address if it contains a port + tag: grok_destination_port + field: destination.address + patterns: + - '%{IPV4:destination.ip}:%{NUMBER:destination.port:long}' + - '\[%{IPV6:destination.ip}\]:%{NUMBER:destination.port:long}' + ignore_missing: true + if: ctx.destination?.address != null && ctx.destination.address.contains(':') + on_failure: + - append: + tag: append_error_message_1e8f0b9e + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + if: ctx.destination?.ip != null + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + if: ctx.destination?.ip != null + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + - append: + field: related.ip + value: '{{{destination.ip}}}' + allow_duplicates: false + if: ctx.destination?.ip != null + + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + field: tags + tag: append_tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/azure_frontdoor/data_stream/health_probe/fields/agent.yml b/packages/azure_frontdoor/data_stream/health_probe/fields/agent.yml new file mode 100644 index 00000000000..4b15225a4d4 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/fields/agent.yml @@ -0,0 +1,12 @@ +- name: cloud.image.id + type: keyword + description: Image ID for the cloud instance. +- name: host.containerized + type: boolean + description: If the host is a container. +- name: host.os.build + type: keyword + description: OS build information. +- name: host.os.codename + type: keyword + description: OS codename, if any. diff --git a/packages/azure_frontdoor/data_stream/health_probe/fields/base-fields.yml b/packages/azure_frontdoor/data_stream/health_probe/fields/base-fields.yml new file mode 100644 index 00000000000..208aa91eeb8 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/fields/base-fields.yml @@ -0,0 +1,18 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: input.type + type: keyword + description: Input type. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml b/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml new file mode 100644 index 00000000000..7bbbb215dcc --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml @@ -0,0 +1,36 @@ +- name: azure.frontdoor + type: group + fields: + - name: resource_id + type: keyword + description: Azure Resource ID. + - name: category + type: keyword + description: Azure Frontdoor category name. + - name: operation_name + type: keyword + description: Azure operation name. + - name: health_probe + type: group + fields: + - name: health_probe_id + type: keyword + description: Unique identifier for the health probe request. + - name: pop + type: keyword + description: The edge PoP (Point of Presence) that issued the health probe. + - name: result + type: keyword + description: "Result of the health probe (examples: OriginError, Success)." + - name: origin_name + type: keyword + description: The hostname of the origin that was probed. + - name: total_latency_milliseconds + type: long + description: Total latency of the health probe in milliseconds. + - name: connection_latency_milliseconds + type: long + description: TCP connection latency of the health probe in milliseconds. + - name: dns_latency_microseconds + type: long + description: DNS resolution latency of the health probe in microseconds. diff --git a/packages/azure_frontdoor/data_stream/health_probe/manifest.yml b/packages/azure_frontdoor/data_stream/health_probe/manifest.yml new file mode 100644 index 00000000000..28d452ceacc --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/manifest.yml @@ -0,0 +1,40 @@ +title: "FrontDoor Health Probe" +type: logs +streams: + - input: "azure-eventhub" + template_path: "azure-eventhub.yml.hbs" + title: "Azure Frontdoor Health Probe logs" + description: "Collect Azure Frontdoor health probe logs using azure-eventhub input" + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - azure-frontdoor-health-probe + - forwarded + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: storage_account_container + type: text + title: Storage Account Container + multi: false + required: false + show_user: false + description: > + The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified. + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. " diff --git a/packages/azure_frontdoor/data_stream/health_probe/sample_event.json b/packages/azure_frontdoor/data_stream/health_probe/sample_event.json new file mode 100644 index 00000000000..da7ffe2967b --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/sample_event.json @@ -0,0 +1,100 @@ +{ + "@timestamp": "2026-07-07T09:31:41.000Z", + "agent": { + "ephemeral_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "id": "d7426e8a-1535-4d9a-8f1e-1d5eab23567b", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.6.0" + }, + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 71, + "dns_latency_microseconds": 2000, + "health_probe_id": "259ffc3865154b8d8ca2c82692b60cf1", + "origin_name": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "pop": "BL", + "result": "OriginError", + "total_latency_milliseconds": 142 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "data_stream": { + "dataset": "azure_frontdoor.health_probe", + "namespace": "ep", + "type": "logs" + }, + "destination": { + "address": "216.160.83.56", + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.56" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "d7426e8a-1535-4d9a-8f1e-1d5eab23567b", + "snapshot": false, + "version": "8.6.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "network" + ], + "dataset": "azure_frontdoor.health_probe", + "ingested": "2026-07-07T09:32:00Z", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "HEAD" + }, + "response": { + "status_code": 404 + } + }, + "input": { + "type": "azure-eventhub" + }, + "related": { + "ip": [ + "4.246.53.113" + ] + }, + "tags": [ + "azure-frontdoor-health-probe", + "forwarded" + ], + "url": { + "domain": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "full": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "original": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "path": "/", + "port": 80, + "scheme": "http" + } +} \ No newline at end of file diff --git a/packages/azure_frontdoor/docs/README.md b/packages/azure_frontdoor/docs/README.md index 756bdfde5a8..430836efd58 100644 --- a/packages/azure_frontdoor/docs/README.md +++ b/packages/azure_frontdoor/docs/README.md @@ -11,15 +11,17 @@ The Azure Frontdoor logs integration retrieves the following types of log data f - **Access Logs**: Logs categorized as `FrontDoorAccessLog`. - **Web Application Firewall (WAF) Logs**: Logs categorized as `FrontDoorWebApplicationFirewallLog`. +- **Health Probe Logs**: Logs categorized as `FrontDoorHealthProbeLog`. -Currently, the integration does not support **Activity Logs** or **Health Probe logs**. +Currently, the integration does not support **Activity Logs**. ## Data streams -This integration collects two types of data streams: +This integration collects three types of data streams: - access log - waf logs +- health probe logs ## Requirements @@ -220,3 +222,30 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th | input.type | Input type. | keyword | | log.offset | Log offset. | long | + +## Health Probe Logs + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| azure.frontdoor.category | Azure Frontdoor category name. | keyword | +| azure.frontdoor.health_probe.connection_latency_milliseconds | TCP connection latency of the health probe in milliseconds. | long | +| azure.frontdoor.health_probe.dns_latency_microseconds | DNS resolution latency of the health probe in microseconds. | long | +| azure.frontdoor.health_probe.health_probe_id | Unique identifier for the health probe request. | keyword | +| azure.frontdoor.health_probe.origin_name | The hostname of the origin that was probed. | keyword | +| azure.frontdoor.health_probe.pop | The edge PoP (Point of Presence) that issued the health probe. | keyword | +| azure.frontdoor.health_probe.result | Result of the health probe (examples: OriginError, Success). | keyword | +| azure.frontdoor.health_probe.total_latency_milliseconds | Total latency of the health probe in milliseconds. | long | +| azure.frontdoor.operation_name | Azure operation name. | keyword | +| azure.frontdoor.resource_id | Azure Resource ID. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| host.containerized | If the host is a container. | boolean | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| input.type | Input type. | keyword | +| log.offset | Log offset. | long | diff --git a/packages/azure_frontdoor/manifest.yml b/packages/azure_frontdoor/manifest.yml index 5e36842e958..e914e9cbffd 100644 --- a/packages/azure_frontdoor/manifest.yml +++ b/packages/azure_frontdoor/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: azure_frontdoor title: "Azure Frontdoor" -version: "2.3.0" +version: "2.4.0" description: "This Elastic integration collects logs from Azure Frontdoor." type: integration categories: