From b4c2a0fb72374b696daa87448a37d44fc1b9d7af Mon Sep 17 00:00:00 2001 From: "forkline-dev[bot]" Date: Thu, 5 Mar 2026 22:41:35 +0000 Subject: [PATCH 1/7] fix: move dashboard to templates directory The dashboard ConfigMap was placed in helm/dashboards/ which is not rendered by Helm. Move it to helm/templates/ so it gets rendered correctly when metrics.grafanaDashboard.enabled is true. Resolves: #55 --- helm/{dashboards => templates}/dashboard.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename helm/{dashboards => templates}/dashboard.yaml (100%) diff --git a/helm/dashboards/dashboard.yaml b/helm/templates/dashboard.yaml similarity index 100% rename from helm/dashboards/dashboard.yaml rename to helm/templates/dashboard.yaml From abeef3dae2df103eeba0e39323534bd6ba04a4b9 Mon Sep 17 00:00:00 2001 From: "forkline-dev[bot]" Date: Thu, 5 Mar 2026 22:41:42 +0000 Subject: [PATCH 2/7] feat: add Helm unittests for dashboard and prometheus rules Add comprehensive unittests to ensure: - Dashboard renders correctly when enabled - Prometheus rules render correctly when enabled - All expected metrics are present in templates - ServiceMonitor and Service templates work correctly This prevents future regressions where templates may not render or reference non-existent metrics. Resolves: #55 --- .github/workflows/tests.yaml | 9 +++ helm/tests/dashboard_test.yaml | 96 ++++++++++++++++++++++++++++ helm/tests/prometheusrules_test.yaml | 81 +++++++++++++++++++++++ helm/tests/service_test.yaml | 44 +++++++++++++ helm/tests/servicemonitor_test.yaml | 42 ++++++++++++ 5 files changed, 272 insertions(+) create mode 100644 helm/tests/dashboard_test.yaml create mode 100644 helm/tests/prometheusrules_test.yaml create mode 100644 helm/tests/service_test.yaml create mode 100644 helm/tests/servicemonitor_test.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 061155a..2f16b8d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -14,3 +14,12 @@ jobs: override: true components: rustfmt, clippy - uses: pre-commit/action@v3.0.1 + + helm-unittest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Helm unittest plugin + run: helm plugin install https://github.com/helm-unittest/helm-unittest.git + - name: Run Helm unittests + run: helm unittest helm diff --git a/helm/tests/dashboard_test.yaml b/helm/tests/dashboard_test.yaml new file mode 100644 index 0000000..c13ef86 --- /dev/null +++ b/helm/tests/dashboard_test.yaml @@ -0,0 +1,96 @@ +suite: Dashboard +templates: + - dashboard.yaml +tests: + - it: should not render dashboard by default + set: + metrics.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should not render dashboard when metrics disabled + set: + metrics.enabled: false + metrics.grafanaDashboard.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should render dashboard when both enabled + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: RELEASE-NAME-robotlb-dashboard + - isNotNull: + path: data["robotlb-dashboard.json"] + + - it: should include correct labels + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - equal: + path: metadata.labels["grafana_dashboard"] + value: "1" + - equal: + path: metadata.labels["app.kubernetes.io/name"] + value: robotlb + + - it: should include custom labels when provided + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + metrics.grafanaDashboard.labels: + custom_label: custom_value + another_label: another_value + asserts: + - equal: + path: metadata.labels["custom_label"] + value: custom_value + - equal: + path: metadata.labels["another_label"] + value: another_value + + - it: should contain all expected metrics in dashboard + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'robotlb_services_managed' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'robotlb_leader_status' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'robotlb_reconcile_duration_seconds' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'robotlb_reconcile_operations_total' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'robotlb_reconcile_failures_total' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'robotlb_hcloud_api_requests_total' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'robotlb_hcloud_api_errors_total' + + - it: should use correct controller label in queries + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'controller=\\"robotlb\\"' \ No newline at end of file diff --git a/helm/tests/prometheusrules_test.yaml b/helm/tests/prometheusrules_test.yaml new file mode 100644 index 0000000..760020a --- /dev/null +++ b/helm/tests/prometheusrules_test.yaml @@ -0,0 +1,81 @@ +suite: Prometheus Rules +templates: + - prometheusrules.yaml +tests: + - it: should not render prometheus rules by default + set: + metrics.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should not render prometheus rules when metrics disabled + set: + metrics.enabled: false + metrics.prometheusRules.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should render prometheus rules when both enabled + set: + metrics.enabled: true + metrics.prometheusRules.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: PrometheusRule + - equal: + path: metadata.name + value: RELEASE-NAME-robotlb + + - it: should include all expected alerts + set: + metrics.enabled: true + metrics.prometheusRules.enabled: true + asserts: + - equal: + path: spec.groups[0].name + value: robotlb.rules + - lengthEqual: + path: spec.groups[0].rules + count: 5 + + - it: should include correct metrics in alerts + set: + metrics.enabled: true + metrics.prometheusRules.enabled: true + documentSelector: + path: kind + value: PrometheusRule + asserts: + - matchRegex: + path: spec.groups[0].rules[?(@.alert == "RobotLBReconcileErrors")].expr + pattern: 'robotlb_reconcile_failures_total' + - matchRegex: + path: spec.groups[0].rules[?(@.alert == "RobotLBHighReconcileErrorRate")].expr + pattern: 'robotlb_reconcile_failures_total' + - matchRegex: + path: spec.groups[0].rules[?(@.alert == "RobotLBHighReconcileErrorRate")].expr + pattern: 'robotlb_reconcile_operations_total' + - matchRegex: + path: spec.groups[0].rules[?(@.alert == "RobotLBNotLeader")].expr + pattern: 'robotlb_leader_status' + - matchRegex: + path: spec.groups[0].rules[?(@.alert == "RobotLBHCloudAPIErrors")].expr + pattern: 'robotlb_hcloud_api_errors_total' + - matchRegex: + path: spec.groups[0].rules[?(@.alert == "RobotLBSlowReconcile")].expr + pattern: 'robotlb_reconcile_duration_seconds' + + - it: should include custom labels when provided + set: + metrics.enabled: true + metrics.prometheusRules.enabled: true + metrics.prometheusRules.labels: + custom_label: custom_value + asserts: + - equal: + path: metadata.labels["custom_label"] + value: custom_value \ No newline at end of file diff --git a/helm/tests/service_test.yaml b/helm/tests/service_test.yaml new file mode 100644 index 0000000..f99be91 --- /dev/null +++ b/helm/tests/service_test.yaml @@ -0,0 +1,44 @@ +suite: Service +templates: + - service.yaml +tests: + - it: should render service by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service + - equal: + path: metadata.name + value: RELEASE-NAME-robotlb-metrics + + - it: should not render service when metrics.service disabled + set: + metrics.service.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should use correct port + asserts: + - equal: + path: spec.ports[0].port + value: 8080 + - equal: + path: spec.ports[0].targetPort + value: 8080 + - equal: + path: spec.ports[0].name + value: metrics + - equal: + path: spec.ports[0].protocol + value: TCP + + - it: should include annotations when provided + set: + metrics.service.annotations: + custom_annotation: custom_value + asserts: + - equal: + path: metadata.annotations.custom_annotation + value: custom_value \ No newline at end of file diff --git a/helm/tests/servicemonitor_test.yaml b/helm/tests/servicemonitor_test.yaml new file mode 100644 index 0000000..5e780c7 --- /dev/null +++ b/helm/tests/servicemonitor_test.yaml @@ -0,0 +1,42 @@ +suite: ServiceMonitor +templates: + - servicemonitor.yaml +tests: + - it: should not render service monitor by default + set: + metrics.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should not render service monitor when metrics disabled + set: + metrics.enabled: false + metrics.serviceMonitor.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should render service monitor when both enabled + set: + metrics.enabled: true + metrics.serviceMonitor.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ServiceMonitor + + - it: should use correct interval and timeout + set: + metrics.enabled: true + metrics.serviceMonitor.enabled: true + metrics.serviceMonitor.interval: 60s + metrics.serviceMonitor.scrapeTimeout: 30s + asserts: + - equal: + path: spec.endpoints[0].interval + value: 60s + - equal: + path: spec.endpoints[0].scrapeTimeout + value: 30s \ No newline at end of file From 62042e6700d9d34868f4a0b8afd4b450521f4bf4 Mon Sep 17 00:00:00 2001 From: "forkline-dev[bot]" Date: Thu, 5 Mar 2026 22:54:18 +0000 Subject: [PATCH 3/7] feat: rework dashboard with template variables and improved layout - Add template variables for datasource and controller selection - Reorganize panels into logical sections: Overview, Reconciliation Performance, Errors & Failures, Hetzner Cloud API - Add new panels: Reconcile Success Rate, API Error Percentage, Operations vs Failures - Include percentile metrics (p50, p95, p99) for reconcile duration - Add comprehensive panel descriptions for better documentation - Update to Grafana schema version 39 - Add appropriate dashboard tags (robotlb, operator, kubernetes, hetzner) - Configure auto-refresh interval of 30s - Add tests for new dashboard features --- helm/templates/dashboard.yaml | 1640 +++++++++++++++++++++++--------- helm/tests/dashboard_test.yaml | 85 +- 2 files changed, 1277 insertions(+), 448 deletions(-) diff --git a/helm/templates/dashboard.yaml b/helm/templates/dashboard.yaml index 9018bfc..0ddfb1f 100644 --- a/helm/templates/dashboard.yaml +++ b/helm/templates/dashboard.yaml @@ -16,7 +16,10 @@ data: "list": [ { "builtIn": 1, - "datasource": "-- Prometheus --", + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -25,10 +28,10 @@ data: } ] }, - "description": "RobotLB Operator Dashboard", + "description": "RobotLB Operator Dashboard - Monitors controller health, reconciliation operations, and Hetzner Cloud API interactions", "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, "links": [], "liveNow": false, @@ -43,7 +46,7 @@ data: }, "id": 1, "panels": [], - "title": "Overview", + "title": "📊 Overview", "type": "row" }, { @@ -51,6 +54,7 @@ data: "type": "prometheus", "uid": "${datasource}" }, + "description": "Total number of Kubernetes services currently managed by RobotLB operator", "fieldConfig": { "defaults": { "color": { @@ -72,7 +76,7 @@ data: }, "gridPos": { "h": 4, - "w": 6, + "w": 4, "x": 0, "y": 1 }, @@ -98,7 +102,7 @@ data: "type": "prometheus", "uid": "${datasource}" }, - "expr": "robotlb_services_managed{controller=\"robotlb\"}", + "expr": "robotlb_services_managed{controller=\"$controller\"}", "refId": "A" } ], @@ -106,594 +110,1338 @@ data: "type": "stat" }, { - "datasource":{ + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Current leader election status: Leader (1) or Standby (0)", "fieldConfig": { - "defaults":{ - "color":{ + "defaults": { + "color": { "mode": "thresholds" }, - "mappings":[ + "mappings": [ { - "options":{ - "0":{ - "color":"yellow", - "index":0, - "text":"Standby" + "options": { + "0": { + "color": "yellow", + "index": 0, + "text": "Standby" }, - "1":{ - "color":"green", - "index":1, - "text":"Leader" + "1": { + "color": "green", + "index": 1, + "text": "Leader" } }, - "type":"value" + "type": "value" } ], - "thresholds":{ - "mode":"absolute", - "steps":[ + "thresholds": { + "mode": "absolute", + "steps": [ { - "color":"green", - "value":null + "color": "green", + "value": null } ] }, - "unit":"none" + "unit": "none" }, - "overrides":[] + "overrides": [] }, - "gridPos":{ + "gridPos": { "h": 4, - "w": 6, - "x": 6, + "w": 4, + "x": 4, "y": 1 }, "id": 3, - "options":{ - "colorMode":"value", - "graphMode":"area", - "justifyMode":"auto", - "orientation":"auto", - "reduceOptions":{ - "calcs":[ + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ "lastNotNull" ], - "fields":"", - "values":false + "fields": "", + "values": false }, - "textMode":"auto" + "textMode": "auto" }, - "pluginVersion":"11.0.0", - "targets":[ + "pluginVersion": "11.0.0", + "targets": [ { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "expr":"robotlb_leader_status{controller=\"robotlb\"}", - "refId":"A" + "expr": "robotlb_leader_status{controller=\"$controller\"}", + "refId": "A" } ], - "title":"Leader Status", - "type":"stat" + "title": "Leader Status", + "type": "stat" }, { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig":{ - "defaults":{ - "color":{ - "mode":"palette-classic" - }, - "custom":{ - "axisBorderShow":false, - "axisCenteredZero":false, - "axisColorMode":"text", - "axisLabel":"", - "axisPlacement":"auto", - "barAlignment":0, - "drawStyle":"line", - "fillOpacity":0, - "gradientMode":"none", - "hideFrom":{ - "legend":false, - "tooltip":false, - "viz":false - }, - "insertNulls":false, - "lineInterpolation":"linear", - "lineWidth":1, - "pointSize":5, - "scaleDistribution":{ - "type":"linear" - }, - "showPoints":"auto", - "spanNulls":false, - "stacking":{ - "group":"A", - "mode":"none" - }, - "thresholdsStyle":{ - "mode":"off" - } + "description": "Success rate of reconcile operations over the selected time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "mappings":[], - "thresholds":{ - "mode":"absolute", - "steps":[ + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, { - "color":"green", - "value":null + "color": "yellow", + "value": 95 }, { - "color":"red", - "value":80 + "color": "green", + "value": 99 } ] }, - "unit":"s" + "unit": "percent" }, - "overrides":[] + "overrides": [] }, - "gridPos":{ + "gridPos": { "h": 4, - "w": 12, - "x": 12, + "w": 4, + "x": 8, "y": 1 }, - "id": 4, - "options":{ - "legend":{ - "calcs":[ - "mean", - "max" + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "displayMode":"list", - "placement":"bottom", - "showLegend":true + "fields": "", + "values": false }, - "tooltip":{ - "mode":"single", - "sort":"none" - } + "textMode": "auto" }, - "targets":[ + "pluginVersion": "11.0.0", + "targets": [ { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "expr":"robotlb_reconcile_duration_seconds{controller=\"robotlb\"}", - "legendFormat":"Reconcile Duration", - "refId":"A" + "expr": "(\n sum(rate(robotlb_reconcile_operations_total{controller=\"$controller\"}[$__rate_interval]))\n -\n sum(rate(robotlb_reconcile_failures_total{controller=\"$controller\"}[$__rate_interval]))\n)\n/\nsum(rate(robotlb_reconcile_operations_total{controller=\"$controller\"}[$__rate_interval]))\n* 100\n", + "refId": "A" } ], - "title":"Reconcile Duration", - "type":"timeseries" - }, - { - "collapsed":false, - "gridPos":{ - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 5, - "panels":[], - "title":"Operations", - "type":"row" + "title": "Reconcile Success Rate", + "type": "stat" }, { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig":{ - "defaults":{ - "color":{ - "mode":"palette-classic" - }, - "custom":{ - "axisBorderShow":false, - "axisCenteredZero":false, - "axisColorMode":"text", - "axisLabel":"", - "axisPlacement":"auto", - "barAlignment":0, - "drawStyle":"line", - "fillOpacity":10, - "gradientMode":"none", - "hideFrom":{ - "legend":false, - "tooltip":false, - "viz":false - }, - "insertNulls":false, - "lineInterpolation":"linear", - "lineWidth":1, - "pointSize":5, - "scaleDistribution":{ - "type":"linear" - }, - "showPoints":"never", - "spanNulls":false, - "stacking":{ - "group":"A", - "mode":"none" - }, - "thresholdsStyle":{ - "mode":"off" - } + "description": "Current rate of reconcile operations per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "mappings":[], - "thresholds":{ - "mode":"absolute", - "steps":[ + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "color":"green", - "value":null + "color": "green", + "value": null } ] }, - "unit":"ops" + "unit": "ops" }, - "overrides":[] + "overrides": [] }, - "gridPos":{ - "h": 8, - "w": 12, - "x": 0, - "y": 6 + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 }, - "id": 6, - "options":{ - "legend":{ - "calcs":[ - "rate" + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "displayMode":"list", - "placement":"bottom", - "showLegend":true + "fields": "", + "values": false }, - "tooltip":{ - "mode":"multi", - "sort":"none" - } + "textMode": "auto" }, - "targets":[ + "pluginVersion": "11.0.0", + "targets": [ { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "expr":"rate(robotlb_reconcile_operations_total{controller=\"robotlb\"}[5m])", - "legendFormat":"Reconciles/sec", - "refId":"A" + "expr": "sum(rate(robotlb_reconcile_operations_total{controller=\"$controller\"}[$__rate_interval]))", + "refId": "A" } ], - "title":"Reconcile Rate", - "type":"timeseries" + "title": "Reconcile Rate", + "type": "stat" }, { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig":{ - "defaults":{ - "color":{ - "mode":"palette-classic" - }, - "custom":{ - "axisBorderShow":false, - "axisCenteredZero":false, - "axisColorMode":"text", - "axisLabel":"", - "axisPlacement":"auto", - "barAlignment":0, - "drawStyle":"line", - "fillOpacity":10, - "gradientMode":"none", - "hideFrom":{ - "legend":false, - "tooltip":false, - "viz":false - }, - "insertNulls":false, - "lineInterpolation":"linear", - "lineWidth":1, - "pointSize":5, - "scaleDistribution":{ - "type":"linear" - }, - "showPoints":"never", - "spanNulls":false, - "stacking":{ - "group":"A", - "mode":"none" - }, - "thresholdsStyle":{ - "mode":"off" - } + "description": "Average duration of reconcile operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "mappings":[], - "thresholds":{ - "mode":"absolute", - "steps":[ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "color":"green", - "value":null + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 }, { - "color":"red", - "value":0.01 + "color": "red", + "value": 5 } ] }, - "unit":"ops" + "unit": "s" }, - "overrides":[] + "overrides": [] }, - "gridPos":{ - "h": 8, - "w": 12, - "x": 12, - "y": 6 + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 }, - "id": 7, - "options":{ - "legend":{ - "calcs":[ - "rate" + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" ], - "displayMode":"list", - "placement":"bottom", - "showLegend":true + "fields": "", + "values": false }, - "tooltip":{ - "mode":"multi", - "sort":"none" - } + "textMode": "auto" }, - "targets":[ + "pluginVersion": "11.0.0", + "targets": [ { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "expr":"rate(robotlb_reconcile_failures_total{controller=\"robotlb\"}[5m])", - "legendFormat":"Failures/sec", - "refId":"A" + "expr": "rate(robotlb_reconcile_duration_seconds_sum{controller=\"$controller\"}[$__rate_interval])\n/\nrate(robotlb_reconcile_duration_seconds_count{controller=\"$controller\"}[$__rate_interval])\n", + "refId": "A" } ], - "title":"Failure Rate", - "type":"timeseries" - }, - { - "collapsed":false, - "gridPos":{ - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 8, - "panels":[], - "title":"Hetzner Cloud API", - "type":"row" + "title": "Avg Reconcile Duration", + "type": "stat" }, { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig":{ - "defaults":{ - "color":{ - "mode":"palette-classic" - }, - "custom":{ - "axisBorderShow":false, - "axisCenteredZero":false, - "axisColorMode":"text", - "axisLabel":"", - "axisPlacement":"auto", - "barAlignment":0, - "drawStyle":"line", - "fillOpacity":10, - "gradientMode":"none", - "hideFrom":{ - "legend":false, - "tooltip":false, - "viz":false - }, - "insertNulls":false, - "lineInterpolation":"linear", - "lineWidth":1, - "pointSize":5, - "scaleDistribution":{ - "type":"linear" - }, - "showPoints":"never", - "spanNulls":false, - "stacking":{ - "group":"A", - "mode":"none" - }, - "thresholdsStyle":{ - "mode":"off" - } + "description": "Current rate of Hetzner Cloud API requests per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "mappings":[], - "thresholds":{ - "mode":"absolute", - "steps":[ + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "color":"green", - "value":null + "color": "green", + "value": null } ] }, - "unit":"ops" + "unit": "ops" }, - "overrides":[] + "overrides": [] }, - "gridPos":{ - "h": 8, - "w": 12, - "x": 0, - "y": 15 + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 }, - "id": 9, - "options":{ - "legend":{ - "calcs":[ - "rate" + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "displayMode":"list", - "placement":"bottom", - "showLegend":true + "fields": "", + "values": false }, - "tooltip":{ - "mode":"multi", - "sort":"none" - } + "textMode": "auto" }, - "targets":[ + "pluginVersion": "11.0.0", + "targets": [ { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "expr":"rate(robotlb_hcloud_api_requests_total{controller=\"robotlb\"}[5m])", - "legendFormat":"Requests/sec", - "refId":"A" + "expr": "sum(rate(robotlb_hcloud_api_requests_total{controller=\"$controller\"}[$__rate_interval]))", + "refId": "A" } ], - "title":"API Request Rate", - "type":"timeseries" + "title": "API Request Rate", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "panels": [], + "title": "⚙️ Reconciliation Performance", + "type": "row" }, { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig":{ - "defaults":{ - "color":{ - "mode":"palette-classic" - }, - "custom":{ - "axisBorderShow":false, - "axisCenteredZero":false, - "axisColorMode":"text", - "axisLabel":"", - "axisPlacement":"auto", - "barAlignment":0, - "drawStyle":"line", - "fillOpacity":10, - "gradientMode":"none", - "hideFrom":{ - "legend":false, - "tooltip":false, - "viz":false + "description": "Rate of reconcile operations per second over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "insertNulls":false, - "lineInterpolation":"linear", - "lineWidth":1, - "pointSize":5, - "scaleDistribution":{ - "type":"linear" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "showPoints":"never", - "spanNulls":false, - "stacking":{ - "group":"A", - "mode":"none" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "thresholdsStyle":{ - "mode":"off" + "thresholdsStyle": { + "mode": "off" } }, - "mappings":[], - "thresholds":{ - "mode":"absolute", - "steps":[ - { - "color":"green", - "value":null - }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "color":"red", - "value":0.001 + "color": "green", + "value": null } ] }, - "unit":"ops" + "unit": "ops" }, - "overrides":[] + "overrides": [] }, - "gridPos":{ + "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 15 + "x": 0, + "y": 6 }, - "id": 10, - "options":{ - "legend":{ - "calcs":[ - "rate" + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" ], - "displayMode":"list", - "placement":"bottom", - "showLegend":true + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "tooltip":{ - "mode":"multi", - "sort":"none" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "targets":[ + "targets": [ { - "datasource":{ - "type":"prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(robotlb_reconcile_operations_total{controller=\"$controller\"}[$__rate_interval])", + "legendFormat": "Reconciles/sec", + "refId": "A" + } + ], + "title": "Reconcile Operations Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Duration of reconcile operations over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(robotlb_reconcile_duration_seconds_sum{controller=\"$controller\"}[$__rate_interval])\n/\nrate(robotlb_reconcile_duration_seconds_count{controller=\"$controller\"}[$__rate_interval])\n", + "legendFormat": "Avg Duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum(rate(robotlb_reconcile_duration_seconds_bucket{controller=\"$controller\"}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(robotlb_reconcile_duration_seconds_bucket{controller=\"$controller\"}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum(rate(robotlb_reconcile_duration_seconds_bucket{controller=\"$controller\"}[$__rate_interval])) by (le))", + "legendFormat": "p99", + "refId": "D" + } + ], + "title": "Reconcile Duration", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 8, + "panels": [], + "title": "❌ Errors & Failures", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of reconcile failures per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", "uid": "${datasource}" }, - "expr":"rate(robotlb_hcloud_api_errors_total{controller=\"robotlb\"}[5m])", - "legendFormat":"Errors/sec", - "refId":"A" + "expr": "rate(robotlb_reconcile_failures_total{controller=\"$controller\"}[$__rate_interval])", + "legendFormat": "Failures/sec", + "refId": "A" } ], - "title":"API Error Rate", - "type":"timeseries" + "title": "Reconcile Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cumulative count of reconcile operations vs failures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Operations" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Failures" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "robotlb_reconcile_operations_total{controller=\"$controller\"}", + "legendFormat": "Total Operations", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "robotlb_reconcile_failures_total{controller=\"$controller\"}", + "legendFormat": "Total Failures", + "refId": "B" + } + ], + "title": "Operations vs Failures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 9, + "panels": [], + "title": "☁️ Hetzner Cloud API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of Hetzner Cloud API requests per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(robotlb_hcloud_api_requests_total{controller=\"$controller\"}[$__rate_interval])", + "legendFormat": "Requests/sec", + "refId": "A" + } + ], + "title": "API Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of Hetzner Cloud API errors per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.001 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(robotlb_hcloud_api_errors_total{controller=\"$controller\"}[$__rate_interval])", + "legendFormat": "Errors/sec", + "refId": "A" + } + ], + "title": "API Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "API error rate as a percentage of total requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n sum(rate(robotlb_hcloud_api_errors_total{controller=\"$controller\"}[$__rate_interval]))\n /\n sum(rate(robotlb_hcloud_api_requests_total{controller=\"$controller\"}[$__rate_interval]))\n) * 100\n", + "legendFormat": "Error Rate %", + "refId": "A" + } + ], + "title": "API Error Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total cumulative API requests and errors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "robotlb_hcloud_api_requests_total{controller=\"$controller\"}", + "legendFormat": "Total Requests", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "robotlb_hcloud_api_errors_total{controller=\"$controller\"}", + "legendFormat": "Total Errors", + "refId": "B" + } + ], + "title": "API Requests vs Errors", + "type": "timeseries" } ], - "refresh":"", - "schemaVersion":38, - "style":"dark", - "tags":[ + "refresh": "30s", + "schemaVersion": 39, + "tags": [ "robotlb", - "operator" + "operator", + "kubernetes", + "hetzner" ], - "templating":{ - "list":[] + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(robotlb_services_managed, controller)", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": true, + "name": "controller", + "options": [], + "query": { + "query": "label_values(robotlb_services_managed, controller)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" }, - "time":{ - "from":"now-1h", - "to":"now" + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] }, - "timepicker":{}, - "timezone":"", - "title":"RobotLB", - "uid":"robotlb", - "version":1, - "weekStart":"" + "timezone": "browser", + "title": "RobotLB", + "uid": "robotlb", + "version": 2, + "weekStart": "" } -{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/tests/dashboard_test.yaml b/helm/tests/dashboard_test.yaml index c13ef86..2d88bf0 100644 --- a/helm/tests/dashboard_test.yaml +++ b/helm/tests/dashboard_test.yaml @@ -86,11 +86,92 @@ tests: path: data["robotlb-dashboard.json"] pattern: 'robotlb_hcloud_api_errors_total' - - it: should use correct controller label in queries + - it: should use template variable for controller label in queries set: metrics.enabled: true metrics.grafanaDashboard.enabled: true asserts: - matchRegex: path: data["robotlb-dashboard.json"] - pattern: 'controller=\\"robotlb\\"' \ No newline at end of file + pattern: 'controller=\\"\$controller\\"' + + - it: should include template variables for flexibility + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: '"name": "datasource"' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: '"name": "controller"' + + - it: should use modern Grafana schema version + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: '"schemaVersion": 39' + + - it: should include all dashboard sections + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'Overview' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'Reconciliation Performance' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'Errors.*Failures' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'Hetzner Cloud API' + + - it: should include percentile metrics in duration panel + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'histogram_quantile.*0\.50' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'histogram_quantile.*0\.95' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'histogram_quantile.*0\.99' + + - it: should calculate success rate correctly + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: 'Reconcile Success Rate' + + - it: should include appropriate dashboard tags + set: + metrics.enabled: true + metrics.grafanaDashboard.enabled: true + asserts: + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: '"tags".*"robotlb"' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: '"tags".*"operator"' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: '"tags".*"kubernetes"' + - matchRegex: + path: data["robotlb-dashboard.json"] + pattern: '"tags".*"hetzner"' \ No newline at end of file From cf867c83cf15623b3991ad98871db36f1759e8bf Mon Sep 17 00:00:00 2001 From: "forkline-dev[bot]" Date: Thu, 5 Mar 2026 23:05:39 +0000 Subject: [PATCH 4/7] fix: add trailing newlines to test files --- helm/tests/dashboard_test.yaml | 2 +- helm/tests/prometheusrules_test.yaml | 2 +- helm/tests/service_test.yaml | 2 +- helm/tests/servicemonitor_test.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/tests/dashboard_test.yaml b/helm/tests/dashboard_test.yaml index 2d88bf0..b3356a3 100644 --- a/helm/tests/dashboard_test.yaml +++ b/helm/tests/dashboard_test.yaml @@ -174,4 +174,4 @@ tests: pattern: '"tags".*"kubernetes"' - matchRegex: path: data["robotlb-dashboard.json"] - pattern: '"tags".*"hetzner"' \ No newline at end of file + pattern: '"tags".*"hetzner"' diff --git a/helm/tests/prometheusrules_test.yaml b/helm/tests/prometheusrules_test.yaml index 760020a..be043cb 100644 --- a/helm/tests/prometheusrules_test.yaml +++ b/helm/tests/prometheusrules_test.yaml @@ -78,4 +78,4 @@ tests: asserts: - equal: path: metadata.labels["custom_label"] - value: custom_value \ No newline at end of file + value: custom_value diff --git a/helm/tests/service_test.yaml b/helm/tests/service_test.yaml index f99be91..0a4f7f0 100644 --- a/helm/tests/service_test.yaml +++ b/helm/tests/service_test.yaml @@ -41,4 +41,4 @@ tests: asserts: - equal: path: metadata.annotations.custom_annotation - value: custom_value \ No newline at end of file + value: custom_value diff --git a/helm/tests/servicemonitor_test.yaml b/helm/tests/servicemonitor_test.yaml index 5e780c7..4a858b6 100644 --- a/helm/tests/servicemonitor_test.yaml +++ b/helm/tests/servicemonitor_test.yaml @@ -39,4 +39,4 @@ tests: value: 60s - equal: path: spec.endpoints[0].scrapeTimeout - value: 30s \ No newline at end of file + value: 30s From 4d41adfed0c3f514b3fe7b57fdd1ef53539e935b Mon Sep 17 00:00:00 2001 From: opencode Date: Thu, 5 Mar 2026 23:09:20 +0000 Subject: [PATCH 5/7] fix: use [\s\S]* in regex to match across newlines in JSON The dashboard JSON is multi-line, so .* doesn't match across newlines. Changed to [\s\S]* which matches any character including newlines. --- helm/tests/dashboard_test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/tests/dashboard_test.yaml b/helm/tests/dashboard_test.yaml index b3356a3..d39b92d 100644 --- a/helm/tests/dashboard_test.yaml +++ b/helm/tests/dashboard_test.yaml @@ -165,13 +165,13 @@ tests: asserts: - matchRegex: path: data["robotlb-dashboard.json"] - pattern: '"tags".*"robotlb"' + pattern: '"tags"[\s\S]*"robotlb"' - matchRegex: path: data["robotlb-dashboard.json"] - pattern: '"tags".*"operator"' + pattern: '"tags"[\s\S]*"operator"' - matchRegex: path: data["robotlb-dashboard.json"] - pattern: '"tags".*"kubernetes"' + pattern: '"tags"[\s\S]*"kubernetes"' - matchRegex: path: data["robotlb-dashboard.json"] - pattern: '"tags".*"hetzner"' + pattern: '"tags"[\s\S]*"hetzner"' From f2b3ed931ea807e6bcd33ab9fa6a7e6b3df9b819 Mon Sep 17 00:00:00 2001 From: opencode Date: Thu, 5 Mar 2026 23:15:39 +0000 Subject: [PATCH 6/7] ci: re-trigger CI after transient infrastructure failure From bae216b1fd672dcb43b6ef242e9e0302b6fe38e8 Mon Sep 17 00:00:00 2001 From: "forkline-dev[bot]" Date: Fri, 6 Mar 2026 13:29:58 +0000 Subject: [PATCH 7/7] style: add missing newline at end of dashboard.yaml --- helm/templates/dashboard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/dashboard.yaml b/helm/templates/dashboard.yaml index 0ddfb1f..0418b10 100644 --- a/helm/templates/dashboard.yaml +++ b/helm/templates/dashboard.yaml @@ -1444,4 +1444,4 @@ data: "version": 2, "weekStart": "" } -{{- end }} \ No newline at end of file +{{- end }}