diff --git a/core/internal/cnpgi/backup.go b/core/internal/cnpgi/backup.go index 90688b32..86eaea77 100644 --- a/core/internal/cnpgi/backup.go +++ b/core/internal/cnpgi/backup.go @@ -117,8 +117,8 @@ func (b backupServiceImplementation) Backup( ) backupStart := time.Now() - recordBackupStart(ctx) - defer recordBackupFinished(ctx) + recordBackupStart(ctx, cluster.Name) + defer recordBackupFinished(ctx, cluster.Name) metadata, err := b.runBackup( ctx, @@ -126,7 +126,7 @@ func (b backupServiceImplementation) Backup( isPrimary, ) if err != nil { - recordBackupFailure(ctx, time.Since(backupStart), err) + recordBackupFailure(ctx, cluster.Name, time.Since(backupStart), err) span.RecordError(err) span.SetStatus(codes.Error, "backup failed") @@ -138,14 +138,14 @@ func (b backupServiceImplementation) Backup( // verification is folded into the successful-backup recording below. corruption, verifyErr := b.runVerify(ctx, backupName) if corruption { - recordBackupFailure(ctx, time.Since(backupStart), verifyErr) + recordBackupFailure(ctx, cluster.Name, time.Since(backupStart), verifyErr) span.RecordError(verifyErr) span.SetStatus(codes.Error, "verification detected corruption") return nil, verifyErr } - recordBackupSuccess(ctx, time.Since(backupStart)) + recordBackupSuccess(ctx, cluster.Name, time.Since(backupStart)) return &backup.BackupResult{ BackupName: backupName, diff --git a/core/internal/cnpgi/metrics.go b/core/internal/cnpgi/metrics.go index fe0141f8..908ff563 100644 --- a/core/internal/cnpgi/metrics.go +++ b/core/internal/cnpgi/metrics.go @@ -23,44 +23,63 @@ import ( "context" "time" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "github.com/cloudnative-pg/klio/core/internal/opentelemetry" ) +// clusterAttr returns the `cluster_name` attribute every plugin backup metric +// carries, so panels can attribute backup activity to a specific PostgreSQL +// cluster even when several clusters share one namespace. +func clusterAttr(clusterName string) attribute.KeyValue { + return opentelemetry.AttributeKeyClusterName.Of(clusterName) +} + // recordBackupStart records that a backup has started. Callers must pair this // with a deferred recordBackupFinished so the in-progress counter decrements // on every exit path, including panics. -func recordBackupStart(ctx context.Context) { - opentelemetry.PluginBackup.LatestStartTime.Record(ctx, time.Now().Unix()) - opentelemetry.PluginBackup.InProgress.Add(ctx, 1) +func recordBackupStart(ctx context.Context, clusterName string) { + cluster := clusterAttr(clusterName) + opentelemetry.PluginBackup.LatestStartTime.Record(ctx, time.Now().Unix(), + metric.WithAttributes(cluster)) + opentelemetry.PluginBackup.InProgress.Add(ctx, 1, metric.WithAttributes(cluster)) } // recordBackupFinished decrements the in-progress counter. Always invoke via // defer immediately after recordBackupStart so concurrent backup accounting -// stays correct even when a backup panics or returns early. -func recordBackupFinished(ctx context.Context) { - opentelemetry.PluginBackup.InProgress.Add(ctx, -1) +// stays correct even when a backup panics or returns early. It must pass the +// same clusterName as recordBackupStart so the up/down counter cancels out per +// cluster. +func recordBackupFinished(ctx context.Context, clusterName string) { + opentelemetry.PluginBackup.InProgress.Add(ctx, -1, + metric.WithAttributes(clusterAttr(clusterName))) } // recordBackupSuccess records a successful backup completion. -func recordBackupSuccess(ctx context.Context, duration time.Duration) { - opentelemetry.PluginBackup.LatestCompletionTime.Record(ctx, time.Now().Unix()) - opentelemetry.PluginBackup.LatestDuration.Record(ctx, duration.Seconds()) +func recordBackupSuccess(ctx context.Context, clusterName string, duration time.Duration) { + cluster := clusterAttr(clusterName) + opentelemetry.PluginBackup.LatestCompletionTime.Record(ctx, time.Now().Unix(), + metric.WithAttributes(cluster)) + opentelemetry.PluginBackup.LatestDuration.Record(ctx, duration.Seconds(), + metric.WithAttributes(cluster)) opentelemetry.PluginBackup.Duration.Record(ctx, duration.Seconds(), - metric.WithAttributes(opentelemetry.OutcomeSuccess.Attribute())) + metric.WithAttributes(cluster, opentelemetry.OutcomeSuccess.Attribute())) opentelemetry.PluginBackup.Runs.Add(ctx, 1, - metric.WithAttributes(opentelemetry.OutcomeSuccess.Attribute())) + metric.WithAttributes(cluster, opentelemetry.OutcomeSuccess.Attribute())) } // recordBackupFailure records a failed backup. -func recordBackupFailure(ctx context.Context, duration time.Duration, err error) { +func recordBackupFailure(ctx context.Context, clusterName string, duration time.Duration, err error) { + cluster := clusterAttr(clusterName) category := classifyRunBackupError(ctx, err) - opentelemetry.PluginBackup.LatestFailureTime.Record(ctx, time.Now().Unix()) + opentelemetry.PluginBackup.LatestFailureTime.Record(ctx, time.Now().Unix(), + metric.WithAttributes(cluster)) opentelemetry.PluginBackup.Duration.Record(ctx, duration.Seconds(), - metric.WithAttributes(opentelemetry.OutcomeFailure.Attribute())) + metric.WithAttributes(cluster, opentelemetry.OutcomeFailure.Attribute())) opentelemetry.PluginBackup.Runs.Add(ctx, 1, metric.WithAttributes( + cluster, opentelemetry.OutcomeFailure.Attribute(), opentelemetry.AttributeKeyFailureCategory.Of(category.Name), )) diff --git a/core/internal/cnpgi/metrics_test.go b/core/internal/cnpgi/metrics_test.go index 666e7e5f..273d384c 100644 --- a/core/internal/cnpgi/metrics_test.go +++ b/core/internal/cnpgi/metrics_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" @@ -36,6 +37,10 @@ import ( "github.com/cloudnative-pg/klio/core/internal/opentelemetry" ) +// testClusterName is the PostgreSQL cluster name every record* call in these +// tests tags its metrics with. +const testClusterName = "cluster-test" + // setupTestMeter installs a test MeterProvider with a ManualReader, // re-creates all instruments against it, and returns the reader. func setupTestMeter(t *testing.T) *sdkmetric.ManualReader { @@ -197,7 +202,7 @@ func TestRecordBackupStart(t *testing.T) { reader := setupTestMeter(t) before := time.Now().Unix() - recordBackupStart(context.Background()) + recordBackupStart(context.Background(), testClusterName) rm := collectOTelMetrics(t, reader) @@ -213,11 +218,11 @@ func TestRecordBackupStart(t *testing.T) { func TestRecordBackupSuccess(t *testing.T) { reader := setupTestMeter(t) - recordBackupStart(context.Background()) + recordBackupStart(context.Background(), testClusterName) duration := 42 * time.Second before := time.Now().Unix() - recordBackupSuccess(context.Background(), duration) - recordBackupFinished(context.Background()) + recordBackupSuccess(context.Background(), testClusterName, duration) + recordBackupFinished(context.Background(), testClusterName) rm := collectOTelMetrics(t, reader) @@ -252,15 +257,15 @@ func TestRecordBackupSuccess(t *testing.T) { func TestRecordBackupFailure(t *testing.T) { reader := setupTestMeter(t) - recordBackupStart(t.Context()) + recordBackupStart(t.Context(), testClusterName) before := time.Now().Unix() //nolint:gosec // hardcoded test input exitErr := exec.CommandContext(t.Context(), "sh", "-c", fmt.Sprintf("exit %d", backupfailure.RepositoryError.ExitCode)).Run() - recordBackupFailure(t.Context(), 7*time.Second, exitErr) - recordBackupFinished(t.Context()) + recordBackupFailure(t.Context(), testClusterName, 7*time.Second, exitErr) + recordBackupFinished(t.Context(), testClusterName) rm := collectOTelMetrics(t, reader) @@ -294,16 +299,16 @@ func TestRecordBackupFailureCategoriesAreSeparateSeries(t *testing.T) { expiredCtx, cancelTimeout := context.WithTimeout(t.Context(), 0) defer cancelTimeout() - recordBackupFailure(expiredCtx, time.Second, nil) + recordBackupFailure(expiredCtx, testClusterName, time.Second, nil) canceledCtx, cancelCanceled := context.WithCancel(t.Context()) cancelCanceled() - recordBackupFailure(canceledCtx, time.Second, nil) - recordBackupFailure(canceledCtx, time.Second, nil) + recordBackupFailure(canceledCtx, testClusterName, time.Second, nil) + recordBackupFailure(canceledCtx, testClusterName, time.Second, nil) //nolint:gosec // hardcoded test input exitErr := exec.CommandContext(t.Context(), "sh", "-c", fmt.Sprintf("exit %d", backupfailure.Verification.ExitCode)).Run() - recordBackupFailure(t.Context(), time.Second, exitErr) + recordBackupFailure(t.Context(), testClusterName, time.Second, exitErr) rm := collectOTelMetrics(t, reader) @@ -327,7 +332,7 @@ func TestRecordBackupFailureCategoriesAreSeparateSeries(t *testing.T) { func TestRecordBackupFailureNilErrorDefaultsToUnknown(t *testing.T) { reader := setupTestMeter(t) - recordBackupFailure(context.Background(), time.Second, nil) + recordBackupFailure(context.Background(), testClusterName, time.Second, nil) rm := collectOTelMetrics(t, reader) @@ -337,23 +342,113 @@ func TestRecordBackupFailureNilErrorDefaultsToUnknown(t *testing.T) { assert.Equal(t, int64(1), unknown) } +// metricAttributeSets returns the attribute set of every data point on a +// metric, across the aggregation types the plugin backup instruments use +// (Int64/Float64 gauges, the Int64 up/down counter and runs counter, and the +// Float64 duration histogram). +func metricAttributeSets(a metricdata.Aggregation) []attribute.Set { + var sets []attribute.Set + switch d := a.(type) { + case metricdata.Gauge[int64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + case metricdata.Gauge[float64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + case metricdata.Sum[int64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + case metricdata.Histogram[float64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + } + + return sets +} + +// dataPointClusterNames collects the distinct `cluster_name` attribute values +// seen across every plugin backup instrument (the empty string keys the data +// points that carry no cluster_name), so a test can assert every series is +// attributed to a cluster. +func dataPointClusterNames(rm metricdata.ResourceMetrics) map[string]int { + names := map[string]int{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + for _, set := range metricAttributeSets(m.Data) { + if v, ok := set.Value("cluster_name"); ok { + names[v.AsString()]++ + } else { + names[""]++ + } + } + } + } + + return names +} + +func TestBackupMetricsCarryClusterName(t *testing.T) { + reader := setupTestMeter(t) + + recordBackupStart(context.Background(), testClusterName) + recordBackupSuccess(context.Background(), testClusterName, 5*time.Second) + recordBackupFinished(context.Background(), testClusterName) + recordBackupFailure(context.Background(), testClusterName, time.Second, assert.AnError) + + rm := collectOTelMetrics(t, reader) + + names := dataPointClusterNames(rm) + require.NotEmpty(t, names) + assert.Equal(t, 0, names[""], "every plugin backup data point must carry a cluster_name attribute") + assert.Contains(t, names, testClusterName) +} + +func TestBackupMetricsClustersAreSeparateSeries(t *testing.T) { + reader := setupTestMeter(t) + + recordBackupStart(context.Background(), "cluster-a") + recordBackupSuccess(context.Background(), "cluster-a", 5*time.Second) + recordBackupFinished(context.Background(), "cluster-a") + + recordBackupStart(context.Background(), "cluster-b") + recordBackupSuccess(context.Background(), "cluster-b", 9*time.Second) + recordBackupFinished(context.Background(), "cluster-b") + + rm := collectOTelMetrics(t, reader) + + // The runs counter must expose one data point per (cluster_name, outcome), + // not a single folded series. + perCluster := map[string]int64{} + for _, dp := range findInt64SumDataPoints(rm, opentelemetry.PluginBackupRunsMetric) { + v, ok := dp.Attributes.Value("cluster_name") + require.True(t, ok, "runs data point missing cluster_name") + perCluster[v.AsString()] += dp.Value + } + assert.Equal(t, int64(1), perCluster["cluster-a"]) + assert.Equal(t, int64(1), perCluster["cluster-b"]) +} + func TestBackupMetricsMultipleRuns(t *testing.T) { reader := setupTestMeter(t) // First backup: success. - recordBackupStart(context.Background()) - recordBackupSuccess(context.Background(), 10*time.Second) - recordBackupFinished(context.Background()) + recordBackupStart(context.Background(), testClusterName) + recordBackupSuccess(context.Background(), testClusterName, 10*time.Second) + recordBackupFinished(context.Background(), testClusterName) // Second backup: failure. - recordBackupStart(context.Background()) - recordBackupFailure(context.Background(), 3*time.Second, assert.AnError) - recordBackupFinished(context.Background()) + recordBackupStart(context.Background(), testClusterName) + recordBackupFailure(context.Background(), testClusterName, 3*time.Second, assert.AnError) + recordBackupFinished(context.Background(), testClusterName) // Third backup: success. - recordBackupStart(context.Background()) - recordBackupSuccess(context.Background(), 30*time.Second) - recordBackupFinished(context.Background()) + recordBackupStart(context.Background(), testClusterName) + recordBackupSuccess(context.Background(), testClusterName, 30*time.Second) + recordBackupFinished(context.Background(), testClusterName) rm := collectOTelMetrics(t, reader) diff --git a/core/internal/opentelemetry/catalog.go b/core/internal/opentelemetry/catalog.go index eb1eb66a..9fa217a8 100644 --- a/core/internal/opentelemetry/catalog.go +++ b/core/internal/opentelemetry/catalog.go @@ -137,7 +137,10 @@ const ( ) // PluginBackupMetrics holds OTel instruments for backup lifecycle tracking. -// The Runs counter carries an `outcome` attribute (`success` / `failure`) +// Every instrument carries a `cluster_name` attribute identifying the +// PostgreSQL cluster the plugin sidecar serves, so panels can attribute backup +// activity per cluster even when several clusters share a namespace. The Runs +// counter additionally carries an `outcome` attribute (`success` / `failure`) // so a single instrument exposes both flavors. Runs failure data points // additionally carry a `failure_category` attribute classifying the failure // (see opentelemetry.FailureCategory); verification failures are recorded @@ -254,34 +257,37 @@ func InitPluginBackupMetrics() { meter := otel.Meter(Meter) PluginBackup.LatestStartTime, _ = meter.Int64Gauge(PluginBackupLatestStartTimeMetric, - metric.WithDescription("Unix epoch timestamp when the most recent backup started."), + metric.WithDescription("Unix epoch timestamp when the most recent backup started, "+ + "broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.LatestCompletionTime, _ = meter.Int64Gauge(PluginBackupLatestCompletionTimeMetric, - metric.WithDescription("Unix epoch timestamp when the most recent backup completed successfully."), + metric.WithDescription("Unix epoch timestamp when the most recent backup completed "+ + "successfully, broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.LatestFailureTime, _ = meter.Int64Gauge(PluginBackupLatestFailureTimeMetric, - metric.WithDescription("Unix epoch timestamp when the most recent backup failed."), + metric.WithDescription("Unix epoch timestamp when the most recent backup failed, "+ + "broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.LatestDuration, _ = meter.Float64Gauge(PluginBackupLatestDurationMetric, - metric.WithDescription("Duration of the most recent backup."), + metric.WithDescription("Duration of the most recent backup, broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.Duration, _ = meter.Float64Histogram(PluginBackupDurationMetric, - metric.WithDescription("Distribution of backup durations, split by the `outcome` "+ - "attribute (`success` / `failure`)."), + metric.WithDescription("Distribution of backup durations, broken down by `cluster_name` "+ + "and split by the `outcome` attribute (`success` / `failure`)."), metric.WithUnit("s"), ) PluginBackup.InProgress, _ = meter.Int64UpDownCounter(PluginBackupInProgressMetric, - metric.WithDescription("Number of backups currently in progress."), + metric.WithDescription("Number of backups currently in progress, broken down by `cluster_name`."), metric.WithUnit("{backups}"), ) PluginBackup.Runs, _ = meter.Int64Counter(PluginBackupRunsMetric, - metric.WithDescription("Total number of backup runs, split by the `outcome` "+ - "attribute (`success` / `failure`). Failure data points additionally "+ - "carry a `failure_category` attribute (`"+ + metric.WithDescription("Total number of backup runs, broken down by `cluster_name` and "+ + "split by the `outcome` attribute (`success` / `failure`). Failure data points "+ + "additionally carry a `failure_category` attribute (`"+ strings.Join(backupfailure.Names(), "`, `")+"`)."), metric.WithUnit("{backups}"), ) diff --git a/documentation/web/docs/user/grafana-dashboards.md b/documentation/web/docs/user/grafana-dashboards.md index 4010b9a4..2a83fe83 100644 --- a/documentation/web/docs/user/grafana-dashboards.md +++ b/documentation/web/docs/user/grafana-dashboards.md @@ -113,8 +113,25 @@ this. The dashboard declares a `datasource` template variable, so it is portable across Grafana installations and is not tied to a specific data source UID. -The `namespace` and `cluster` template variables at the top filter the panels -by Kubernetes namespace and PostgreSQL cluster. +Three more template variables at the top filter the panels: + +- `namespace`: the Kubernetes namespace, matched against + `k8s.namespace.name`. It scopes the Client / Plugin and WAL Replication + Lag panels, whose metrics are emitted from the PostgreSQL pods and + therefore carry the *cluster's* namespace. +- `server`: the Klio server, matched against the OpenTelemetry + `service.name` (not the pod host name, which two servers of the same name + in different namespaces would share). It scopes the Server panels. +- `cluster`: the PostgreSQL cluster, matched against `cluster_name`. It + scopes every per-cluster panel across all sections. Because the + server-side metrics carry the server's own namespace, per-cluster Server + panels are filtered by `server` and `cluster` rather than `namespace`, so a + cluster backed up by a server in another namespace is still attributed + correctly. + +Every aggregation groups by the identifying label (cluster, server, tier), +so multiple clusters or servers are never folded into a single misleading +value. ### Example: kube-prometheus-stack diff --git a/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png b/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png index 038e2a3c..ecaaba85 100644 Binary files a/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png and b/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png differ diff --git a/documentation/web/docs/user/images/klio_server_metrics.png b/documentation/web/docs/user/images/klio_server_metrics.png index 0357dbad..861540ef 100644 Binary files a/documentation/web/docs/user/images/klio_server_metrics.png and b/documentation/web/docs/user/images/klio_server_metrics.png differ diff --git a/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png b/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png index 7cd48860..6024bcf7 100644 Binary files a/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png and b/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png differ diff --git a/documentation/web/docs/user/opentelemetry.md b/documentation/web/docs/user/opentelemetry.md index 1d7ec731..f40204cb 100644 --- a/documentation/web/docs/user/opentelemetry.md +++ b/documentation/web/docs/user/opentelemetry.md @@ -161,7 +161,7 @@ attribute key. | Attribute | Values | Applies to | |---|---|---| | `tier` | `tier1` (local disk on the Klio server), `tier2` (remote object store) | All `klio.server.wal.*` and `klio.server.backup.*` instruments. | -| `cluster_name` | Name of the PostgreSQL cluster the recording belongs to | All `klio.server.wal.*` instruments (counters, gauges, and the WAL duration histograms), `klio.client.wal.*`, the `klio.server.backup.*` PostgreSQL backup gauges (`backups`, `latest_backup_*`, `oldest_backup_*`) and the `klio.server.backup.relay` / `klio.server.backup.maintenance` counters. | +| `cluster_name` | Name of the PostgreSQL cluster the recording belongs to | All `klio.plugin.backup.*` instruments, all `klio.server.wal.*` instruments (counters, gauges, and the WAL duration histograms), `klio.client.wal.*`, the `klio.server.backup.*` PostgreSQL backup gauges (`backups`, `latest_backup_*`, `oldest_backup_*`) and the `klio.server.backup.relay` / `klio.server.backup.maintenance` counters. | | `outcome` | `success`, `failure` | `klio.plugin.backup.runs`, `klio.server.backup.relay`, `klio.server.backup.maintenance`, `klio.server.backup.verifications`, and all WAL duration histograms (`klio.server.wal.*_duration`, `klio.client.wal.block_duration`). | | `failure_category` | `repository_error`, `source_error`, `verification`, `timeout`, `canceled`, `unknown` | `klio.plugin.backup.runs` failure data points only. | | `path` | `put` (WAL ingest), `get` (WAL serve) | `klio.server.wal.block_duration`, `klio.client.wal.block_duration`. | @@ -172,17 +172,20 @@ attribute key. ### Backup lifecycle metrics (plugin sidecar) These metrics are emitted by the plugin sidecar and track backup -operations on each PostgreSQL instance: +operations on each PostgreSQL instance. Every instrument carries a +`cluster_name` attribute identifying the PostgreSQL cluster the sidecar +serves, so backup activity can be attributed per cluster even when several +clusters share a namespace: | Metric Name | Type | Unit | Description | |---|---|---|---| -| `klio.plugin.backup.in_progress` | UpDownCounter | `{backups}` | Number of backups currently in progress | -| `klio.plugin.backup.latest_start_time` | Gauge | s | Unix epoch timestamp when the most recent backup started | -| `klio.plugin.backup.latest_completion_time` | Gauge | s | Unix epoch timestamp when the most recent backup completed successfully | -| `klio.plugin.backup.latest_failure_time` | Gauge | s | Unix epoch timestamp when the most recent backup failed | -| `klio.plugin.backup.latest_duration` | Gauge | s | Duration of the most recent backup | -| `klio.plugin.backup.duration` | Histogram | s | Distribution of backup durations, split by the `outcome` attribute (`success` / `failure`) | -| `klio.plugin.backup.runs` | Counter | `{backups}` | Total number of backup runs, split by the `outcome` attribute (`success` / `failure`). Failure data points additionally carry a `failure_category` attribute classifying the failure. Backup verification is part of a run: a verification failure is recorded here with `failure_category="verification"`, and a clean verification is included in the `outcome="success"` count | +| `klio.plugin.backup.in_progress` | UpDownCounter | `{backups}` | Number of backups currently in progress, broken down by `cluster_name` | +| `klio.plugin.backup.latest_start_time` | Gauge | s | Unix epoch timestamp when the most recent backup started, broken down by `cluster_name` | +| `klio.plugin.backup.latest_completion_time` | Gauge | s | Unix epoch timestamp when the most recent backup completed successfully, broken down by `cluster_name` | +| `klio.plugin.backup.latest_failure_time` | Gauge | s | Unix epoch timestamp when the most recent backup failed, broken down by `cluster_name` | +| `klio.plugin.backup.latest_duration` | Gauge | s | Duration of the most recent backup, broken down by `cluster_name` | +| `klio.plugin.backup.duration` | Histogram | s | Distribution of backup durations, broken down by `cluster_name` and split by the `outcome` attribute (`success` / `failure`) | +| `klio.plugin.backup.runs` | Counter | `{backups}` | Total number of backup runs, broken down by `cluster_name` and split by the `outcome` attribute (`success` / `failure`). Failure data points additionally carry a `failure_category` attribute classifying the failure. Backup verification is part of a run: a verification failure is recorded here with `failure_category="verification"`, and a clean verification is included in the `outcome="success"` count | The `failure_category` attribute on `klio.plugin.backup.runs` failure data points takes one of the following values: diff --git a/observability/grafana/build.go b/observability/grafana/build.go index 8eed8ce2..214e4514 100644 --- a/observability/grafana/build.go +++ b/observability/grafana/build.go @@ -56,13 +56,21 @@ func build() *dashboard.DashboardBuilder { labelVariable("namespace", "Namespace", `label_values({__name__=~"klio_.+"}, k8s_namespace_name)`), ). + // Server identity is the OpenTelemetry service.name, not the pod + // host_name: two Servers with the same name in different namespaces + // share a host_name (e.g. both "klio-a-klio-0") and would collapse into + // one ambiguous entry, whereas their service.name stays distinct. WithVariable( labelVariable("server", "Server", - `label_values(klio_server_uptime_seconds{k8s_namespace_name=~"$namespace"}, host_name)`), + `label_values(klio_server_uptime_seconds, service_name)`), ). + // The cluster list is narrowed by the selected server (service.name), + // NOT by namespace: a server tags its series with its own namespace, so + // filtering clusters by $namespace would drop clusters served + // cross-namespace. WithVariable( labelVariable("cluster", "Cluster", - `label_values(klio_server_wal_written_total{k8s_namespace_name=~"$namespace",host_name=~"$server"}, cluster_name)`), + `label_values(klio_server_wal_written_total{service_name=~"$server"}, cluster_name)`), ) y := 0 diff --git a/observability/grafana/client.go b/observability/grafana/client.go index 4dcda25f..6c880384 100644 --- a/observability/grafana/client.go +++ b/observability/grafana/client.go @@ -23,110 +23,117 @@ import ( "fmt" ) -// clientWalMatcher selects the WAL streaming client's per-cluster series -// (klio_client_wal_*). The streaming client runs as a child process of the -// plugin sidecar (spawned via `klio send-wal`, same container, different -// PID), but unlike klio_plugin_backup_* it carries cluster_name and no -// host_name, so these panels are scoped by $namespace and $cluster. -const clientWalMatcher = `k8s_namespace_name=~"$namespace",cluster_name=~"$cluster"` - // clientPanels returns the "Client / Plugin" section panels. These metrics are // emitted by the Klio plugin sidecar that runs in each PostgreSQL pod: the // backup lifecycle (`klio.plugin.backup.*`, exported to Prometheus as // `klio_plugin_backup_*`) and the WAL streaming client it supervises as a -// child process (`klio.client.wal.*`, exported as `klio_client_wal_*`). -// Backup queries are scoped by $namespace; WAL streaming queries additionally -// carry cluster_name and are scoped by $cluster. +// child process (`klio.client.wal.*`, exported as `klio_client_wal_*`). Both +// families carry a cluster_name, so every panel groups by cluster_name and is +// scoped by $namespace and $cluster; nothing folds several clusters that share +// a namespace into a single value. func clientPanels() []sizedPanel { return []sizedPanel{ - // Current backup state. + // Current backup state, grouped by cluster so a namespace hosting + // several clusters shows one series each instead of a folded total. sized(4, panelHeight, statPanel("Backups in progress", "none", - query(fmt.Sprintf("sum(klio_plugin_backup_in_progress{%s})", nsMatcher), "in progress"), + query(fmt.Sprintf("sum by (cluster_name) (klio_plugin_backup_in_progress{%s})", clientMatcher), + "{{cluster_name}}"), ).Decimals(0). - Description("Base backups currently running across the plugin sidecars in the namespace.")), + Description("Base backups currently running, per cluster.")), sized(4, panelHeight, statPanel("Time since last successful backup", "dtdurations", - query(fmt.Sprintf("time() - max(klio_plugin_backup_latest_completion_time_seconds{%s})", nsMatcher), - "since success"), - ).Description("Elapsed time since the most recent base backup completed successfully. A value well "+ - "above the backup interval means backups have stopped succeeding.")), + query(fmt.Sprintf("time() - max by (cluster_name) (klio_plugin_backup_latest_completion_time_seconds{%s})", + clientMatcher), "{{cluster_name}}"), + ).Description("Elapsed time since the most recent base backup completed successfully, per cluster. A "+ + "value well above the backup interval means that cluster's backups have stopped succeeding.")), sized(4, panelHeight, statPanel("Time since last failed backup", "dtdurations", - query(fmt.Sprintf("time() - max(klio_plugin_backup_latest_failure_time_seconds{%s})", nsMatcher), - "since failure"), - ).Description("Elapsed time since the most recent base backup failure. A small value means a "+ - "failure happened recently.")), + query(fmt.Sprintf("time() - max by (cluster_name) (klio_plugin_backup_latest_failure_time_seconds{%s})", + clientMatcher), "{{cluster_name}}"), + ).Description("Elapsed time since the most recent base backup failure, per cluster. A small value means "+ + "a failure happened recently.")), sized(4, panelHeight, statPanel("Latest backup duration", "dtdurations", - query(fmt.Sprintf("max(klio_plugin_backup_latest_duration_seconds{%s})", nsMatcher), "latest duration"), - ).Description("Wall-clock duration of the most recent base backup.")), + query(fmt.Sprintf("max by (cluster_name) (klio_plugin_backup_latest_duration_seconds{%s})", clientMatcher), + "{{cluster_name}}"), + ).Description("Wall-clock duration of the most recent base backup, per cluster.")), sized(4, panelHeight, statPanel("Time since last backup started", "dtdurations", - query(fmt.Sprintf("time() - max(klio_plugin_backup_latest_start_time_seconds{%s})", nsMatcher), - "since start"), - ).Description("Elapsed time since the most recent base backup started. Compare against the latest "+ - "duration to tell whether a backup is still running or overdue.")), - // Derived: share of backup runs that succeeded over the selected range. + query(fmt.Sprintf("time() - max by (cluster_name) (klio_plugin_backup_latest_start_time_seconds{%s})", + clientMatcher), "{{cluster_name}}"), + ).Description("Elapsed time since the most recent base backup started, per cluster. Compare against the "+ + "latest duration to tell whether a backup is still running or overdue.")), + // Derived: share of backup runs that succeeded over the selected range, + // per cluster. sized(4, panelHeight, statPanel("Backup success ratio", "percentunit", query( - fmt.Sprintf("sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}[$__range])) / "+ - "clamp_min(sum(increase(klio_plugin_backup_runs_total{%s}[$__range])), 1)", nsMatcher, nsMatcher), - "success ratio", + fmt.Sprintf("sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}"+ + "[$__range])) / clamp_min(sum by (cluster_name) (increase(klio_plugin_backup_runs_total{%s}"+ + "[$__range])), 1)", clientMatcher, clientMatcher), + "{{cluster_name}}", ), ).Description("Fraction of base backup runs that succeeded over the selected time range "+ - "(successful runs / total runs).")), + "(successful runs / total runs), per cluster.")), // Derived: count of successful backups over fixed trailing windows. The // runs counter resets when the plugin sidecar restarts, so increase() // over long windows is an approximation across restarts. sized(4, panelHeight, statPanel("Successful backups (24h)", "none", - query(fmt.Sprintf("sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}[24h]))", nsMatcher), - "last 24h"), + query(fmt.Sprintf("sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}"+ + "[24h]))", clientMatcher), "{{cluster_name}}"), ).Decimals(0). - Description("Base backups that completed successfully in the last 24 hours. Counter resets on "+ - "plugin restart make long-window counts approximate.")), + Description("Base backups that completed successfully in the last 24 hours, per cluster. Counter "+ + "resets on plugin restart make long-window counts approximate.")), sized(4, panelHeight, statPanel("Successful backups (7d)", "none", - query(fmt.Sprintf("sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}[7d]))", nsMatcher), - "last 7d"), + query(fmt.Sprintf("sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}"+ + "[7d]))", clientMatcher), "{{cluster_name}}"), ).Decimals(0). - Description("Base backups that completed successfully in the last 7 days. Counter resets on "+ - "plugin restart make long-window counts approximate.")), + Description("Base backups that completed successfully in the last 7 days, per cluster. Counter "+ + "resets on plugin restart make long-window counts approximate.")), - // Backup throughput and outcomes. - sized(8, panelHeight, timeseriesPanel("Backup run rate by outcome", "ops", - query(fmt.Sprintf("sum by (outcome) (rate(klio_plugin_backup_runs_total{%s}[$__rate_interval]))", nsMatcher), - "{{outcome}}"), - ).Description("Rate of base backup runs, broken down by outcome (success or failure).")), - sized(8, panelHeight, timeseriesPanel("Backup failure rate by category", "ops", + // Backup throughput and outcomes, split by cluster and outcome/category. + sized(8, panelHeight, timeseriesPanel("Backup run rate by cluster and outcome", "ops", + query(fmt.Sprintf("sum by (cluster_name, outcome) (rate(klio_plugin_backup_runs_total{%s}"+ + "[$__rate_interval]))", clientMatcher), "{{cluster_name}} / {{outcome}}"), + ).Description("Rate of base backup runs, broken down by cluster and outcome (success or failure).")), + sized(8, panelHeight, timeseriesPanel("Backup failure rate by cluster and category", "ops", + query( + fmt.Sprintf("sum by (cluster_name, failure_category) "+ + "(rate(klio_plugin_backup_runs_total{outcome=\"failure\",%s}[$__rate_interval]))", clientMatcher), + "{{cluster_name}} / {{failure_category}}", + ), + ).Description("Rate of failed base backup runs, broken down by cluster and failure category, to show "+ + "why backups are failing.")), + // Distribution of backup wall-clock durations per cluster. The + // latest-duration stat tile above shows only the most recent run; these + // percentiles show the spread and let an admin see backup runtime + // trending up over time. + sized(8, panelHeight, timeseriesPanel("Backup duration p95 by cluster", "s", query( - fmt.Sprintf("sum by (failure_category) "+ - "(rate(klio_plugin_backup_runs_total{outcome=\"failure\",%s}[$__rate_interval]))", nsMatcher), - "{{failure_category}}", + fmt.Sprintf("histogram_quantile(0.95, sum by (le, cluster_name) "+ + "(increase(klio_plugin_backup_duration_seconds_bucket{%s}[$__range])))", clientMatcher), + "p95 {{cluster_name}}", ), - ).Description("Rate of failed base backup runs, broken down by failure category, to show why "+ - "backups are failing.")), - // Distribution of backup wall-clock durations. The latest-duration stat - // tile above shows only the most recent run; these percentiles show the - // spread and let an admin see backup runtime trending up over time. - sized(8, panelHeight, timeseriesPanel("Backup duration (p50/p95/p99)", "s", - quantileTargetsRange("klio_plugin_backup_duration_seconds_bucket", "le", nsMatcher, "")..., - ).Description("Percentile wall-clock duration of base backup runs (across all outcomes), computed "+ - "over the whole selected range so the lines stay populated between runs. Widen the dashboard "+ - "range to span several backups for a stable reading; if the range contains no backup, the "+ - "panel is empty.")), - // Backup volume over time: count of runs per bucket, split by outcome. - // Bars aggregate how many backups happened, which is more useful for - // infrequent backups than instantaneous duration percentiles. - sized(8, panelHeight, barPanel("Backups by outcome", "short", + ).Description("95th-percentile wall-clock duration of base backup runs (across all outcomes), per "+ + "cluster, computed over the whole selected range so the lines stay populated between runs. Widen "+ + "the dashboard range to span several backups for a stable reading; if the range contains no backup "+ + "for a cluster, that cluster's line is empty.")), + // Backup volume over time: count of runs per bucket, split by cluster + // and outcome. Bars aggregate how many backups happened, which is more + // useful for infrequent backups than instantaneous duration percentiles. + sized(8, panelHeight, barPanel("Backups by cluster and outcome", "short", query( - fmt.Sprintf("sum by (outcome) (increase(klio_plugin_backup_runs_total{%s}[$__interval]))", nsMatcher), - "{{outcome}}"), - ).Description("Count of base backup runs per time bucket, split by outcome. Bars aggregate the "+ - "number of backups over each interval (per day on a multi-day range).")), + fmt.Sprintf("sum by (cluster_name, outcome) (increase(klio_plugin_backup_runs_total{%s}"+ + "[$__interval]))", clientMatcher), "{{cluster_name}} / {{outcome}}"), + ).Description("Count of base backup runs per time bucket, split by cluster and outcome. Bars aggregate "+ + "the number of backups over each interval (per day on a multi-day range).")), - // WAL streaming client, run as a child process of this same sidecar. - sized(8, panelHeight, barGaugePanel("Streaming timeline by cluster", - query(fmt.Sprintf("max by (cluster_name) (klio_client_wal_timeline{%s})", clientWalMatcher), + // WAL streaming client, run as a child process of this same sidecar. A + // stepped time series shows when the streamed timeline changed (a + // failover), not just its current value. + sized(8, panelHeight, timelinePanel("Streaming timeline by cluster", + query(fmt.Sprintf("max by (cluster_name) (klio_client_wal_timeline{%s})", clientMatcher), "{{cluster_name}}"), - ).Description("PostgreSQL timeline the WAL streaming client is currently streaming, per cluster.")), + ).Description("PostgreSQL timeline the WAL streaming client is streaming, per cluster, over time. A "+ + "step up marks a failover on that cluster.")), sized(16, panelHeight, timeseriesPanel("WAL block send duration (p50/p95/p99) by cluster", "ns", quantileTargets("klio_client_wal_block_duration_nanoseconds_bucket", "le, cluster_name", - clientWalMatcher, "{{cluster_name}}")..., + clientMatcher, "{{cluster_name}}")..., ).Description("Percentile latency of the client's gRPC send of a WAL block to the server, per "+ "cluster. Most meaningful under active write load; on an idle or low-write cluster, WAL "+ "blocks are sent too infrequently for the underlying histogram_quantile to be reliable, so "+ diff --git a/observability/grafana/klio-dashboard.json b/observability/grafana/klio-dashboard.json index 90b2cc82..603c8b1e 100644 --- a/observability/grafana/klio-dashboard.json +++ b/observability/grafana/klio-dashboard.json @@ -39,14 +39,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(klio_plugin_backup_in_progress{k8s_namespace_name=~\"$namespace\"})", + "expr": "sum by (cluster_name) (klio_plugin_backup_in_progress{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "in progress", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Backups in progress", - "description": "Base backups currently running across the plugin sidecars in the namespace.", + "description": "Base backups currently running, per cluster.", "transparent": false, "datasource": { "type": "prometheus", @@ -97,14 +97,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_plugin_backup_latest_completion_time_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "time() - max by (cluster_name) (klio_plugin_backup_latest_completion_time_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "since success", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Time since last successful backup", - "description": "Elapsed time since the most recent base backup completed successfully. A value well above the backup interval means backups have stopped succeeding.", + "description": "Elapsed time since the most recent base backup completed successfully, per cluster. A value well above the backup interval means that cluster's backups have stopped succeeding.", "transparent": false, "datasource": { "type": "prometheus", @@ -154,14 +154,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_plugin_backup_latest_failure_time_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "time() - max by (cluster_name) (klio_plugin_backup_latest_failure_time_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "since failure", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Time since last failed backup", - "description": "Elapsed time since the most recent base backup failure. A small value means a failure happened recently.", + "description": "Elapsed time since the most recent base backup failure, per cluster. A small value means a failure happened recently.", "transparent": false, "datasource": { "type": "prometheus", @@ -211,14 +211,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_plugin_backup_latest_duration_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "max by (cluster_name) (klio_plugin_backup_latest_duration_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "latest duration", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Latest backup duration", - "description": "Wall-clock duration of the most recent base backup.", + "description": "Wall-clock duration of the most recent base backup, per cluster.", "transparent": false, "datasource": { "type": "prometheus", @@ -268,14 +268,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_plugin_backup_latest_start_time_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "time() - max by (cluster_name) (klio_plugin_backup_latest_start_time_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "since start", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Time since last backup started", - "description": "Elapsed time since the most recent base backup started. Compare against the latest duration to tell whether a backup is still running or overdue.", + "description": "Elapsed time since the most recent base backup started, per cluster. Compare against the latest duration to tell whether a backup is still running or overdue.", "transparent": false, "datasource": { "type": "prometheus", @@ -325,14 +325,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\"}[$__range])) / clamp_min(sum(increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\"}[$__range])), 1)", + "expr": "sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range])) / clamp_min(sum by (cluster_name) (increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range])), 1)", "instant": false, - "legendFormat": "success ratio", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Backup success ratio", - "description": "Fraction of base backup runs that succeeded over the selected time range (successful runs / total runs).", + "description": "Fraction of base backup runs that succeeded over the selected time range (successful runs / total runs), per cluster.", "transparent": false, "datasource": { "type": "prometheus", @@ -382,14 +382,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\"}[24h]))", + "expr": "sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[24h]))", "instant": false, - "legendFormat": "last 24h", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Successful backups (24h)", - "description": "Base backups that completed successfully in the last 24 hours. Counter resets on plugin restart make long-window counts approximate.", + "description": "Base backups that completed successfully in the last 24 hours, per cluster. Counter resets on plugin restart make long-window counts approximate.", "transparent": false, "datasource": { "type": "prometheus", @@ -440,14 +440,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\"}[7d]))", + "expr": "sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[7d]))", "instant": false, - "legendFormat": "last 7d", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Successful backups (7d)", - "description": "Base backups that completed successfully in the last 7 days. Counter resets on plugin restart make long-window counts approximate.", + "description": "Base backups that completed successfully in the last 7 days, per cluster. Counter resets on plugin restart make long-window counts approximate.", "transparent": false, "datasource": { "type": "prometheus", @@ -498,14 +498,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome) (rate(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, outcome) (rate(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{outcome}}", + "legendFormat": "{{cluster_name}} / {{outcome}}", "range": true } ], - "title": "Backup run rate by outcome", - "description": "Rate of base backup runs, broken down by outcome (success or failure).", + "title": "Backup run rate by cluster and outcome", + "description": "Rate of base backup runs, broken down by cluster and outcome (success or failure).", "transparent": false, "datasource": { "type": "prometheus", @@ -549,14 +549,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (failure_category) (rate(klio_plugin_backup_runs_total{outcome=\"failure\",k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, failure_category) (rate(klio_plugin_backup_runs_total{outcome=\"failure\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{failure_category}}", + "legendFormat": "{{cluster_name}} / {{failure_category}}", "range": true } ], - "title": "Backup failure rate by category", - "description": "Rate of failed base backup runs, broken down by failure category, to show why backups are failing.", + "title": "Backup failure rate by cluster and category", + "description": "Rate of failed base backup runs, broken down by cluster and failure category, to show why backups are failing.", "transparent": false, "datasource": { "type": "prometheus", @@ -600,34 +600,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le) (increase(klio_plugin_backup_duration_seconds_bucket{k8s_namespace_name=~\"$namespace\"}[$__range])))", - "instant": false, - "legendFormat": "p50", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "histogram_quantile(0.95, sum by (le) (increase(klio_plugin_backup_duration_seconds_bucket{k8s_namespace_name=~\"$namespace\"}[$__range])))", - "instant": false, - "legendFormat": "p95", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "histogram_quantile(0.99, sum by (le) (increase(klio_plugin_backup_duration_seconds_bucket{k8s_namespace_name=~\"$namespace\"}[$__range])))", + "expr": "histogram_quantile(0.95, sum by (le, cluster_name) (increase(klio_plugin_backup_duration_seconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range])))", "instant": false, - "legendFormat": "p99", + "legendFormat": "p95 {{cluster_name}}", "range": true } ], - "title": "Backup duration (p50/p95/p99)", - "description": "Percentile wall-clock duration of base backup runs (across all outcomes), computed over the whole selected range so the lines stay populated between runs. Widen the dashboard range to span several backups for a stable reading; if the range contains no backup, the panel is empty.", + "title": "Backup duration p95 by cluster", + "description": "95th-percentile wall-clock duration of base backup runs (across all outcomes), per cluster, computed over the whole selected range so the lines stay populated between runs. Widen the dashboard range to span several backups for a stable reading; if the range contains no backup for a cluster, that cluster's line is empty.", "transparent": false, "datasource": { "type": "prometheus", @@ -671,14 +651,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome) (increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\"}[$__interval]))", + "expr": "sum by (cluster_name, outcome) (increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__interval]))", "instant": false, - "legendFormat": "{{outcome}}", + "legendFormat": "{{cluster_name}} / {{outcome}}", "range": true } ], - "title": "Backups by outcome", - "description": "Count of base backup runs per time bucket, split by outcome. Bars aggregate the number of backups over each interval (per day on a multi-day range).", + "title": "Backups by cluster and outcome", + "description": "Count of base backup runs per time bucket, split by cluster and outcome. Bars aggregate the number of backups over each interval (per day on a multi-day range).", "transparent": false, "datasource": { "type": "prometheus", @@ -720,7 +700,7 @@ } }, { - "type": "bargauge", + "type": "timeseries", "targets": [ { "datasource": { @@ -734,7 +714,7 @@ } ], "title": "Streaming timeline by cluster", - "description": "PostgreSQL timeline the WAL streaming client is currently streaming, per cluster.", + "description": "PostgreSQL timeline the WAL streaming client is streaming, per cluster, over time. A step up marks a failover on that cluster.", "transparent": false, "datasource": { "type": "prometheus", @@ -748,31 +728,27 @@ }, "repeatDirection": "h", "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", - "showLegend": false, + "showLegend": true, "calcs": [] }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" + "tooltip": { + "mode": "", + "sort": "" + } }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0 + "unit": "short", + "decimals": 0, + "custom": { + "gradientMode": "opacity", + "lineWidth": 2, + "lineInterpolation": "stepAfter", + "fillOpacity": 0 + } }, "overrides": [] } @@ -869,14 +845,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_uptime_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "max by (service_name) (klio_server_uptime_seconds{service_name=~\"$server\"})", "instant": false, - "legendFormat": "uptime", + "legendFormat": "{{service_name}}", "range": true } ], "title": "Server uptime", - "description": "Time since the Klio server process started. A sudden drop means the server StatefulSet restarted.", + "description": "Time since the Klio server process started, per server. A sudden drop means that server's StatefulSet restarted.", "transparent": false, "datasource": { "type": "prometheus", @@ -919,21 +895,21 @@ } }, { - "type": "bargauge", + "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (tier) (klio_server_wal_latest_written_timeline{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_wal_latest_written_timeline{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Latest WAL timeline by tier", - "description": "Current PostgreSQL timeline of the latest WAL written per tier. A change reflects a promotion or failover.", + "title": "Latest WAL timeline by cluster and tier", + "description": "PostgreSQL timeline of the latest WAL written per cluster and tier, over time. A step up marks a promotion or failover.", "transparent": false, "datasource": { "type": "prometheus", @@ -941,37 +917,33 @@ }, "gridPos": { "h": 6, - "w": 4, + "w": 8, "x": 4, "y": 26 }, "repeatDirection": "h", "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", - "showLegend": false, + "showLegend": true, "calcs": [] }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" + "tooltip": { + "mode": "", + "sort": "" + } }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0 + "unit": "short", + "decimals": 0, + "custom": { + "gradientMode": "opacity", + "lineWidth": 2, + "lineInterpolation": "stepAfter", + "fillOpacity": 0 + } }, "overrides": [] } @@ -984,14 +956,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (klio_server_backup_snapshots{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "sum by (cluster, tier) (label_replace(klio_server_backup_snapshots{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], - "title": "Base snapshots by tier", - "description": "Base backup snapshots currently retained per tier.", + "title": "Base snapshots by cluster and tier", + "description": "Base backup snapshots currently retained per cluster and tier (the cluster is derived from the Kopia snapshot source).", "transparent": false, "datasource": { "type": "prometheus", @@ -1000,7 +972,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 8, + "x": 12, "y": 26 }, "repeatDirection": "h", @@ -1042,14 +1014,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_backup_latest_snapshot_size_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_size_bytes{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "latest size", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], "title": "Latest snapshot size", - "description": "Size on the backend of the most recent base backup snapshot.", + "description": "Size on the backend of the most recent base backup snapshot, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1058,7 +1030,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 12, + "x": 16, "y": 26 }, "repeatDirection": "h", @@ -1099,14 +1071,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_backup_latest_snapshot_files{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_files{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "latest files", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], "title": "Latest snapshot files", - "description": "Number of files in the most recent base backup snapshot.", + "description": "Number of files in the most recent base backup snapshot, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1115,7 +1087,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 16, + "x": 20, "y": 26 }, "repeatDirection": "h", @@ -1157,14 +1129,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_backup_latest_snapshot_dirs{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_dirs{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "latest dirs", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], "title": "Latest snapshot dirs", - "description": "Number of directories in the most recent base backup snapshot.", + "description": "Number of directories in the most recent base backup snapshot, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1173,8 +1145,8 @@ "gridPos": { "h": 6, "w": 4, - "x": 20, - "y": 26 + "x": 0, + "y": 32 }, "repeatDirection": "h", "options": { @@ -1215,14 +1187,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_server_backup_latest_snapshot_timestamp_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "time() - max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_timestamp_seconds{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "latest age", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], "title": "Latest snapshot age", - "description": "Age of the most recent base backup snapshot. Should stay below the backup interval.", + "description": "Age of the most recent base backup snapshot, per cluster and tier. Should stay below the backup interval; a tier-2 value drifting above tier-1 means remote relay is lagging.", "transparent": false, "datasource": { "type": "prometheus", @@ -1231,7 +1203,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 0, + "x": 4, "y": 32 }, "repeatDirection": "h", @@ -1272,14 +1244,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - min(klio_server_backup_oldest_snapshot_timestamp_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "time() - min by (cluster, tier) (label_replace(klio_server_backup_oldest_snapshot_timestamp_seconds{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "oldest age", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], "title": "Oldest snapshot age", - "description": "Age of the oldest retained base backup snapshot, reflecting the effective retention horizon.", + "description": "Age of the oldest retained base backup snapshot, per cluster and tier, reflecting each tier's effective retention horizon.", "transparent": false, "datasource": { "type": "prometheus", @@ -1288,7 +1260,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 4, + "x": 8, "y": 32 }, "repeatDirection": "h", @@ -1329,14 +1301,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_server_backup_latest_backup_start_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "time() - max by (cluster_name, tier) (klio_server_backup_latest_backup_start_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "latest start age", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], "title": "Latest backup age (start)", - "description": "Elapsed time since the most recently retained PostgreSQL backup started.", + "description": "Elapsed time since the most recently retained PostgreSQL backup started, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1345,7 +1317,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 8, + "x": 12, "y": 32 }, "repeatDirection": "h", @@ -1386,14 +1358,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_server_backup_latest_backup_completion_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "time() - max by (cluster_name, tier) (klio_server_backup_latest_backup_completion_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "latest completion age", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], "title": "Latest backup age (completion)", - "description": "Elapsed time since the most recently retained PostgreSQL backup completed.", + "description": "Elapsed time since the most recently retained PostgreSQL backup completed, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1402,7 +1374,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 12, + "x": 16, "y": 32 }, "repeatDirection": "h", @@ -1443,14 +1415,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - min(klio_server_backup_oldest_backup_start_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "time() - min by (cluster_name, tier) (klio_server_backup_oldest_backup_start_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "oldest start age", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], "title": "Oldest backup age (start)", - "description": "Elapsed time since the oldest retained PostgreSQL backup started, reflecting the effective retention horizon.", + "description": "Elapsed time since the oldest retained PostgreSQL backup started, per cluster and tier, reflecting each tier's effective retention horizon.", "transparent": false, "datasource": { "type": "prometheus", @@ -1459,7 +1431,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 16, + "x": 20, "y": 32 }, "repeatDirection": "h", @@ -1500,14 +1472,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - min(klio_server_backup_oldest_backup_completion_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "time() - min by (cluster_name, tier) (klio_server_backup_oldest_backup_completion_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "oldest completion age", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], "title": "Oldest backup age (completion)", - "description": "Elapsed time since the oldest retained PostgreSQL backup completed, reflecting the effective retention horizon.", + "description": "Elapsed time since the oldest retained PostgreSQL backup completed, per cluster and tier, reflecting each tier's effective retention horizon.", "transparent": false, "datasource": { "type": "prometheus", @@ -1515,9 +1487,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 20, - "y": 32 + "w": 6, + "x": 0, + "y": 38 }, "repeatDirection": "h", "options": { @@ -1557,14 +1529,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (klio_server_backup_backups{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "sum by (cluster_name, tier) (klio_server_backup_backups{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "PostgreSQL backups retained by tier", - "description": "Number of PostgreSQL backups currently retained per tier, across clusters.", + "title": "PostgreSQL backups retained by cluster and tier", + "description": "Number of PostgreSQL backups currently retained per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1572,8 +1544,8 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 0, + "w": 9, + "x": 6, "y": 38 }, "repeatDirection": "h", @@ -1615,9 +1587,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_latest_backup_start_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_backup_latest_backup_start_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} start", + "legendFormat": "{{cluster_name}} {{tier}} start", "range": true }, { @@ -1625,14 +1597,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_latest_backup_end_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_backup_latest_backup_end_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} end", + "legendFormat": "{{cluster_name}} {{tier}} end", "range": true } ], - "title": "Latest backup LSN by cluster", - "description": "Start and end LSN of the latest retained PostgreSQL backup, per cluster.", + "title": "Latest backup LSN by cluster and tier", + "description": "Start and end LSN of the latest retained PostgreSQL backup, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1640,8 +1612,8 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 8, + "w": 9, + "x": 15, "y": 38 }, "repeatDirection": "h", @@ -1676,9 +1648,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_oldest_backup_start_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_backup_oldest_backup_start_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} start", + "legendFormat": "{{cluster_name}} {{tier}} start", "range": true }, { @@ -1686,14 +1658,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_oldest_backup_end_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_backup_oldest_backup_end_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} end", + "legendFormat": "{{cluster_name}} {{tier}} end", "range": true } ], - "title": "Oldest backup LSN by cluster", - "description": "Start and end LSN of the oldest retained PostgreSQL backup, per cluster.", + "title": "Oldest backup LSN by cluster and tier", + "description": "Start and end LSN of the oldest retained PostgreSQL backup, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1702,8 +1674,8 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 38 + "x": 0, + "y": 44 }, "repeatDirection": "h", "options": { @@ -1730,16 +1702,16 @@ } }, { - "type": "bargauge", + "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_latest_backup_timeline{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_backup_latest_backup_timeline{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} latest", + "legendFormat": "{{cluster_name}} {{tier}} latest", "range": true }, { @@ -1747,14 +1719,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_oldest_backup_timeline{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_backup_oldest_backup_timeline{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} oldest", + "legendFormat": "{{cluster_name}} {{tier}} oldest", "range": true } ], - "title": "Backup timeline by cluster", - "description": "PostgreSQL timeline of the latest and oldest retained backup, per cluster. A mismatch means the retention window spans a promotion or failover.", + "title": "Backup timeline by cluster and tier", + "description": "PostgreSQL timeline of the latest and oldest retained backup, per cluster and tier, over time. The latest and oldest lines diverging means the retention window spans a promotion or failover.", "transparent": false, "datasource": { "type": "prometheus", @@ -1763,36 +1735,32 @@ "gridPos": { "h": 6, "w": 8, - "x": 0, + "x": 8, "y": 44 }, "repeatDirection": "h", "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", - "showLegend": false, + "showLegend": true, "calcs": [] }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" + "tooltip": { + "mode": "", + "sort": "" + } }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0 + "unit": "short", + "decimals": 0, + "custom": { + "gradientMode": "opacity", + "lineWidth": 2, + "lineInterpolation": "stepAfter", + "fillOpacity": 0 + } }, "overrides": [] } @@ -1805,14 +1773,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (rate(klio_server_wal_written_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, tier) (rate(klio_server_wal_written_total{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "WAL files written rate by tier", - "description": "Rate of WAL files written by the server, split by storage tier.", + "title": "WAL files written rate by cluster and tier", + "description": "Rate of WAL files written by the server, split by cluster and storage tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1821,7 +1789,7 @@ "gridPos": { "h": 6, "w": 8, - "x": 8, + "x": 16, "y": 44 }, "repeatDirection": "h", @@ -1856,14 +1824,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (rate(klio_server_wal_written_size_bytes_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, tier) (rate(klio_server_wal_written_size_bytes_total{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "WAL bytes written rate by tier", - "description": "Rate of WAL bytes written by the server, split by storage tier.", + "title": "WAL bytes written rate by cluster and tier", + "description": "Rate of WAL bytes written by the server, split by cluster and storage tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1872,8 +1840,8 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 44 + "x": 0, + "y": 50 }, "repeatDirection": "h", "options": { @@ -1907,14 +1875,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max by (tier) (klio_server_wal_latest_written_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "time() - max by (cluster_name, tier) (klio_server_wal_latest_written_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Time since last WAL written by tier", - "description": "Elapsed time since the server last wrote a WAL file for each tier. A stale tier-1 value means PostgreSQL stopped shipping WALs; a stale tier-2 value means the remote backend stopped receiving them.", + "title": "Time since last WAL written by cluster and tier", + "description": "Elapsed time since the server last wrote a WAL file for each cluster and tier. A stale tier-1 value means PostgreSQL stopped shipping WALs; a stale tier-2 value means the remote backend stopped receiving them.", "transparent": false, "datasource": { "type": "prometheus", @@ -1923,7 +1891,7 @@ "gridPos": { "h": 6, "w": 8, - "x": 0, + "x": 8, "y": 50 }, "repeatDirection": "h", @@ -1958,14 +1926,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (tier) (klio_server_wal_latest_written_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster_name, tier) (klio_server_wal_latest_written_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Latest written LSN by tier", - "description": "Most recent WAL LSN the server has written for each tier, as a byte offset.", + "title": "Latest written LSN by cluster and tier", + "description": "Most recent WAL LSN the server has written for each cluster and tier, as a byte offset.", "transparent": false, "datasource": { "type": "prometheus", @@ -1974,7 +1942,7 @@ "gridPos": { "h": 6, "w": 8, - "x": 8, + "x": 16, "y": 50 }, "repeatDirection": "h", @@ -2009,14 +1977,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome, tier) (rate(klio_server_backup_verifications_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"}[$__rate_interval]))", + "expr": "sum by (service_name, outcome, tier) (rate(klio_server_backup_verifications_total{service_name=~\"$server\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{tier}} / {{outcome}}", + "legendFormat": "{{service_name}} {{tier}} / {{outcome}}", "range": true } ], "title": "Backup verification rate by outcome and tier", - "description": "Rate of base backup verification checks, broken down by outcome and tier.", + "description": "Rate of base backup verification checks, broken down by outcome and tier (the verification counter carries no cluster_name, so it is a per-server signal).", "transparent": false, "datasource": { "type": "prometheus", @@ -2025,8 +1993,8 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 50 + "x": 0, + "y": 56 }, "repeatDirection": "h", "options": { @@ -2060,7 +2028,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p50 {{path}}/{{stage}}", "range": true @@ -2070,7 +2038,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p95 {{path}}/{{stage}}", "range": true @@ -2080,14 +2048,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p99 {{path}}/{{stage}}", "range": true } ], "title": "WAL block duration (p50/p95/p99) by path and stage", - "description": "Percentile per-block WAL processing duration on the server, split by `path` (put ingest / get serve) and `stage`.", + "description": "Percentile per-block WAL processing duration on the server, split by `path` (put ingest / get serve) and `stage`, aggregated across the selected clusters.", "transparent": false, "datasource": { "type": "prometheus", @@ -2096,7 +2064,7 @@ "gridPos": { "h": 6, "w": 8, - "x": 0, + "x": 8, "y": 56 }, "repeatDirection": "h", @@ -2131,7 +2099,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p50 {{tier}}", "range": true @@ -2141,7 +2109,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p95 {{tier}}", "range": true @@ -2151,14 +2119,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p99 {{tier}}", "range": true } ], "title": "WAL file get duration (p50/p95/p99) by tier", - "description": "Percentile duration of a complete WAL file gRPC get, split by the tier that served it.", + "description": "Percentile duration of a complete WAL file gRPC get, split by the tier that served it, aggregated across the selected clusters.", "transparent": false, "datasource": { "type": "prometheus", @@ -2167,7 +2135,7 @@ "gridPos": { "h": 6, "w": 8, - "x": 8, + "x": 16, "y": 56 }, "repeatDirection": "h", @@ -2202,7 +2170,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p50 {{cluster_name}}", "range": true @@ -2212,7 +2180,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p95 {{cluster_name}}", "range": true @@ -2222,7 +2190,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, "legendFormat": "p99 {{cluster_name}}", "range": true @@ -2237,9 +2205,9 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 16, - "y": 56 + "w": 10, + "x": 0, + "y": 62 }, "repeatDirection": "h", "options": { @@ -2273,14 +2241,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome) (rate(klio_server_backup_relay_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, outcome) (rate(klio_server_backup_relay_total{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{outcome}}", + "legendFormat": "{{cluster_name}} / {{outcome}}", "range": true } ], - "title": "Tier-2 relay rate by outcome", - "description": "Rate of tier-2 relay attempts after a backup (migration and verification), by outcome.", + "title": "Tier-2 relay rate by cluster and outcome", + "description": "Rate of tier-2 relay attempts after a backup (migration and verification), per cluster and outcome.", "transparent": false, "datasource": { "type": "prometheus", @@ -2288,8 +2256,8 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 0, + "w": 14, + "x": 10, "y": 62 }, "repeatDirection": "h", @@ -2324,14 +2292,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier, outcome) (rate(klio_server_backup_maintenance_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, tier, outcome) (rate(klio_server_backup_maintenance_total{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{tier}} / {{outcome}}", + "legendFormat": "{{cluster_name}} {{tier}}/{{outcome}}", "range": true } ], - "title": "Maintenance run rate by tier and outcome", - "description": "Rate of post-backup maintenance runs (base-snapshot retention and WAL cleanup), by tier and outcome.", + "title": "Maintenance run rate by cluster, tier and outcome", + "description": "Rate of post-backup maintenance runs (base-snapshot retention and WAL cleanup), per cluster, tier and outcome.", "transparent": false, "datasource": { "type": "prometheus", @@ -2339,9 +2307,9 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 62 + "w": 14, + "x": 0, + "y": 68 }, "repeatDirection": "h", "options": { @@ -2375,14 +2343,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (stream) (klio_server_queue_messages{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "sum by (service_name, stream) (klio_server_queue_messages{service_name=~\"$server\"})", "instant": false, - "legendFormat": "{{stream}}", + "legendFormat": "{{service_name}} / {{stream}}", "range": true } ], "title": "Queue messages by stream", - "description": "Messages currently held in each NATS JetStream stream of the embedded queue.", + "description": "Messages currently held in each NATS JetStream stream of the embedded queue, per server.", "transparent": false, "datasource": { "type": "prometheus", @@ -2390,8 +2358,8 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 0, + "w": 10, + "x": 14, "y": 68 }, "repeatDirection": "h", @@ -2426,14 +2394,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (stream) (klio_server_queue_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "sum by (service_name, stream) (klio_server_queue_bytes{service_name=~\"$server\"})", "instant": false, - "legendFormat": "{{stream}}", + "legendFormat": "{{service_name}} / {{stream}}", "range": true } ], "title": "Queue bytes by stream", - "description": "Bytes currently held in each NATS JetStream stream of the embedded queue.", + "description": "Bytes currently held in each NATS JetStream stream of the embedded queue, per server.", "transparent": false, "datasource": { "type": "prometheus", @@ -2441,9 +2409,9 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 68 + "w": 24, + "x": 0, + "y": 74 }, "repeatDirection": "h", "options": { @@ -2477,7 +2445,7 @@ "h": 1, "w": 24, "x": 0, - "y": 74 + "y": 80 }, "id": 0, "panels": [] @@ -2507,7 +2475,7 @@ "h": 6, "w": 8, "x": 0, - "y": 75 + "y": 81 }, "repeatDirection": "h", "options": { @@ -2558,7 +2526,7 @@ "h": 6, "w": 8, "x": 8, - "y": 75 + "y": 81 }, "repeatDirection": "h", "options": { @@ -2592,14 +2560,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "(klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"} - on (cluster_name) klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}) / 1024 / 1024", + "expr": "(max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",service_name=~\"$server\",cluster_name=~\"$cluster\"}) - max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",service_name=~\"$server\",cluster_name=~\"$cluster\"})) / 1024 / 1024", "instant": false, "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Tier-2 archival lag (Tier-1 LSN gap)", - "description": "LSN distance between tier 1 (local disk) and tier 2 (remote storage) per cluster. A growing gap means remote archival is falling behind.", + "description": "LSN distance between tier 1 (local disk) and tier 2 (remote storage) per cluster. A growing gap means remote archival is falling behind. Scoped by $server and $cluster (not $namespace), since the server tags these series with its own namespace.", "transparent": false, "datasource": { "type": "prometheus", @@ -2609,7 +2577,7 @@ "h": 6, "w": 8, "x": 16, - "y": 75 + "y": 81 }, "repeatDirection": "h", "options": { @@ -2685,7 +2653,7 @@ "name": "server", "label": "Server", "skipUrlSync": false, - "query": "label_values(klio_server_uptime_seconds{k8s_namespace_name=~\"$namespace\"}, host_name)", + "query": "label_values(klio_server_uptime_seconds, service_name)", "datasource": { "type": "prometheus", "uid": "${datasource}" @@ -2710,7 +2678,7 @@ "name": "cluster", "label": "Cluster", "skipUrlSync": false, - "query": "label_values(klio_server_wal_written_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"}, cluster_name)", + "query": "label_values(klio_server_wal_written_total{service_name=~\"$server\"}, cluster_name)", "datasource": { "type": "prometheus", "uid": "${datasource}" diff --git a/observability/grafana/main.go b/observability/grafana/main.go index 0f0e1fa0..d82a11ea 100644 --- a/observability/grafana/main.go +++ b/observability/grafana/main.go @@ -58,17 +58,25 @@ const ( // dashboard grid packs flush without vertical gaps. panelHeight = 6 - // nsMatcher is the namespace label selector every klio metric carries - // (the Prometheus export of the k8s.namespace.name resource attribute). - nsMatcher = `k8s_namespace_name=~"$namespace"` - // serverMatcher additionally selects on host_name, which the klio_server_* - // series carry as the Klio server host (the StatefulSet pod), and combines - // it with the namespace selector. - serverMatcher = `k8s_namespace_name=~"$namespace",host_name=~"$server"` - // walMatcher additionally selects on cluster_name, which only the - // klio_server_wal_* series carry, and combines it with the namespace and - // server selectors. - walMatcher = `k8s_namespace_name=~"$namespace",host_name=~"$server",cluster_name=~"$cluster"` + // clientMatcher selects the plugin sidecar's per-cluster series + // (klio_plugin_backup_* and klio_client_wal_*). These carry the PostgreSQL + // pod's k8s.namespace.name and a cluster_name, so they are scoped by + // $namespace and $cluster. Because they identify their cluster directly, + // several clusters sharing one namespace never fold into a single series. + clientMatcher = `k8s_namespace_name=~"$namespace",cluster_name=~"$cluster"` + // serverMatcher selects the server-level klio_server_* series that carry no + // cluster_name (uptime, the embedded queue, verifications and the Kopia base + // snapshots) by the Klio server's OpenTelemetry service.name. service.name + // identifies a server uniquely even when two servers share a pod host name + // (host_name collides when two Servers have the same name in different + // namespaces); k8s.namespace.name is deliberately NOT used here because a + // server tags every series with its OWN namespace, which would hide the + // metrics of a cluster it serves cross-namespace. + serverMatcher = `service_name=~"$server"` + // walMatcher additionally selects on cluster_name, which the per-cluster + // klio_server_wal_* and klio_server_backup_* (backups / latest_backup_* / + // oldest_backup_*) series carry, and combines it with the server selector. + walMatcher = `service_name=~"$server",cluster_name=~"$cluster"` ) func main() { @@ -172,16 +180,6 @@ func quantileTargets(bucketMetric, groupBy, matcher, legend string) []cog.Builde return quantileTargetsWindow(bucketMetric, groupBy, matcher, legend, "rate", "$__rate_interval") } -// quantileTargetsRange builds p50/p95/p99 targets for an infrequent-event -// histogram (e.g. backups), using increase() over the whole visible range -// ($__range). A short rate() window almost never catches a rare event, so the -// percentiles would otherwise collapse to no data except at the instant the -// event fires; the range window keeps them populated whenever the selected -// range spans at least one event. -func quantileTargetsRange(bucketMetric, groupBy, matcher, legend string) []cog.Builder[variants.Dataquery] { - return quantileTargetsWindow(bucketMetric, groupBy, matcher, legend, "increase", "$__range") -} - // tableLegend renders a compact table legend at the bottom of a panel. func tableLegend() *common.VizLegendOptionsBuilder { return common.NewVizLegendOptionsBuilder(). @@ -213,6 +211,18 @@ func timeseriesPanel(title, unit string, targets ...cog.Builder[variants.Dataque return panel } +// timelinePanel renders an integer PostgreSQL timeline ID over time as a +// stepped line, so an operator sees not only the current timeline but exactly +// when it changed (a promotion or failover), which a single current value (a +// stat or bar gauge) cannot convey. The unit is a plain count with no decimals. +func timelinePanel(title string, targets ...cog.Builder[variants.Dataquery]) *timeseries.PanelBuilder { + return timeseriesPanel(title, "short", targets...). + FillOpacity(0). + LineInterpolation(common.LineInterpolationStepAfter). + LineWidth(2). + Decimals(0) +} + // barPanel builds a stacked bar chart (a timeseries in bar draw style) for // counting discrete events over time, such as backups per bucket. func barPanel(title, unit string, targets ...cog.Builder[variants.Dataquery]) *timeseries.PanelBuilder { diff --git a/observability/grafana/replication.go b/observability/grafana/replication.go index 42f9ccd3..ecfc473c 100644 --- a/observability/grafana/replication.go +++ b/observability/grafana/replication.go @@ -56,19 +56,20 @@ func replicationPanels() []sizedPanel { "corresponding WAL, from CloudNativePG's pg_stat_replication.")), // Derived: tier-2 archival backlog as the LSN gap between what the // server has on local disk (tier1) and what has been archived remotely - // (tier2), in MiB. max by (cluster_name) collapses each tier to one - // series per cluster so the subtraction is one-to-one even when several - // namespaces or server instances export the metric. + // (tier2), in MiB. Each side is reduced with max by (cluster_name) first + // so the subtraction is a one-to-one vector match on cluster_name even + // when several servers export the same cluster's WAL series. sized(8, panelHeight, timeseriesPanel("Tier-2 archival lag (Tier-1 LSN gap)", "mbytes", query( fmt.Sprintf( - "(klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",%s} - "+ - "on (cluster_name) klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",%s}) "+ + "(max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",%s}) - "+ + "max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",%s})) "+ "/ 1024 / 1024", walMatcher, walMatcher), "{{cluster_name}}", ), ).Description("LSN distance between tier 1 (local disk) and tier 2 (remote storage) per cluster. "+ - "A growing gap means remote archival is falling behind.")), + "A growing gap means remote archival is falling behind. Scoped by $server and $cluster (not "+ + "$namespace), since the server tags these series with its own namespace.")), } } diff --git a/observability/grafana/server.go b/observability/grafana/server.go index 723babc2..13f2b447 100644 --- a/observability/grafana/server.go +++ b/observability/grafana/server.go @@ -23,166 +23,208 @@ import ( "fmt" ) +// snapshotCluster wraps a Kopia base-snapshot selector in a label_replace that +// derives a `cluster` label from the snapshot_source attribute (formatted +// `userName@clusterName:path`). The per-source snapshot gauges carry +// snapshot_source and tier but no cluster_name, so this lets them be grouped +// per PostgreSQL cluster instead of folding every cluster on a server into one +// value. +func snapshotCluster(selector string) string { + return fmt.Sprintf(`label_replace(%s, "cluster", "$1", "snapshot_source", "[^@]*@([^:]+):.*")`, selector) +} + // serverPanels returns the "Server" section panels. These metrics are emitted // by the Klio server StatefulSet (the `klio.server.*` family, exported to // Prometheus as `klio_server_*`): WAL ingest, backup verification, base // snapshots, the retention window of physical PostgreSQL backups, and the -// embedded NATS JetStream queue. Queries are scoped by $namespace and -// $server; the WAL and PostgreSQL-backup series additionally carry -// cluster_name and are scoped by $cluster. +// embedded NATS JetStream queue. Server-level series (uptime, verifications, +// snapshots, queue) carry no cluster_name and are grouped by service_name (the +// server identity, scoped by $server); the per-cluster WAL and PostgreSQL-backup +// series carry cluster_name and are additionally scoped by $cluster. func serverPanels() []sizedPanel { return []sizedPanel{ // Compact single/low-cardinality values first (stat tiles and bar - // gauges), then the wider time series. + // gauges), then the wider time series. Server-level values are grouped + // by service_name so two servers (even two with the same pod host name + // in different namespaces) never collapse into one number. sized(4, panelHeight, statPanel("Server uptime", "dtdurations", - query(fmt.Sprintf("max(klio_server_uptime_seconds{%s})", serverMatcher), "uptime"), - ).Description("Time since the Klio server process started. A sudden drop means the server "+ - "StatefulSet restarted.")), - // A bar gauge shows one labeled bar per tier, so both tiers are always - // visible (a narrow stat tile hides all but the first series). - sized(4, panelHeight, barGaugePanel("Latest WAL timeline by tier", - query(fmt.Sprintf("max by (tier) (klio_server_wal_latest_written_timeline{%s})", walMatcher), "{{tier}}"), - ).Description("Current PostgreSQL timeline of the latest WAL written per tier. A change reflects "+ - "a promotion or failover.")), - sized(4, panelHeight, barGaugePanel("Base snapshots by tier", - query(fmt.Sprintf("sum by (tier) (klio_server_backup_snapshots{%s})", serverMatcher), "{{tier}}"), - ).Description("Base backup snapshots currently retained per tier.")), + query(fmt.Sprintf("max by (service_name) (klio_server_uptime_seconds{%s})", serverMatcher), + "{{service_name}}"), + ).Description("Time since the Klio server process started, per server. A sudden drop means that "+ + "server's StatefulSet restarted.")), + // A stepped time series shows the timeline per cluster/tier over time, so + // a promotion or failover is visible as the step where the line jumps, + // not just the value it currently sits at. + sized(8, panelHeight, timelinePanel("Latest WAL timeline by cluster and tier", + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_wal_latest_written_timeline{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("PostgreSQL timeline of the latest WAL written per cluster and tier, over time. A step "+ + "up marks a promotion or failover.")), + sized(4, panelHeight, barGaugePanel("Base snapshots by cluster and tier", + query(fmt.Sprintf("sum by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_snapshots{%s}", serverMatcher))), "{{cluster}} {{tier}}"), + ).Description("Base backup snapshots currently retained per cluster and tier (the cluster is derived "+ + "from the Kopia snapshot source).")), sized(4, panelHeight, statPanel("Latest snapshot size", "bytes", - query(fmt.Sprintf("max(klio_server_backup_latest_snapshot_size_bytes{%s})", serverMatcher), "latest size"), - ).Description("Size on the backend of the most recent base backup snapshot.")), + query(fmt.Sprintf("max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_size_bytes{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), + ).Description("Size on the backend of the most recent base backup snapshot, per cluster and tier.")), sized(4, panelHeight, statPanel("Latest snapshot files", "short", - query(fmt.Sprintf("max(klio_server_backup_latest_snapshot_files{%s})", serverMatcher), "latest files"), + query(fmt.Sprintf("max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_files{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), ).Decimals(0). - Description("Number of files in the most recent base backup snapshot.")), + Description("Number of files in the most recent base backup snapshot, per cluster and tier.")), sized(4, panelHeight, statPanel("Latest snapshot dirs", "short", - query(fmt.Sprintf("max(klio_server_backup_latest_snapshot_dirs{%s})", serverMatcher), "latest dirs"), + query(fmt.Sprintf("max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_dirs{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), ).Decimals(0). - Description("Number of directories in the most recent base backup snapshot.")), + Description("Number of directories in the most recent base backup snapshot, per cluster and tier.")), sized(4, panelHeight, statPanel("Latest snapshot age", "dtdurations", - query(fmt.Sprintf("time() - max(klio_server_backup_latest_snapshot_timestamp_seconds{%s})", serverMatcher), - "latest age"), - ).Description("Age of the most recent base backup snapshot. Should stay below the backup interval.")), + query(fmt.Sprintf("time() - max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_timestamp_seconds{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), + ).Description("Age of the most recent base backup snapshot, per cluster and tier. Should stay below "+ + "the backup interval; a tier-2 value drifting above tier-1 means remote relay is lagging.")), sized(4, panelHeight, statPanel("Oldest snapshot age", "dtdurations", - query(fmt.Sprintf("time() - min(klio_server_backup_oldest_snapshot_timestamp_seconds{%s})", serverMatcher), - "oldest age"), - ).Description("Age of the oldest retained base backup snapshot, reflecting the effective "+ - "retention horizon.")), + query(fmt.Sprintf("time() - min by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_oldest_snapshot_timestamp_seconds{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), + ).Description("Age of the oldest retained base backup snapshot, per cluster and tier, reflecting each "+ + "tier's effective retention horizon.")), // Retention window of the physical PostgreSQL backups (distinct from // the Kopia base-snapshot gauges above): the klio.server.backup.backups - // / latest_backup_* / oldest_backup_* family, scoped by cluster_name - // via walMatcher. + // / latest_backup_* / oldest_backup_* family, which carry cluster_name + // and are scoped by cluster_name via walMatcher. sized(4, panelHeight, statPanel("Latest backup age (start)", "dtdurations", - query(fmt.Sprintf("time() - max(klio_server_backup_latest_backup_start_time_seconds{%s})", walMatcher), - "latest start age"), - ).Description("Elapsed time since the most recently retained PostgreSQL backup started.")), + query(fmt.Sprintf("time() - max by (cluster_name, tier) "+ + "(klio_server_backup_latest_backup_start_time_seconds{%s})", walMatcher), "{{cluster_name}} {{tier}}"), + ).Description("Elapsed time since the most recently retained PostgreSQL backup started, per cluster "+ + "and tier.")), sized(4, panelHeight, statPanel("Latest backup age (completion)", "dtdurations", query( - fmt.Sprintf("time() - max(klio_server_backup_latest_backup_completion_time_seconds{%s})", walMatcher), - "latest completion age"), - ).Description("Elapsed time since the most recently retained PostgreSQL backup completed.")), + fmt.Sprintf("time() - max by (cluster_name, tier) "+ + "(klio_server_backup_latest_backup_completion_time_seconds{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("Elapsed time since the most recently retained PostgreSQL backup completed, per cluster "+ + "and tier.")), sized(4, panelHeight, statPanel("Oldest backup age (start)", "dtdurations", - query(fmt.Sprintf("time() - min(klio_server_backup_oldest_backup_start_time_seconds{%s})", walMatcher), - "oldest start age"), - ).Description("Elapsed time since the oldest retained PostgreSQL backup started, reflecting the "+ - "effective retention horizon.")), + query(fmt.Sprintf("time() - min by (cluster_name, tier) "+ + "(klio_server_backup_oldest_backup_start_time_seconds{%s})", walMatcher), "{{cluster_name}} {{tier}}"), + ).Description("Elapsed time since the oldest retained PostgreSQL backup started, per cluster and tier, "+ + "reflecting each tier's effective retention horizon.")), sized(4, panelHeight, statPanel("Oldest backup age (completion)", "dtdurations", query( - fmt.Sprintf("time() - min(klio_server_backup_oldest_backup_completion_time_seconds{%s})", walMatcher), - "oldest completion age"), - ).Description("Elapsed time since the oldest retained PostgreSQL backup completed, reflecting the "+ - "effective retention horizon.")), - sized(8, panelHeight, barGaugePanel("PostgreSQL backups retained by tier", - query(fmt.Sprintf("sum by (tier) (klio_server_backup_backups{%s})", walMatcher), "{{tier}}"), + fmt.Sprintf("time() - min by (cluster_name, tier) "+ + "(klio_server_backup_oldest_backup_completion_time_seconds{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("Elapsed time since the oldest retained PostgreSQL backup completed, per cluster and "+ + "tier, reflecting each tier's effective retention horizon.")), + sized(8, panelHeight, barGaugePanel("PostgreSQL backups retained by cluster and tier", + query(fmt.Sprintf("sum by (cluster_name, tier) (klio_server_backup_backups{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), ).Decimals(0). - Description("Number of PostgreSQL backups currently retained per tier, across clusters.")), - sized(8, panelHeight, timeseriesPanel("Latest backup LSN by cluster", "bytes", - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_latest_backup_start_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} start"), - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_latest_backup_end_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} end"), - ).Description("Start and end LSN of the latest retained PostgreSQL backup, per cluster.")), - sized(8, panelHeight, timeseriesPanel("Oldest backup LSN by cluster", "bytes", - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_oldest_backup_start_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} start"), - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_oldest_backup_end_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} end"), - ).Description("Start and end LSN of the oldest retained PostgreSQL backup, per cluster.")), - sized(8, panelHeight, barGaugePanel("Backup timeline by cluster", - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_latest_backup_timeline{%s})", walMatcher), - "{{cluster_name}} latest"), - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_oldest_backup_timeline{%s})", walMatcher), - "{{cluster_name}} oldest"), - ).Description("PostgreSQL timeline of the latest and oldest retained backup, per cluster. A "+ - "mismatch means the retention window spans a promotion or failover.")), + Description("Number of PostgreSQL backups currently retained per cluster and tier.")), + sized(8, panelHeight, timeseriesPanel("Latest backup LSN by cluster and tier", "bytes", + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_latest_backup_start_lsn_bytes{%s})", + walMatcher), "{{cluster_name}} {{tier}} start"), + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_latest_backup_end_lsn_bytes{%s})", + walMatcher), "{{cluster_name}} {{tier}} end"), + ).Description("Start and end LSN of the latest retained PostgreSQL backup, per cluster and tier.")), + sized(8, panelHeight, timeseriesPanel("Oldest backup LSN by cluster and tier", "bytes", + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_oldest_backup_start_lsn_bytes{%s})", + walMatcher), "{{cluster_name}} {{tier}} start"), + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_oldest_backup_end_lsn_bytes{%s})", + walMatcher), "{{cluster_name}} {{tier}} end"), + ).Description("Start and end LSN of the oldest retained PostgreSQL backup, per cluster and tier.")), + sized(8, panelHeight, timelinePanel("Backup timeline by cluster and tier", + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_latest_backup_timeline{%s})", + walMatcher), "{{cluster_name}} {{tier}} latest"), + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_oldest_backup_timeline{%s})", + walMatcher), "{{cluster_name}} {{tier}} oldest"), + ).Description("PostgreSQL timeline of the latest and oldest retained backup, per cluster and tier, "+ + "over time. The latest and oldest lines diverging means the retention window spans a promotion "+ + "or failover.")), - // WAL ingest, unified across tiers via the `tier` label. - sized(8, panelHeight, timeseriesPanel("WAL files written rate by tier", "wps", - query(fmt.Sprintf("sum by (tier) (rate(klio_server_wal_written_total{%s}[$__rate_interval]))", walMatcher), - "{{tier}}"), - ).Description("Rate of WAL files written by the server, split by storage tier.")), - sized(8, panelHeight, timeseriesPanel("WAL bytes written rate by tier", "Bps", + // WAL ingest, per cluster and tier. + sized(8, panelHeight, timeseriesPanel("WAL files written rate by cluster and tier", "wps", + query(fmt.Sprintf("sum by (cluster_name, tier) (rate(klio_server_wal_written_total{%s}[$__rate_interval]))", + walMatcher), "{{cluster_name}} {{tier}}"), + ).Description("Rate of WAL files written by the server, split by cluster and storage tier.")), + sized(8, panelHeight, timeseriesPanel("WAL bytes written rate by cluster and tier", "Bps", query( - fmt.Sprintf("sum by (tier) (rate(klio_server_wal_written_size_bytes_total{%s}[$__rate_interval]))", - walMatcher), - "{{tier}}"), - ).Description("Rate of WAL bytes written by the server, split by storage tier.")), - sized(8, panelHeight, timeseriesPanel("Time since last WAL written by tier", "dtdurations", - query(fmt.Sprintf("time() - max by (tier) (klio_server_wal_latest_written_time_seconds{%s})", walMatcher), - "{{tier}}"), - ).Description("Elapsed time since the server last wrote a WAL file for each tier. A stale tier-1 "+ - "value means PostgreSQL stopped shipping WALs; a stale tier-2 value means the remote backend "+ - "stopped receiving them.")), - sized(8, panelHeight, timeseriesPanel("Latest written LSN by tier", "bytes", - query(fmt.Sprintf("max by (tier) (klio_server_wal_latest_written_lsn_bytes{%s})", walMatcher), "{{tier}}"), - ).Description("Most recent WAL LSN the server has written for each tier, as a byte offset.")), + fmt.Sprintf("sum by (cluster_name, tier) "+ + "(rate(klio_server_wal_written_size_bytes_total{%s}[$__rate_interval]))", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("Rate of WAL bytes written by the server, split by cluster and storage tier.")), + sized(8, panelHeight, timeseriesPanel("Time since last WAL written by cluster and tier", "dtdurations", + query(fmt.Sprintf("time() - max by (cluster_name, tier) (klio_server_wal_latest_written_time_seconds{%s})", + walMatcher), "{{cluster_name}} {{tier}}"), + ).Description("Elapsed time since the server last wrote a WAL file for each cluster and tier. A stale "+ + "tier-1 value means PostgreSQL stopped shipping WALs; a stale tier-2 value means the remote "+ + "backend stopped receiving them.")), + sized(8, panelHeight, timeseriesPanel("Latest written LSN by cluster and tier", "bytes", + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_wal_latest_written_lsn_bytes{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("Most recent WAL LSN the server has written for each cluster and tier, as a byte "+ + "offset.")), sized(8, panelHeight, timeseriesPanel("Backup verification rate by outcome and tier", "ops", query( - fmt.Sprintf("sum by (outcome, tier) (rate(klio_server_backup_verifications_total{%s}[$__rate_interval]))", - serverMatcher), - "{{tier}} / {{outcome}}", + fmt.Sprintf("sum by (service_name, outcome, tier) "+ + "(rate(klio_server_backup_verifications_total{%s}[$__rate_interval]))", serverMatcher), + "{{service_name}} {{tier}} / {{outcome}}", ), - ).Description("Rate of base backup verification checks, broken down by outcome and tier.")), + ).Description("Rate of base backup verification checks, broken down by outcome and tier (the "+ + "verification counter carries no cluster_name, so it is a per-server signal).")), // WAL processing latency, from the per-block and per-file duration - // histograms introduced alongside the unified WAL ingest series. + // histograms. These percentile panels aggregate the WAL distribution of + // the selected clusters; narrow $cluster to isolate one cluster. sized(8, panelHeight, timeseriesPanel("WAL block duration (p50/p95/p99) by path and stage", "ns", quantileTargets("klio_server_wal_block_duration_nanoseconds_bucket", "le, path, stage", walMatcher, "{{path}}/{{stage}}")..., ).Description("Percentile per-block WAL processing duration on the server, split by `path` "+ - "(put ingest / get serve) and `stage`.")), + "(put ingest / get serve) and `stage`, aggregated across the selected clusters.")), sized(8, panelHeight, timeseriesPanel("WAL file get duration (p50/p95/p99) by tier", "ns", quantileTargets("klio_server_wal_get_duration_nanoseconds_bucket", "le, tier", walMatcher, "{{tier}}")..., ).Description("Percentile duration of a complete WAL file gRPC get, split by the tier that "+ - "served it.")), + "served it, aggregated across the selected clusters.")), sized(8, panelHeight, timeseriesPanel("WAL tier-2 upload duration (p50/p95/p99) by cluster", "ns", quantileTargets("klio_server_wal_upload_duration_nanoseconds_bucket", "le, cluster_name", walMatcher, "{{cluster_name}}")..., ).Description("Percentile duration of the tier-2 archival upload to remote storage, per cluster.")), // Post-backup processing: tier-2 relay and per-tier maintenance runs. - sized(12, panelHeight, timeseriesPanel("Tier-2 relay rate by outcome", "ops", + sized(12, panelHeight, timeseriesPanel("Tier-2 relay rate by cluster and outcome", "ops", query( - fmt.Sprintf("sum by (outcome) (rate(klio_server_backup_relay_total{%s}[$__rate_interval]))", walMatcher), - "{{outcome}}", + fmt.Sprintf("sum by (cluster_name, outcome) (rate(klio_server_backup_relay_total{%s}"+ + "[$__rate_interval]))", walMatcher), + "{{cluster_name}} / {{outcome}}", ), - ).Description("Rate of tier-2 relay attempts after a backup (migration and verification), by outcome.")), - sized(12, panelHeight, timeseriesPanel("Maintenance run rate by tier and outcome", "ops", + ).Description("Rate of tier-2 relay attempts after a backup (migration and verification), per cluster "+ + "and outcome.")), + sized(12, panelHeight, timeseriesPanel("Maintenance run rate by cluster, tier and outcome", "ops", query( - fmt.Sprintf("sum by (tier, outcome) (rate(klio_server_backup_maintenance_total{%s}[$__rate_interval]))", - walMatcher), - "{{tier}} / {{outcome}}", + fmt.Sprintf("sum by (cluster_name, tier, outcome) (rate(klio_server_backup_maintenance_total{%s}"+ + "[$__rate_interval]))", walMatcher), + "{{cluster_name}} {{tier}}/{{outcome}}", ), ).Description("Rate of post-backup maintenance runs (base-snapshot retention and WAL cleanup), "+ - "by tier and outcome.")), + "per cluster, tier and outcome.")), - // Embedded NATS JetStream queue, broken down by stream. + // Embedded NATS JetStream queue, per server and stream. sized(8, panelHeight, timeseriesPanel("Queue messages by stream", "none", - query(fmt.Sprintf("sum by (stream) (klio_server_queue_messages{%s})", serverMatcher), "{{stream}}"), - ).Description("Messages currently held in each NATS JetStream stream of the embedded queue.")), + query(fmt.Sprintf("sum by (service_name, stream) (klio_server_queue_messages{%s})", serverMatcher), + "{{service_name}} / {{stream}}"), + ).Description("Messages currently held in each NATS JetStream stream of the embedded queue, per "+ + "server.")), sized(8, panelHeight, timeseriesPanel("Queue bytes by stream", "bytes", - query(fmt.Sprintf("sum by (stream) (klio_server_queue_bytes{%s})", serverMatcher), "{{stream}}"), - ).Description("Bytes currently held in each NATS JetStream stream of the embedded queue.")), + query(fmt.Sprintf("sum by (service_name, stream) (klio_server_queue_bytes{%s})", serverMatcher), + "{{service_name}} / {{stream}}"), + ).Description("Bytes currently held in each NATS JetStream stream of the embedded queue, per server.")), } }