diff --git a/internal/knowledge/extractor/plugins/compute/host_utilization.go b/internal/knowledge/extractor/plugins/compute/host_utilization.go index 73f7c1ac2..93ac768b2 100644 --- a/internal/knowledge/extractor/plugins/compute/host_utilization.go +++ b/internal/knowledge/extractor/plugins/compute/host_utilization.go @@ -17,16 +17,19 @@ type HostUtilization struct { // VCPU resource usage VCPUsUsed float64 `db:"vcpus_used" json:"vcpusUsed"` VCPUsUtilizedPct float64 `db:"vcpus_utilized_pct" json:"vcpusUtilizedPct"` + PhysicalVCPUs float64 `db:"physical_vcpus" json:"physicalVCPUs"` TotalVCPUsAllocatable float64 `db:"total_vcpus_allocatable" json:"totalVCPUsAllocatable"` // RAM resource usage RAMUsedMB float64 `db:"ram_used_mb" json:"ramUsedMB"` RAMUtilizedPct float64 `db:"ram_utilized_pct" json:"ramUtilizedPct"` + PhysicalRAMMB float64 `db:"physical_ram_mb" json:"physicalRAMMB"` TotalRAMAllocatableMB float64 `db:"total_ram_allocatable_mb" json:"totalRAMAllocatableMB"` // Disk resource usage DiskUsedGB float64 `db:"disk_used_gb" json:"diskUsedGB"` DiskUtilizedPct float64 `db:"disk_utilized_pct" json:"diskUtilizedPct"` + PhysicalDiskGB float64 `db:"physical_disk_gb" json:"physicalDiskGB"` TotalDiskAllocatableGB float64 `db:"total_disk_allocatable_gb" json:"totalDiskAllocatableGB"` } diff --git a/internal/knowledge/extractor/plugins/compute/host_utilization.sql b/internal/knowledge/extractor/plugins/compute/host_utilization.sql index e3e455c38..c3ff42127 100644 --- a/internal/knowledge/extractor/plugins/compute/host_utilization.sql +++ b/internal/knowledge/extractor/plugins/compute/host_utilization.sql @@ -1,6 +1,9 @@ WITH host_resources AS ( SELECT h.service_host AS compute_host, + CAST((i_memory_mb.total - i_memory_mb.reserved) AS FLOAT) AS physical_ram_mb, + CAST((i_vcpu.total - i_vcpu.reserved) AS FLOAT) AS physical_vcpus, + CAST((i_disk_gb.total - i_disk_gb.reserved) AS FLOAT) AS physical_disk_gb, CAST((i_memory_mb.total - i_memory_mb.reserved) * i_memory_mb.allocation_ratio AS FLOAT) AS total_ram_allocatable_mb, CAST((i_vcpu.total - i_vcpu.reserved) * i_vcpu.allocation_ratio AS FLOAT) AS total_vcpus_allocatable, CAST((i_disk_gb.total - i_disk_gb.reserved) * i_disk_gb.allocation_ratio AS FLOAT) AS total_disk_allocatable_gb, @@ -21,6 +24,8 @@ WITH host_resources AS ( -- Resource usage formulas: +-- - "PhysicalCapacity": Raw usable resource after subtracting reserved capacity, before overcommit. +-- Formula: placement.total - placement.reserved -- - "TotalAllocatableCapacity": The maximum usable resource after reserving capacity and applying overcommit. -- Formula: (placement.total - placement.reserved) * placement.allocation_ratio -- - "UsedAbsolute": The actual amount of resource currently in use (includes overcommit). @@ -35,6 +40,9 @@ WITH host_resources AS ( SELECT compute_host, + physical_ram_mb, + physical_vcpus, + physical_disk_gb, total_ram_allocatable_mb, total_vcpus_allocatable, total_disk_allocatable_gb, diff --git a/internal/knowledge/extractor/plugins/compute/host_utilization_test.go b/internal/knowledge/extractor/plugins/compute/host_utilization_test.go index 984823564..39060fa01 100644 --- a/internal/knowledge/extractor/plugins/compute/host_utilization_test.go +++ b/internal/knowledge/extractor/plugins/compute/host_utilization_test.go @@ -72,6 +72,9 @@ func TestHostUtilizationExtractor_Extract(t *testing.T) { expected: []HostUtilization{ { ComputeHost: "host1", + PhysicalVCPUs: 100, + PhysicalRAMMB: 1000, + PhysicalDiskGB: 2000, TotalVCPUsAllocatable: 100, TotalRAMAllocatableMB: 1000, TotalDiskAllocatableGB: 2000, @@ -121,6 +124,9 @@ func TestHostUtilizationExtractor_Extract(t *testing.T) { expected: []HostUtilization{ { ComputeHost: "host1", + PhysicalVCPUs: 100, + PhysicalRAMMB: 1000, + PhysicalDiskGB: 2000, TotalVCPUsAllocatable: 100, TotalRAMAllocatableMB: 1000, TotalDiskAllocatableGB: 2000, @@ -171,6 +177,9 @@ func TestHostUtilizationExtractor_Extract(t *testing.T) { expected: []HostUtilization{ { ComputeHost: "host1", + PhysicalVCPUs: 100, + PhysicalRAMMB: 1000, + PhysicalDiskGB: 2000, TotalVCPUsAllocatable: 200, TotalRAMAllocatableMB: 1000, TotalDiskAllocatableGB: 2000, diff --git a/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go b/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go index 399648403..53bc90fa4 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go +++ b/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go @@ -27,9 +27,10 @@ type hostReservationResources struct { type KVMHostCapacityKPI struct { // Common base for all KPIs that provides standard functionality. - plugins.BaseKPI[struct{}] // No options passed through yaml config - totalCapacityPerHost *prometheus.Desc - capacityPerHost *prometheus.Desc + plugins.BaseKPI[struct{}] // No options passed through yaml config + totalCapacityPerHost *prometheus.Desc + totalPhysicalCapacityPerHost *prometheus.Desc + capacityPerHost *prometheus.Desc } func (KVMHostCapacityKPI) GetName() string { @@ -40,6 +41,12 @@ func (k *KVMHostCapacityKPI) Init(db *db.DB, client client.Client, opts conf.Raw if err := k.BaseKPI.Init(db, client, opts); err != nil { return err } + k.totalPhysicalCapacityPerHost = prometheus.NewDesc( + "cortex_kvm_host_physical_capacity_total", + "Total physical resource capacity on the KVM hosts (individually by host, ignoring overcommit factor). CPU in vCPUs, memory in bytes.", + append(kvmHostLabels, "resource"), + nil, + ) k.totalCapacityPerHost = prometheus.NewDesc( "cortex_kvm_host_capacity_total", "Total resource capacity on the KVM hosts (individually by host). CPU in vCPUs, memory in bytes.", @@ -57,6 +64,7 @@ func (k *KVMHostCapacityKPI) Init(db *db.DB, client client.Client, opts conf.Raw func (k *KVMHostCapacityKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.totalCapacityPerHost + ch <- k.totalPhysicalCapacityPerHost ch <- k.capacityPerHost } @@ -160,7 +168,6 @@ func (k *KVMHostCapacityKPI) Collect(ch chan<- prometheus.Metric) { for _, hypervisor := range hypervisors { cpuTotal, hasCPUTotal := hypervisor.getResourceCapacity(hv1.ResourceCPU) - ramTotal, hasRAMTotal := hypervisor.getResourceCapacity(hv1.ResourceMemory) if !hasCPUTotal || !hasRAMTotal { @@ -168,6 +175,14 @@ func (k *KVMHostCapacityKPI) Collect(ch chan<- prometheus.Metric) { continue } + cpuPhysical, hasCPUPhysical := hypervisor.getPhysicalCapacity(hv1.ResourceCPU) + ramPhysical, hasRAMPhysical := hypervisor.getPhysicalCapacity(hv1.ResourceMemory) + + if !hasCPUPhysical || !hasRAMPhysical { + slog.Warn("hypervisor missing physical cpu or ram capacity, skipping", "host", hypervisor.Name) + continue + } + cpuUsed := hypervisor.getResourceAllocation(hv1.ResourceCPU) ramUsed := hypervisor.getResourceAllocation(hv1.ResourceMemory) @@ -183,6 +198,9 @@ func (k *KVMHostCapacityKPI) Collect(ch chan<- prometheus.Metric) { labels := hypervisor.getHostLabels() + ch <- prometheus.MustNewConstMetric(k.totalPhysicalCapacityPerHost, prometheus.GaugeValue, cpuPhysical.AsApproximateFloat64(), append(labels, "cpu")...) + ch <- prometheus.MustNewConstMetric(k.totalPhysicalCapacityPerHost, prometheus.GaugeValue, ramPhysical.AsApproximateFloat64(), append(labels, "ram")...) + ch <- prometheus.MustNewConstMetric(k.totalCapacityPerHost, prometheus.GaugeValue, cpuTotal.AsApproximateFloat64(), append(labels, "cpu")...) ch <- prometheus.MustNewConstMetric(k.totalCapacityPerHost, prometheus.GaugeValue, ramTotal.AsApproximateFloat64(), append(labels, "ram")...) diff --git a/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity_test.go b/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity_test.go index 317b478cc..bf30bbcb1 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity_test.go +++ b/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity_test.go @@ -25,6 +25,12 @@ func TestKVMResourceCapacityKPI_Init(t *testing.T) { } } +func kvmPhysicalMetric(host, res, az string, value float64) collectedVMwareMetric { + l := mockKVMHostLabels(host, az) + l["resource"] = res + return collectedVMwareMetric{Name: "cortex_kvm_host_physical_capacity_total", Labels: l, Value: value} +} + func kvmTotalMetric(host, res, az string, value float64) collectedVMwareMetric { l := mockKVMHostLabels(host, az) l["resource"] = res @@ -117,6 +123,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 128), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 128), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 64), @@ -157,6 +165,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 128), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 128), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 64), @@ -231,6 +241,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("128"), hv1.ResourceMemory: resource.MustParse("512Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("128"), + hv1.ResourceMemory: resource.MustParse("512Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("64"), hv1.ResourceMemory: resource.MustParse("256Gi"), @@ -240,6 +254,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 128), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 128), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 64), @@ -267,6 +283,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("256"), hv1.ResourceMemory: resource.MustParse("1Ti"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("256"), + hv1.ResourceMemory: resource.MustParse("1Ti"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("128"), hv1.ResourceMemory: resource.MustParse("512Gi"), @@ -290,6 +310,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { return m } return []collectedVMwareMetric{ + {Name: "cortex_kvm_host_physical_capacity_total", Labels: l("cpu", ""), Value: 256}, + {Name: "cortex_kvm_host_physical_capacity_total", Labels: l("ram", ""), Value: 1099511627776}, // 1Ti {Name: "cortex_kvm_host_capacity_total", Labels: l("cpu", ""), Value: 256}, {Name: "cortex_kvm_host_capacity_total", Labels: l("ram", ""), Value: 1099511627776}, // 1Ti {Name: "cortex_kvm_host_capacity_usage", Labels: l("cpu", "utilized"), Value: 128}, @@ -318,6 +340,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("64"), hv1.ResourceMemory: resource.MustParse("256Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("64"), + hv1.ResourceMemory: resource.MustParse("256Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("32"), hv1.ResourceMemory: resource.MustParse("128Gi"), @@ -341,6 +367,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { return m } return []collectedVMwareMetric{ + {Name: "cortex_kvm_host_physical_capacity_total", Labels: l("cpu", ""), Value: 64}, + {Name: "cortex_kvm_host_physical_capacity_total", Labels: l("ram", ""), Value: 274877906944}, // 256Gi {Name: "cortex_kvm_host_capacity_total", Labels: l("cpu", ""), Value: 64}, {Name: "cortex_kvm_host_capacity_total", Labels: l("ram", ""), Value: 274877906944}, // 256Gi {Name: "cortex_kvm_host_capacity_usage", Labels: l("cpu", "utilized"), Value: 32}, @@ -369,6 +397,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("100"), hv1.ResourceMemory: resource.MustParse("200Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("100"), + hv1.ResourceMemory: resource.MustParse("200Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("50"), hv1.ResourceMemory: resource.MustParse("100Gi"), @@ -388,6 +420,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("200"), hv1.ResourceMemory: resource.MustParse("400Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("200"), + hv1.ResourceMemory: resource.MustParse("400Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("150"), hv1.ResourceMemory: resource.MustParse("300Gi"), @@ -407,6 +443,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { return m } return []collectedVMwareMetric{ + kvmPhysicalMetric("node010-bb100", "cpu", "qa-1a", 100), + kvmPhysicalMetric("node010-bb100", "ram", "qa-1a", 214748364800), // 200Gi kvmTotalMetric("node010-bb100", "cpu", "qa-1a", 100), kvmTotalMetric("node010-bb100", "ram", "qa-1a", 214748364800), // 200Gi kvmUsageMetric("node010-bb100", "cpu", "utilized", "qa-1a", 50), @@ -417,6 +455,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { kvmUsageMetric("node010-bb100", "ram", "failover", "qa-1a", 0), kvmUsageMetric("node010-bb100", "cpu", "available", "qa-1a", 50), // 100-50-0-0 kvmUsageMetric("node010-bb100", "ram", "available", "qa-1a", 107374182400), // 200Gi-100Gi + {Name: "cortex_kvm_host_physical_capacity_total", Labels: sapphire("cpu", ""), Value: 200}, + {Name: "cortex_kvm_host_physical_capacity_total", Labels: sapphire("ram", ""), Value: 429496729600}, // 400Gi {Name: "cortex_kvm_host_capacity_total", Labels: sapphire("cpu", ""), Value: 200}, {Name: "cortex_kvm_host_capacity_total", Labels: sapphire("ram", ""), Value: 429496729600}, // 400Gi {Name: "cortex_kvm_host_capacity_usage", Labels: sapphire("cpu", "utilized"), Value: 150}, @@ -445,12 +485,18 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("96"), hv1.ResourceMemory: resource.MustParse("384Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("96"), + hv1.ResourceMemory: resource.MustParse("384Gi"), + }, Allocation: nil, Traits: []string{}, }, }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node004-bb091", "cpu", "qa-1d", 96), + kvmPhysicalMetric("node004-bb091", "ram", "qa-1d", 412316860416), // 384Gi kvmTotalMetric("node004-bb091", "cpu", "qa-1d", 96), kvmTotalMetric("node004-bb091", "ram", "qa-1d", 412316860416), // 384Gi kvmUsageMetric("node004-bb091", "cpu", "utilized", "qa-1d", 0), @@ -478,6 +524,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("128"), hv1.ResourceMemory: resource.MustParse("512Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("128"), + hv1.ResourceMemory: resource.MustParse("512Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("64"), hv1.ResourceMemory: resource.MustParse("256Gi"), @@ -509,6 +559,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 128), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 128), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 64), @@ -536,6 +588,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("128"), hv1.ResourceMemory: resource.MustParse("512Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("128"), + hv1.ResourceMemory: resource.MustParse("512Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("64"), hv1.ResourceMemory: resource.MustParse("256Gi"), @@ -576,6 +632,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 128), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 128), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 64), @@ -604,6 +662,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("128"), hv1.ResourceMemory: resource.MustParse("512Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("128"), + hv1.ResourceMemory: resource.MustParse("512Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("64"), hv1.ResourceMemory: resource.MustParse("256Gi"), @@ -635,6 +697,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 128), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 128), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 64), @@ -663,6 +727,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("128"), hv1.ResourceMemory: resource.MustParse("512Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("128"), + hv1.ResourceMemory: resource.MustParse("512Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("64"), hv1.ResourceMemory: resource.MustParse("256Gi"), @@ -714,6 +782,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 128), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 128), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 549755813888), // 512Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 64), @@ -740,6 +810,10 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { hv1.ResourceCPU: resource.MustParse("100"), hv1.ResourceMemory: resource.MustParse("200Gi"), }, + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: resource.MustParse("100"), + hv1.ResourceMemory: resource.MustParse("200Gi"), + }, Allocation: map[hv1.ResourceName]resource.Quantity{ hv1.ResourceCPU: resource.MustParse("80"), hv1.ResourceMemory: resource.MustParse("150Gi"), @@ -790,6 +864,8 @@ func TestKVMResourceCapacityKPI_Collect(t *testing.T) { }, }, expectedMetrics: []collectedVMwareMetric{ + kvmPhysicalMetric("node001-bb088", "cpu", "qa-1a", 100), + kvmPhysicalMetric("node001-bb088", "ram", "qa-1a", 214748364800), // 200Gi kvmTotalMetric("node001-bb088", "cpu", "qa-1a", 100), kvmTotalMetric("node001-bb088", "ram", "qa-1a", 214748364800), // 200Gi kvmUsageMetric("node001-bb088", "cpu", "utilized", "qa-1a", 80), diff --git a/internal/knowledge/kpis/plugins/infrastructure/shared.go b/internal/knowledge/kpis/plugins/infrastructure/shared.go index d39790521..6d1346824 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/shared.go +++ b/internal/knowledge/kpis/plugins/infrastructure/shared.go @@ -7,7 +7,6 @@ import ( "fmt" "regexp" "strconv" - "strings" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -73,6 +72,7 @@ var vmwareHostLabels = []string{ var kvmHostLabels = []string{ "compute_host", + "compute_cluster", "availability_zone", "building_block", "cpu_architecture", @@ -99,11 +99,14 @@ func (h kvmHost) getHostLabels() []string { availabilityZone = "unknown" } - buildingBlock := "unknown" - // Assuming hypervisor names are in the format nodeXXX-bbYY - parts := strings.Split(h.Name, "-") - if len(parts) > 1 { - buildingBlock = parts[1] + buildingBlock := h.Labels["kubernetes.metal.cloud.sap/bb"] + if buildingBlock == "" { + buildingBlock = "unknown" + } + + computeCluster := h.Labels["kubernetes.metal.cloud.sap/cluster"] + if computeCluster == "" { + computeCluster = "unknown" } osVersion := h.Status.OperatingSystem.Version @@ -128,6 +131,7 @@ func (h kvmHost) getHostLabels() []string { return []string{ h.Name, + computeCluster, availabilityZone, buildingBlock, cpuArchitecture, @@ -140,6 +144,17 @@ func (h kvmHost) getHostLabels() []string { } } +func (k kvmHost) getPhysicalCapacity(resourceName hv1.ResourceName) (capacity resource.Quantity, ok bool) { + if k.Status.Capacity == nil { + return resource.Quantity{}, false + } + qty, exists := k.Status.Capacity[resourceName] + if !exists || qty.IsZero() { + return resource.Quantity{}, false + } + return qty, true +} + // getResourceCapacity attempts to retrieve the effective capacity for the specified resource from the hypervisor status, falling back to the physical capacity if effective capacity is not available. It returns the capacity quantity and a boolean indicating whether any capacity information was found. func (k kvmHost) getResourceCapacity(resourceName hv1.ResourceName) (capacity resource.Quantity, ok bool) { if k.Status.EffectiveCapacity != nil { diff --git a/internal/knowledge/kpis/plugins/infrastructure/shared_test.go b/internal/knowledge/kpis/plugins/infrastructure/shared_test.go index 8fe8c9d7b..a858bad6f 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/shared_test.go +++ b/internal/knowledge/kpis/plugins/infrastructure/shared_test.go @@ -4,7 +4,6 @@ package infrastructure import ( - "strings" "testing" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" @@ -13,15 +12,11 @@ import ( ) func mockKVMHostLabels(host, az string) map[string]string { - bb := "unknown" - parts := strings.Split(host, "-") - if len(parts) > 1 { - bb = parts[1] - } return map[string]string{ "compute_host": host, + "compute_cluster": "unknown", "availability_zone": az, - "building_block": bb, + "building_block": "unknown", "cpu_architecture": "cascade-lake", "workload_type": "general-purpose", "enabled": "true", @@ -136,7 +131,7 @@ func TestKVMHost_GetHostLabels(t *testing.T) { host: kvmHost{hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{Name: "node001-bb01"}, }}, - want: []string{"node001-bb01", "unknown", "bb01", "cascade-lake", "general-purpose", "true", "false", "false", "false"}, + want: []string{"node001-bb01", "unknown", "unknown", "unknown", "cascade-lake", "general-purpose", "true", "false", "false", "false", "unknown"}, }, { name: "availability zone from label", @@ -146,14 +141,20 @@ func TestKVMHost_GetHostLabels(t *testing.T) { Labels: map[string]string{"topology.kubernetes.io/zone": "az1"}, }, }}, - want: []string{"node001-bb01", "az1", "bb01", "cascade-lake", "general-purpose", "true", "false", "false", "false"}, + want: []string{"node001-bb01", "unknown", "az1", "unknown", "cascade-lake", "general-purpose", "true", "false", "false", "false", "unknown"}, }, { - name: "name without dash results in unknown building block", + name: "bb and cluster from labels", host: kvmHost{hv1.Hypervisor{ - ObjectMeta: metav1.ObjectMeta{Name: "nodewithoutdash"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "node001-bb01", + Labels: map[string]string{ + "kubernetes.metal.cloud.sap/bb": "bb01", + "kubernetes.metal.cloud.sap/cluster": "cluster-a", + }, + }, }}, - want: []string{"nodewithoutdash", "unknown", "unknown", "cascade-lake", "general-purpose", "true", "false", "false", "false"}, + want: []string{"node001-bb01", "cluster-a", "unknown", "bb01", "cascade-lake", "general-purpose", "true", "false", "false", "false", "unknown"}, }, { name: "sapphire rapids trait", @@ -161,7 +162,7 @@ func TestKVMHost_GetHostLabels(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node001-bb01"}, Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_HW_SAPPHIRE_RAPIDS"}}, }}, - want: []string{"node001-bb01", "unknown", "bb01", "sapphire-rapids", "general-purpose", "true", "false", "false", "false"}, + want: []string{"node001-bb01", "unknown", "unknown", "unknown", "sapphire-rapids", "general-purpose", "true", "false", "false", "false", "unknown"}, }, { name: "hana exclusive host trait", @@ -169,7 +170,7 @@ func TestKVMHost_GetHostLabels(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node001-bb01"}, Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}}, }}, - want: []string{"node001-bb01", "unknown", "bb01", "cascade-lake", "hana", "true", "false", "false", "false"}, + want: []string{"node001-bb01", "unknown", "unknown", "unknown", "cascade-lake", "hana", "true", "false", "false", "false", "unknown"}, }, { name: "decommissioning trait", @@ -177,7 +178,7 @@ func TestKVMHost_GetHostLabels(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node001-bb01"}, Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_DECOMMISSIONING"}}, }}, - want: []string{"node001-bb01", "unknown", "bb01", "cascade-lake", "general-purpose", "true", "true", "false", "false"}, + want: []string{"node001-bb01", "unknown", "unknown", "unknown", "cascade-lake", "general-purpose", "true", "true", "false", "false", "unknown"}, }, { name: "external customer exclusive trait", @@ -185,7 +186,7 @@ func TestKVMHost_GetHostLabels(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node001-bb01"}, Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE"}}, }}, - want: []string{"node001-bb01", "unknown", "bb01", "cascade-lake", "general-purpose", "true", "false", "true", "false"}, + want: []string{"node001-bb01", "unknown", "unknown", "unknown", "cascade-lake", "general-purpose", "true", "false", "true", "false", "unknown"}, }, { name: "maintenance set", @@ -193,14 +194,18 @@ func TestKVMHost_GetHostLabels(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node001-bb01"}, Spec: hv1.HypervisorSpec{Maintenance: hv1.MaintenanceManual}, }}, - want: []string{"node001-bb01", "unknown", "bb01", "cascade-lake", "general-purpose", "true", "false", "false", "true"}, + want: []string{"node001-bb01", "unknown", "unknown", "unknown", "cascade-lake", "general-purpose", "true", "false", "false", "true", "unknown"}, }, { name: "all traits and maintenance set", host: kvmHost{hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ - Name: "node001-bb42", - Labels: map[string]string{"topology.kubernetes.io/zone": "az3"}, + Name: "node001-bb42", + Labels: map[string]string{ + "topology.kubernetes.io/zone": "az3", + "kubernetes.metal.cloud.sap/bb": "bb42", + "kubernetes.metal.cloud.sap/cluster": "cluster-b", + }, }, Spec: hv1.HypervisorSpec{Maintenance: hv1.MaintenanceAuto}, Status: hv1.HypervisorStatus{Traits: []string{ @@ -210,7 +215,7 @@ func TestKVMHost_GetHostLabels(t *testing.T) { "CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE", }}, }}, - want: []string{"node001-bb42", "az3", "bb42", "sapphire-rapids", "hana", "true", "true", "true", "true"}, + want: []string{"node001-bb42", "cluster-b", "az3", "bb42", "sapphire-rapids", "hana", "true", "true", "true", "true", "unknown"}, }, { name: "os version set", @@ -218,7 +223,7 @@ func TestKVMHost_GetHostLabels(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node001-bb01"}, Status: hv1.HypervisorStatus{OperatingSystem: hv1.OperatingSystemStatus{Version: "1.1.1"}}, }}, - want: []string{"node001-bb01", "unknown", "bb01", "cascade-lake", "general-purpose", "true", "false", "false", "false", "1.1.1"}, + want: []string{"node001-bb01", "unknown", "unknown", "unknown", "cascade-lake", "general-purpose", "true", "false", "false", "false", "1.1.1"}, }, } diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go index 45508636f..49c87aadc 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go @@ -19,8 +19,9 @@ import ( type VMwareHostCapacityKPI struct { plugins.BaseKPI[struct{}] - capacityUsagePerHost *prometheus.Desc - capacityTotalPerHost *prometheus.Desc + capacityUsagePerHost *prometheus.Desc + capacityPhysicalPerHost *prometheus.Desc + capacityTotalPerHost *prometheus.Desc } func (k *VMwareHostCapacityKPI) GetName() string { @@ -36,6 +37,11 @@ func (k *VMwareHostCapacityKPI) Init(dbConn *db.DB, c client.Client, opts conf.R "Capacity usage per VMware host. CPU in vCPUs, memory and disk in bytes.", append(vmwareHostLabels, "resource"), nil, ) + k.capacityPhysicalPerHost = prometheus.NewDesc( + "cortex_vmware_host_physical_capacity_total", + "Usable physical resource capacity per VMware host (total - reserved; ignoring overcommit factor). CPU in vCPUs, memory and disk in bytes.", + append(vmwareHostLabels, "resource"), nil, + ) k.capacityTotalPerHost = prometheus.NewDesc( "cortex_vmware_host_capacity_total", "Total allocatable capacity per VMware host. CPU in vCPUs, memory and disk in bytes.", @@ -46,6 +52,7 @@ func (k *VMwareHostCapacityKPI) Init(dbConn *db.DB, c client.Client, opts conf.R func (k *VMwareHostCapacityKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.capacityUsagePerHost + ch <- k.capacityPhysicalPerHost ch <- k.capacityTotalPerHost } @@ -73,6 +80,10 @@ func (k *VMwareHostCapacityKPI) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(k.capacityUsagePerHost, prometheus.GaugeValue, util.RAMUsedMB*1024*1024, append(labels, "ram")...) ch <- prometheus.MustNewConstMetric(k.capacityUsagePerHost, prometheus.GaugeValue, util.DiskUsedGB*1024*1024*1024, append(labels, "disk")...) + ch <- prometheus.MustNewConstMetric(k.capacityPhysicalPerHost, prometheus.GaugeValue, util.PhysicalVCPUs, append(labels, "cpu")...) + ch <- prometheus.MustNewConstMetric(k.capacityPhysicalPerHost, prometheus.GaugeValue, util.PhysicalRAMMB*1024*1024, append(labels, "ram")...) + ch <- prometheus.MustNewConstMetric(k.capacityPhysicalPerHost, prometheus.GaugeValue, util.PhysicalDiskGB*1024*1024*1024, append(labels, "disk")...) + ch <- prometheus.MustNewConstMetric(k.capacityTotalPerHost, prometheus.GaugeValue, util.TotalVCPUsAllocatable, append(labels, "cpu")...) ch <- prometheus.MustNewConstMetric(k.capacityTotalPerHost, prometheus.GaugeValue, util.TotalRAMAllocatableMB*1024*1024, append(labels, "ram")...) ch <- prometheus.MustNewConstMetric(k.capacityTotalPerHost, prometheus.GaugeValue, util.TotalDiskAllocatableGB*1024*1024*1024, append(labels, "disk")...) diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go index dd2f50637..69a64b9e6 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go @@ -168,10 +168,13 @@ func TestVMwareHostCapacityKPI_Collect(t *testing.T) { { ComputeHost: "nova-compute-1", VCPUsUsed: 4, + PhysicalVCPUs: 8, TotalVCPUsAllocatable: 16, RAMUsedMB: 2048, + PhysicalRAMMB: 4096, TotalRAMAllocatableMB: 8192, DiskUsedGB: 50, + PhysicalDiskGB: 250, TotalDiskAllocatableGB: 500, }, }, @@ -179,6 +182,9 @@ func TestVMwareHostCapacityKPI_Collect(t *testing.T) { {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 4}, {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 50 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 8}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 4096 * 1024 * 1024}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 250 * 1024 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 16}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 8192 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 500 * 1024 * 1024 * 1024}, @@ -191,19 +197,25 @@ func TestVMwareHostCapacityKPI_Collect(t *testing.T) { {ComputeHost: "nova-compute-2", AvailabilityZone: "az2"}, }, utilizations: []compute.HostUtilization{ - {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, - {ComputeHost: "nova-compute-2", VCPUsUsed: 6, TotalVCPUsAllocatable: 12, RAMUsedMB: 1024, TotalRAMAllocatableMB: 4096, DiskUsedGB: 20, TotalDiskAllocatableGB: 200}, + {ComputeHost: "nova-compute-1", VCPUsUsed: 2, PhysicalVCPUs: 4, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, PhysicalRAMMB: 1024, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, PhysicalDiskGB: 50, TotalDiskAllocatableGB: 100}, + {ComputeHost: "nova-compute-2", VCPUsUsed: 6, PhysicalVCPUs: 6, TotalVCPUsAllocatable: 12, RAMUsedMB: 1024, PhysicalRAMMB: 2048, TotalRAMAllocatableMB: 4096, DiskUsedGB: 20, PhysicalDiskGB: 100, TotalDiskAllocatableGB: 200}, }, expectedMetrics: []collectedVMwareMetric{ {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 2}, {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 512 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 10 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 4}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 50 * 1024 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 8}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 100 * 1024 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-2", "az2", "cpu"), Value: 6}, {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-2", "az2", "ram"), Value: 1024 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-2", "az2", "disk"), Value: 20 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "cpu"), Value: 6}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "ram"), Value: 2048 * 1024 * 1024}, + {Name: "cortex_vmware_host_physical_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "disk"), Value: 100 * 1024 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "cpu"), Value: 12}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "ram"), Value: 4096 * 1024 * 1024}, {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "disk"), Value: 200 * 1024 * 1024 * 1024},