feat(reservations): steer CR VM live migrations toward reservation slot hosts

cmd/manager/main.go

@@ -57,6 +57,7 @@ import (
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/nova"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs"
 	novafilters "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters"
+	novaweighers "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/pods"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/capacity"
@@ -419,6 +420,8 @@ func main() {
 	// filter runs.
 	novafilters.QuotaEnforcementMetricsSingleton = novafilters.NewQuotaEnforcementMetrics()
 	metrics.Registry.MustRegister(novafilters.QuotaEnforcementMetricsSingleton)
+	novaweighers.CRMigrationSlotMetricsSingleton = novaweighers.NewCRMigrationSlotMetrics()
+	metrics.Registry.MustRegister(novaweighers.CRMigrationSlotMetricsSingleton)
 
 	// Initialize commitments API for LIQUID interface (Postgres-backed usage reporting).
 	commitmentsConfig := conf.GetConfigOrDie[commitments.Config]()

helm/bundles/cortex-nova/templates/pipelines_kvm.yaml

@@ -151,6 +151,16 @@ spec:
         matching the request's project, resource group, and availability zone,
         with enough free memory capacity for the requested VM. Hosts without a
         matching reservation or without enough free capacity receive a lower weight.
+    - name: kvm_cr_migration_slot
+      description: |
+        During live migrations of VMs that occupy a committed-resource reservation
+        slot, this weigher boosts hosts that have a ready CR reservation with
+        sufficient remaining capacity for the full slot size. This steers migrations
+        toward hosts where the reservation can follow the VM, minimising the
+        double-blocking window. If no candidate has a matching slot, or the VM has
+        no CR reservation, all candidates receive equal weight.
+        Only activates for live_migrate requests. All other intents pass through.
+        Emits cortex_nova_weigh_cr_migration_slot_requests_total.
 ---
 apiVersion: cortex.cloud/v1alpha1
 kind: Pipeline
@@ -300,6 +310,16 @@ spec:
         matching the request's project, resource group, and availability zone,
         with enough free memory capacity for the requested VM. Hosts without a
         matching reservation or without enough free capacity receive a lower weight.
+    - name: kvm_cr_migration_slot
+      description: |
+        During live migrations of VMs that occupy a committed-resource reservation
+        slot, this weigher boosts hosts that have a ready CR reservation with
+        sufficient remaining capacity for the full slot size. This steers migrations
+        toward hosts where the reservation can follow the VM, minimising the
+        double-blocking window. If no candidate has a matching slot, or the VM has
+        no CR reservation, all candidates receive equal weight.
+        Only activates for live_migrate requests. All other intents pass through.
+        Emits cortex_nova_weigh_cr_migration_slot_requests_total.
 ---
 apiVersion: cortex.cloud/v1alpha1
 kind: Pipeline
@@ -756,6 +776,16 @@ spec:
         matching the request's project, resource group, and availability zone,
         with enough free memory capacity for the requested VM. Hosts without a
         matching reservation or without enough free capacity receive a lower weight.
+    - name: kvm_cr_migration_slot
+      description: |
+        During live migrations of VMs that occupy a committed-resource reservation
+        slot, this weigher boosts hosts that have a ready CR reservation with
+        sufficient remaining capacity for the full slot size. This steers migrations
+        toward hosts where the reservation can follow the VM, minimising the
+        double-blocking window. If no candidate has a matching slot, or the VM has
+        no CR reservation, all candidates receive equal weight.
+        Only activates for live_migrate requests. All other intents pass through.
+        Emits cortex_nova_weigh_cr_migration_slot_requests_total.
 ---
 apiVersion: cortex.cloud/v1alpha1
 kind: Pipeline
@@ -905,4 +935,14 @@ spec:
         matching the request's project, resource group, and availability zone,
         with enough free memory capacity for the requested VM. Hosts without a
         matching reservation or without enough free capacity receive a lower weight.
+    - name: kvm_cr_migration_slot
+      description: |
+        During live migrations of VMs that occupy a committed-resource reservation
+        slot, this weigher boosts hosts that have a ready CR reservation with
+        sufficient remaining capacity for the full slot size. This steers migrations
+        toward hosts where the reservation can follow the VM, minimising the
+        double-blocking window. If no candidate has a matching slot, or the VM has
+        no CR reservation, all candidates receive equal weight.
+        Only activates for live_migrate requests. All other intents pass through.
+        Emits cortex_nova_weigh_cr_migration_slot_requests_total.
 {{- end }}

internal/scheduling/nova/crs/evaluator.go

@@ -24,6 +24,19 @@ type SlotEvaluator struct {
 // BuildSlotEvaluator lists HV CRDs and CR Reservation CRDs once and returns an evaluator
 // that can answer slot-usability queries without further K8s reads.
 func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, error) {
+	var resList v1alpha1.ReservationList
+	if err := c.List(ctx, &resList,
+		client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource},
+	); err != nil {
+		return nil, err
+	}
+	return BuildSlotEvaluatorFromReservations(ctx, c, resList.Items)
+}
+
+// BuildSlotEvaluatorFromReservations builds a SlotEvaluator from an already-fetched
+// reservation slice. Use this when the caller has already listed reservations to avoid
+// a redundant K8s read.
+func BuildSlotEvaluatorFromReservations(ctx context.Context, c client.Client, reservations []v1alpha1.Reservation) (*SlotEvaluator, error) {
 	eval := &SlotEvaluator{
 		hvFreeMemory:       make(map[string]int64),
 		reservationsByHost: make(map[string][]v1alpha1.Reservation),
@@ -45,13 +58,7 @@ func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, e
 		eval.hvFreeMemory[hv.Name] = max(effectiveMemQ.Value()-allocMemQ.Value(), 0)
 	}
 
-	var resList v1alpha1.ReservationList
-	if err := c.List(ctx, &resList,
-		client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource},
-	); err != nil {
-		return nil, err
-	}
-	for _, res := range resList.Items {
+	for _, res := range reservations {
 		if !res.IsReady() {
 			continue
 		}
@@ -104,6 +111,35 @@ func (e *SlotEvaluator) HasUsableSlot(hostName, projectID, flavorGroup string, v
 	return false
 }
 
+// HasSlotWithCapacity reports whether hostName has at least one ready CR slot
+// matching projectID + flavorGroup whose remaining memory is >= requiredBytes.
+// Unlike HasUsableSlot, this does not apply the overfill model — it is used
+// during migration slot filtering where the full slot size must fit within a
+// single reservation on the target host.
+func (e *SlotEvaluator) HasSlotWithCapacity(hostName, projectID, flavorGroup string, requiredBytes int64) bool {
+	for _, slot := range e.SlotsForHost(hostName, projectID, flavorGroup) {
+		if ReservationRemainingMemory(slot) >= requiredBytes {
+			return true
+		}
+	}
+	return false
+}
+
+// CanAccommodateSlot reports whether hostName has enough free memory to absorb
+// a reservation block of requiredBytes. Used to check whether a slot can follow
+// a migrating VM to this host via the reconciler, even when no existing
+// compatible slot is present.
+//
+// Free memory is computed as: hvFreeMemory - sum(all reservation blocks on host).
+func (e *SlotEvaluator) CanAccommodateSlot(hostName string, requiredBytes int64) bool {
+	var allBlocks int64
+	for _, res := range e.reservationsByHost[hostName] {
+		blockQ := res.Spec.Resources[hv1.ResourceMemory]
+		allBlocks += blockQ.Value()
+	}
+	return e.hvFreeMemory[hostName]-allBlocks >= requiredBytes
+}
+
 // ReservationRemainingMemory returns how many bytes of memory remain
 // unallocated in a reservation slot. Returns 0 if the slot is full or nil.
 func ReservationRemainingMemory(res v1alpha1.Reservation) int64 {

internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go

@@ -0,0 +1,164 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package weighers
+
+import (
+	"context"
+	"log/slog"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	api "github.com/cobaltcore-dev/cortex/api/external/nova"
+	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
+	"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
+	"github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs"
+	hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+)
+
+// Options for the KVM CR migration slot weigher.
+type KVMCRMigrationSlotOpts struct {
+	// Weight assigned to hosts that have a compatible CR reservation slot.
+	// Default: 1.0
+	SlotHostWeight *float64 `json:"slotHostWeight,omitempty"`
+	// Weight assigned to all other hosts when a source slot is found.
+	// Default: 0.1
+	DefaultHostWeight *float64 `json:"defaultHostWeight,omitempty"`
+}
+
+func (o KVMCRMigrationSlotOpts) Validate() error {
+	return nil
+}
+
+func (o KVMCRMigrationSlotOpts) GetSlotHostWeight() float64 {
+	if o.SlotHostWeight == nil {
+		return 0.1
+	}
+	return *o.SlotHostWeight
+}
+
+func (o KVMCRMigrationSlotOpts) GetDefaultHostWeight() float64 {
+	if o.DefaultHostWeight == nil {
+		return 0.0
+	}
+	return *o.DefaultHostWeight
+}
+
+// KVMCRMigrationSlotStep weighs live-migration candidates by whether they can
+// accommodate the CR reservation slot of the migrating VM.
+//
+// When a VM with a CR reservation slot is migrated, this weigher boosts hosts
+// that have a ready CR reservation with sufficient remaining capacity for the
+// slot (not just the VM flavor). This steers the migration toward hosts where
+// the reservation can follow the VM, minimising the double-blocking window.
+//
+// If the VM has no CR reservation, or no candidate can accommodate the slot,
+// all candidates receive zero weight (no effect on ranking).
+//
+// Only activates for LiveMigrationIntent.
+type KVMCRMigrationSlotStep struct {
+	lib.BaseWeigher[api.ExternalSchedulerRequest, KVMCRMigrationSlotOpts]
+}
+
+func (s *KVMCRMigrationSlotStep) Run(
+	traceLog *slog.Logger,
+	request api.ExternalSchedulerRequest,
+) (*lib.FilterWeigherPipelineStepResult, error) {
+	result := s.IncludeAllHostsFromRequest(request)
+
+	intent, err := request.GetIntent()
+	if err != nil || intent != api.LiveMigrationIntent {
+		traceLog.Info("not a live migration, skipping CR migration slot weigher")
+		return result, nil //nolint:nilerr
+	}
+
+	instanceUUID := request.Spec.Data.InstanceUUID
+	projectID := request.Spec.Data.ProjectID
+
+	var allReservations v1alpha1.ReservationList
+	if err := s.Client.List(context.Background(), &allReservations,
+		client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource},
+	); err != nil {
+		return nil, err
+	}
+
+	// Find the source slot that has this VM confirmed.
+	var sourceSlot *v1alpha1.Reservation
+	for i := range allReservations.Items {
+		res := &allReservations.Items[i]
+		if res.Status.CommittedResourceReservation == nil {
+			continue
+		}
+		if _, ok := res.Status.CommittedResourceReservation.Allocations[instanceUUID]; ok {
+			sourceSlot = res
+			break
+		}
+	}
+
+	if sourceSlot == nil {
+		traceLog.Info("migrating VM has no confirmed CR reservation slot, skipping slot weigher",
+			"instanceUUID", instanceUUID)
+		CRMigrationSlotMetricsSingleton.RecordResult("no_source_slot")
+		return result, nil
+	}
+
+	slotMemoryBytes := sourceSlot.Spec.Resources[hv1.ResourceMemory]
+	if slotMemoryBytes.IsZero() {
+		traceLog.Info("source CR slot has no memory resource, skipping slot weigher",
+			"instanceUUID", instanceUUID,
+			"reservation", sourceSlot.Name)
+		CRMigrationSlotMetricsSingleton.RecordResult("no_source_slot")
+		return result, nil
+	}
+
+	resourceGroup := sourceSlot.Spec.CommittedResourceReservation.ResourceGroup
+
+	traceLog.Info("found source CR reservation slot for migrating VM",
+		"instanceUUID", instanceUUID,
+		"reservation", sourceSlot.Name,
+		"slotMemoryBytes", slotMemoryBytes.Value(),
+		"resourceGroup", resourceGroup,
+	)
+
+	evaluator, err := crs.BuildSlotEvaluatorFromReservations(context.Background(), s.Client, allReservations.Items)
+	if err != nil {
+		return nil, err
+	}
+
+	slotHostWeight := s.Options.GetSlotHostWeight()
+	defaultHostWeight := s.Options.GetDefaultHostWeight()
+
+	slotFound := false
+	for host := range result.Activations {
+		hasSlot := evaluator.HasSlotWithCapacity(host, projectID, resourceGroup, slotMemoryBytes.Value())
+		canFit := evaluator.CanAccommodateSlot(host, slotMemoryBytes.Value())
+		if hasSlot {
+			result.Activations[host] = slotHostWeight
+			slotFound = true
+			traceLog.Info("host has existing CR slot for migration, boosting weight",
+				"host", host, "weight", slotHostWeight)
+		} else if canFit {
+			result.Activations[host] = slotHostWeight
+			traceLog.Info("host can accommodate slot via reconciler, boosting weight",
+				"host", host, "weight", slotHostWeight)
+		} else {
+			result.Activations[host] = defaultHostWeight
+			traceLog.Info("host cannot accommodate CR slot, applying low weight",
+				"host", host, "weight", defaultHostWeight)
+		}
+	}
+
+	if slotFound {
+		CRMigrationSlotMetricsSingleton.RecordResult("slot_found")
+	} else {
+		CRMigrationSlotMetricsSingleton.RecordResult("no_slot")
+	}
+
+	return result, nil
+}
+
+func init() {
+	Index["kvm_cr_migration_slot"] = func() NovaWeigher {
+		return &KVMCRMigrationSlotStep{}
+	}
+}

internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_metrics.go

@@ -0,0 +1,65 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package weighers
+
+import (
+	"log/slog"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// CRMigrationSlotMetrics holds Prometheus metrics for the CR migration slot weigher.
+type CRMigrationSlotMetrics struct {
+	// Results counts live migration requests by outcome:
+	//   - slot_found:      at least one candidate has a compatible CR slot
+	//   - no_slot:         source slot found but no candidate is compatible
+	//   - no_source_slot:  migrating VM has no confirmed CR reservation
+	Results *prometheus.CounterVec
+}
+
+func NewCRMigrationSlotMetrics() *CRMigrationSlotMetrics {
+	return &CRMigrationSlotMetrics{
+		Results: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "cortex_nova_weigh_cr_migration_slot_requests_total",
+				Help: "Live migration requests processed by the CR migration slot weigher, " +
+					"labeled by outcome (slot_found, no_slot, no_source_slot).",
+			},
+			[]string{"result"},
+		),
+	}
+}
+
+func (m *CRMigrationSlotMetrics) Describe(ch chan<- *prometheus.Desc) {
+	if m == nil || m.Results == nil {
+		return
+	}
+	m.Results.Describe(ch)
+}
+
+func (m *CRMigrationSlotMetrics) Collect(ch chan<- prometheus.Metric) {
+	if m == nil || m.Results == nil {
+		return
+	}
+	m.Results.Collect(ch)
+}
+
+var recordCRMigrationSlotResultNilOnce = &sync.Once{}
+
+func (m *CRMigrationSlotMetrics) RecordResult(result string) {
+	if m == nil || m.Results == nil {
+		recordCRMigrationSlotResultNilOnce.Do(func() {
+			slog.Warn("CRMigrationSlotMetrics is nil; result metric not recorded "+
+				"(is CRMigrationSlotMetricsSingleton initialized in cmd/manager?)",
+				"result", result,
+			)
+		})
+		return
+	}
+	m.Results.WithLabelValues(result).Inc()
+}
+
+// CRMigrationSlotMetricsSingleton is set from cmd/manager/main.go during initialization.
+var CRMigrationSlotMetricsSingleton *CRMigrationSlotMetrics