From 89adff06141770c43ebe9e901672ed2b631de93c Mon Sep 17 00:00:00 2001 From: Jared Patterson Date: Sat, 1 Aug 2026 18:13:48 +1200 Subject: [PATCH 1/2] feat: support topologySpreadConstraints for vector aggregators Aggregator replicas need to spread across availability zones, for failure isolation and to keep cross zone traffic down, and spec.affinity cannot express that. Preferred pod anti-affinity is only a scheduling score, so under a provisioner like Karpenter, which adds nodes to fit pending pods rather than to satisfy preferences, replicas still pack onto one node. Required anti-affinity has no count parameter, so it allows one pod per zone and leaves every replica above the zone count Pending, which rules it out for HPA driven scaling. VectorAggregator and ClusterVectorAggregator now take an optional spec.topologySpreadConstraints, passed to the aggregator pod template next to affinity, tolerations and schedulerName. With maxSkew and whenUnsatisfiable of ScheduleAnyway that expresses "spread evenly, but never block scheduling". The field lives on the aggregator spec rather than the shared one, since the agent is a DaemonSet that already places one pod per node. Both the Deployment and the StatefulSet paths are covered because they share one pod template, and the event collector inherits it the same way it already inherits affinity. Unset means no constraints, so existing aggregators are untouched. Closes #270 --- api/v1alpha1/vector_common_types.go | 8 + api/v1alpha1/zz_generated.deepcopy.go | 7 + ...y.kaasops.io_clustervectoraggregators.yaml | 180 ++++++++++++++++++ ...vability.kaasops.io_vectoraggregators.yaml | 180 ++++++++++++++++++ docs/aggregator.md | 49 +++++ ...y.kaasops.io_clustervectoraggregators.yaml | 180 ++++++++++++++++++ ...vability.kaasops.io_vectoraggregators.yaml | 180 ++++++++++++++++++ internal/vector/aggregator/deployment.go | 27 +-- internal/vector/aggregator/deployment_test.go | 55 ++++++ internal/vector/aggregator/event_collector.go | 25 +-- .../vector/aggregator/statefulset_test.go | 14 ++ 11 files changed, 880 insertions(+), 25 deletions(-) diff --git a/api/v1alpha1/vector_common_types.go b/api/v1alpha1/vector_common_types.go index e1fc3335..8f8a9511 100644 --- a/api/v1alpha1/vector_common_types.go +++ b/api/v1alpha1/vector_common_types.go @@ -199,6 +199,14 @@ type VectorAggregatorCommon struct { Selector *VectorSelectorSpec `json:"selector,omitempty"` EventCollector EventCollector `json:"eventCollector,omitempty"` Autoscaling VectorAggregatorAutoscaling `json:"autoscaling,omitempty"` + // TopologySpreadConstraints spreads the aggregator pods across failure domains + // such as zones or nodes. Pod anti-affinity cannot express this, because the + // preferred form is only a score and the required form allows just one pod per + // domain, which caps the replica count. A constraint with maxSkew and + // whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + // Pending. When not set the scheduler places the pods freely. + // +optional + TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` // Persistence gives the aggregator durable storage for its data_dir. // When enabled the aggregator is rendered as a StatefulSet with a persistent // volume per replica instead of a Deployment. See VectorAggregatorPersistence. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3b9bdcc8..9c89fc60 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -412,6 +412,13 @@ func (in *VectorAggregatorCommon) DeepCopyInto(out *VectorAggregatorCommon) { } out.EventCollector = in.EventCollector in.Autoscaling.DeepCopyInto(&out.Autoscaling) + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Persistence.DeepCopyInto(&out.Persistence) in.PodDisruptionBudget.DeepCopyInto(&out.PodDisruptionBudget) } diff --git a/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml b/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml index bcafd5f7..caa13b53 100644 --- a/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml +++ b/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml @@ -4412,6 +4412,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml b/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml index 0708e96a..72d909c2 100644 --- a/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml +++ b/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml @@ -4406,6 +4406,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/docs/aggregator.md b/docs/aggregator.md index 24650b7b..c2fcae06 100644 --- a/docs/aggregator.md +++ b/docs/aggregator.md @@ -106,3 +106,52 @@ spec: password: test-password strategy: basic ``` + +## Spreading replicas across zones + +Both aggregator types accept `spec.topologySpreadConstraints`, passed straight to the pod +template. Use it to keep replicas out of a single zone or node, for failure isolation and to +avoid paying for cross zone traffic on every event. + +```yaml +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorAggregator +metadata: + name: vectorAggregator1 + namespace: vector +spec: + image: timberio/vector:0.48.0-debian + replicas: 6 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: vectorAggregator1 + app.kubernetes.io/component: Aggregator +``` + +The label selector decides which pods are counted per zone, so it has to match the aggregator +pods. The operator labels them with `app.kubernetes.io/instance: ` and +`app.kubernetes.io/component: Aggregator`. Select on both, so one aggregator is spread without +catching another in the same namespace, and without counting the event collector pod, which +shares the instance label but is labelled `component: EventCollector`. + +The constraints reach the event collector pod too, the same way `spec.affinity` and +`spec.tolerations` already do. It runs a single replica, so there is nothing there to spread. + +`spec.affinity` cannot express this. Pod anti-affinity has two forms and neither fits: +`preferredDuringSchedulingIgnoredDuringExecution` is only a scheduling score, so under a +provisioner like Karpenter, which adds nodes to fit pending pods rather than to satisfy +preferences, replicas still end up packed onto one node. `requiredDuringSchedulingIgnoredDuringExecution` +has no count, so it allows a single pod per zone and any replica above the zone count stays +Pending, which rules it out for [autoscaling](aggregator-autoscaling.md). + +Keep `whenUnsatisfiable: ScheduleAnyway`. With `DoNotSchedule` a zone outage or a scale up +past what the zones can hold leaves pods Pending rather than merely unevenly placed. + +The field works the same way with [persistence](aggregator-persistence.md) enabled, since the +Deployment and the StatefulSet share one pod template. Keep in mind that a volume is bound to +the zone it was provisioned in, so a rescheduled replica goes back to its original zone no +matter what the constraint says. diff --git a/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml b/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml index bcafd5f7..caa13b53 100644 --- a/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml +++ b/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml @@ -4412,6 +4412,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml b/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml index 0708e96a..72d909c2 100644 --- a/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml +++ b/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml @@ -4406,6 +4406,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/internal/vector/aggregator/deployment.go b/internal/vector/aggregator/deployment.go index 2fdb5daf..0d46fba6 100644 --- a/internal/vector/aggregator/deployment.go +++ b/internal/vector/aggregator/deployment.go @@ -64,19 +64,20 @@ func (ctrl *Controller) aggregatorPodTemplateSpec() corev1.PodTemplateSpec { return corev1.PodTemplateSpec{ ObjectMeta: ctrl.objectMetaVectorAggregator(labels, annotations, ctrl.Namespace), Spec: corev1.PodSpec{ - ServiceAccountName: ctrl.getNameVectorAggregator(), - Volumes: ctrl.generateVectorAggregatorVolume(), - SecurityContext: ctrl.Spec.SecurityContext, - ImagePullSecrets: ctrl.Spec.ImagePullSecrets, - Affinity: ctrl.Spec.Affinity, - RuntimeClassName: ctrl.Spec.RuntimeClassName, - SchedulerName: ctrl.Spec.SchedulerName, - Tolerations: ctrl.Spec.Tolerations, - PriorityClassName: ctrl.Spec.PriorityClassName, - HostNetwork: ctrl.Spec.HostNetwork, - HostAliases: ctrl.Spec.HostAliases, - InitContainers: initContainers, - Containers: containers, + ServiceAccountName: ctrl.getNameVectorAggregator(), + Volumes: ctrl.generateVectorAggregatorVolume(), + SecurityContext: ctrl.Spec.SecurityContext, + ImagePullSecrets: ctrl.Spec.ImagePullSecrets, + Affinity: ctrl.Spec.Affinity, + TopologySpreadConstraints: ctrl.Spec.TopologySpreadConstraints, + RuntimeClassName: ctrl.Spec.RuntimeClassName, + SchedulerName: ctrl.Spec.SchedulerName, + Tolerations: ctrl.Spec.Tolerations, + PriorityClassName: ctrl.Spec.PriorityClassName, + HostNetwork: ctrl.Spec.HostNetwork, + HostAliases: ctrl.Spec.HostAliases, + InitContainers: initContainers, + Containers: containers, }, } } diff --git a/internal/vector/aggregator/deployment_test.go b/internal/vector/aggregator/deployment_test.go index 9c6ac67c..cca15e47 100644 --- a/internal/vector/aggregator/deployment_test.go +++ b/internal/vector/aggregator/deployment_test.go @@ -4,10 +4,26 @@ import ( "testing" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" vectorv1alpha1 "github.com/kaasops/vector-operator/api/v1alpha1" ) +// zoneSpreadConstraint spreads pods evenly across zones without ever leaving one +// Pending, which is the case the field exists for. +func zoneSpreadConstraint() corev1.TopologySpreadConstraint { + return corev1.TopologySpreadConstraint{ + MaxSkew: 1, + TopologyKey: corev1.LabelTopologyZone, + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app.kubernetes.io/instance": "test"}, + }, + } +} + func TestGenerateVolumesKeepsPreUpgradeOrderWhenNotPersistent(t *testing.T) { g := NewWithT(t) @@ -23,3 +39,42 @@ func TestGenerateVolumesKeepsPreUpgradeOrderWhenNotPersistent(t *testing.T) { // pod template rolls every aggregator Deployment on operator upgrade. g.Expect(names).To(Equal([]string{"config", "data", "procfs", "sysfs"})) } + +func TestDeploymentTopologySpreadConstraints(t *testing.T) { + g := NewWithT(t) + + constraint := zoneSpreadConstraint() + ctrl := createTestController("test", "default", &vectorv1alpha1.VectorAggregatorCommon{ + Replicas: ptr.To(int32(3)), + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{constraint}, + }, false) + + deployment := ctrl.createVectorAggregatorDeployment() + + g.Expect(deployment.Spec.Template.Spec.TopologySpreadConstraints).To(Equal([]corev1.TopologySpreadConstraint{constraint})) +} + +func TestDeploymentTopologySpreadConstraintsUnset(t *testing.T) { + g := NewWithT(t) + + ctrl := createTestController("test", "default", &vectorv1alpha1.VectorAggregatorCommon{}, false) + + deployment := ctrl.createVectorAggregatorDeployment() + + // An unset field must leave the pod spec untouched, so existing aggregators do + // not roll on operator upgrade. + g.Expect(deployment.Spec.Template.Spec.TopologySpreadConstraints).To(BeNil()) +} + +func TestEventCollectorInheritsTopologySpreadConstraints(t *testing.T) { + g := NewWithT(t) + + constraint := zoneSpreadConstraint() + ctrl := createTestController("test", "default", &vectorv1alpha1.VectorAggregatorCommon{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{constraint}, + }, false) + + deployment := ctrl.createEventCollectorDeployment() + + g.Expect(deployment.Spec.Template.Spec.TopologySpreadConstraints).To(Equal([]corev1.TopologySpreadConstraint{constraint})) +} diff --git a/internal/vector/aggregator/event_collector.go b/internal/vector/aggregator/event_collector.go index ae81a580..7ce4f83d 100644 --- a/internal/vector/aggregator/event_collector.go +++ b/internal/vector/aggregator/event_collector.go @@ -136,18 +136,19 @@ func (ctrl *Controller) createEventCollectorDeployment() *appsv1.Deployment { Template: corev1.PodTemplateSpec{ ObjectMeta: ctrl.objectMetaVectorAggregator(labels, annotations, ctrl.Namespace), Spec: corev1.PodSpec{ - ServiceAccountName: ctrl.Name + "-event-collector", - Volumes: ctrl.generateEventCollectorVolume(), - SecurityContext: ctrl.Spec.SecurityContext, - ImagePullSecrets: ctrl.Spec.ImagePullSecrets, - Affinity: ctrl.Spec.Affinity, - RuntimeClassName: ctrl.Spec.RuntimeClassName, - SchedulerName: ctrl.Spec.SchedulerName, - Tolerations: ctrl.Spec.Tolerations, - PriorityClassName: ctrl.Spec.PriorityClassName, - HostNetwork: ctrl.Spec.HostNetwork, - HostAliases: ctrl.Spec.HostAliases, - Containers: containers, + ServiceAccountName: ctrl.Name + "-event-collector", + Volumes: ctrl.generateEventCollectorVolume(), + SecurityContext: ctrl.Spec.SecurityContext, + ImagePullSecrets: ctrl.Spec.ImagePullSecrets, + Affinity: ctrl.Spec.Affinity, + TopologySpreadConstraints: ctrl.Spec.TopologySpreadConstraints, + RuntimeClassName: ctrl.Spec.RuntimeClassName, + SchedulerName: ctrl.Spec.SchedulerName, + Tolerations: ctrl.Spec.Tolerations, + PriorityClassName: ctrl.Spec.PriorityClassName, + HostNetwork: ctrl.Spec.HostNetwork, + HostAliases: ctrl.Spec.HostAliases, + Containers: containers, }, }, }, diff --git a/internal/vector/aggregator/statefulset_test.go b/internal/vector/aggregator/statefulset_test.go index 749617b3..e9cb181f 100644 --- a/internal/vector/aggregator/statefulset_test.go +++ b/internal/vector/aggregator/statefulset_test.go @@ -68,6 +68,20 @@ func TestStatefulSetReplicasNilWhenAutoscaling(t *testing.T) { g.Expect(sts.Spec.Replicas).To(BeNil(), "replicas must be nil so the operator does not fight the HPA") } +func TestStatefulSetTopologySpreadConstraints(t *testing.T) { + g := NewWithT(t) + + constraint := zoneSpreadConstraint() + spec := persistentSpec() + spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{constraint} + ctrl := createTestController("test", "default", spec, false) + + sts := ctrl.createVectorAggregatorStatefulSet() + + // Both workloads share aggregatorPodTemplateSpec, so the persistent path spreads too. + g.Expect(sts.Spec.Template.Spec.TopologySpreadConstraints).To(Equal([]corev1.TopologySpreadConstraint{constraint})) +} + func TestVolumeClaimTemplatesEscapeHatch(t *testing.T) { g := NewWithT(t) From 75108f559959ec749a268a2514d5a34249671f5b Mon Sep 17 00:00:00 2001 From: Jared Patterson Date: Thu, 6 Aug 2026 11:59:33 +1200 Subject: [PATCH 2/2] docs: note that spread constraints replace the cluster defaults Setting any constraint drops the scheduler's built in defaults rather than adding to them, so a zone only example quietly gives up the node level spreading that came for free. Carry a hostname constraint alongside the zone one, and say why. Also narrow the cross zone traffic claim. The aggregator services are plain ClusterIP, so an even spread is a precondition for zone local routing rather than something that cuts cross zone traffic on its own, and the annotation that turns routing on is worth naming. --- docs/aggregator.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/aggregator.md b/docs/aggregator.md index c2fcae06..dc5b46fe 100644 --- a/docs/aggregator.md +++ b/docs/aggregator.md @@ -110,8 +110,8 @@ spec: ## Spreading replicas across zones Both aggregator types accept `spec.topologySpreadConstraints`, passed straight to the pod -template. Use it to keep replicas out of a single zone or node, for failure isolation and to -avoid paying for cross zone traffic on every event. +template. Use it to keep replicas out of a single zone or node, so losing one of them does not +take the aggregator down with it. ```yaml apiVersion: observability.kaasops.io/v1alpha1 @@ -130,9 +130,23 @@ spec: matchLabels: app.kubernetes.io/instance: vectorAggregator1 app.kubernetes.io/component: Aggregator + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: vectorAggregator1 + app.kubernetes.io/component: Aggregator ``` -The label selector decides which pods are counted per zone, so it has to match the aggregator +The example carries a second constraint because setting any constraint replaces the cluster +level defaults instead of adding to them. The scheduler's built in defaults spread with +`maxSkew: 3` on `kubernetes.io/hostname` and `maxSkew: 5` on zone, and they apply only to a pod +that defines none of its own. A zone only constraint therefore trades node level spreading +away: six replicas the scheduler used to place one per node can end up two per node, which +undoes the isolation the field is here for. + +The label selector decides which pods are counted per domain, so it has to match the aggregator pods. The operator labels them with `app.kubernetes.io/instance: ` and `app.kubernetes.io/component: Aggregator`. Select on both, so one aggregator is spread without catching another in the same namespace, and without counting the event collector pod, which @@ -151,6 +165,12 @@ Pending, which rules it out for [autoscaling](aggregator-autoscaling.md). Keep `whenUnsatisfiable: ScheduleAnyway`. With `DoNotSchedule` a zone outage or a scale up past what the zones can hold leaves pods Pending rather than merely unevenly placed. +An even spread is a precondition for zone local routing, not a way to get it. The operator +creates plain ClusterIP services, so kube-proxy picks an endpoint anywhere in the cluster and +events still cross zones until topology aware routing is turned on for the service itself. +`spec.annotations` is applied to the aggregator services, so +`service.kubernetes.io/topology-mode: Auto` can be set there. + The field works the same way with [persistence](aggregator-persistence.md) enabled, since the Deployment and the StatefulSet share one pod template. Keep in mind that a volume is bound to the zone it was provisioned in, so a rescheduled replica goes back to its original zone no