diff --git a/api/v1alpha1/vector_common_types.go b/api/v1alpha1/vector_common_types.go index e1fc333..8f8a951 100644 --- a/api/v1alpha1/vector_common_types.go +++ b/api/v1alpha1/vector_common_types.go @@ -199,6 +199,14 @@ type VectorAggregatorCommon struct { Selector *VectorSelectorSpec `json:"selector,omitempty"` EventCollector EventCollector `json:"eventCollector,omitempty"` Autoscaling VectorAggregatorAutoscaling `json:"autoscaling,omitempty"` + // TopologySpreadConstraints spreads the aggregator pods across failure domains + // such as zones or nodes. Pod anti-affinity cannot express this, because the + // preferred form is only a score and the required form allows just one pod per + // domain, which caps the replica count. A constraint with maxSkew and + // whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + // Pending. When not set the scheduler places the pods freely. + // +optional + TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` // Persistence gives the aggregator durable storage for its data_dir. // When enabled the aggregator is rendered as a StatefulSet with a persistent // volume per replica instead of a Deployment. See VectorAggregatorPersistence. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3b9bdcc..9c89fc6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -412,6 +412,13 @@ func (in *VectorAggregatorCommon) DeepCopyInto(out *VectorAggregatorCommon) { } out.EventCollector = in.EventCollector in.Autoscaling.DeepCopyInto(&out.Autoscaling) + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Persistence.DeepCopyInto(&out.Persistence) in.PodDisruptionBudget.DeepCopyInto(&out.PodDisruptionBudget) } diff --git a/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml b/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml index bcafd5f..caa13b5 100644 --- a/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml +++ b/config/crd/bases/observability.kaasops.io_clustervectoraggregators.yaml @@ -4412,6 +4412,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml b/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml index 0708e96..72d909c 100644 --- a/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml +++ b/config/crd/bases/observability.kaasops.io_vectoraggregators.yaml @@ -4406,6 +4406,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/docs/aggregator.md b/docs/aggregator.md index 24650b7..dc5b46f 100644 --- a/docs/aggregator.md +++ b/docs/aggregator.md @@ -106,3 +106,72 @@ spec: password: test-password strategy: basic ``` + +## Spreading replicas across zones + +Both aggregator types accept `spec.topologySpreadConstraints`, passed straight to the pod +template. Use it to keep replicas out of a single zone or node, so losing one of them does not +take the aggregator down with it. + +```yaml +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorAggregator +metadata: + name: vectorAggregator1 + namespace: vector +spec: + image: timberio/vector:0.48.0-debian + replicas: 6 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: vectorAggregator1 + app.kubernetes.io/component: Aggregator + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: vectorAggregator1 + app.kubernetes.io/component: Aggregator +``` + +The example carries a second constraint because setting any constraint replaces the cluster +level defaults instead of adding to them. The scheduler's built in defaults spread with +`maxSkew: 3` on `kubernetes.io/hostname` and `maxSkew: 5` on zone, and they apply only to a pod +that defines none of its own. A zone only constraint therefore trades node level spreading +away: six replicas the scheduler used to place one per node can end up two per node, which +undoes the isolation the field is here for. + +The label selector decides which pods are counted per domain, so it has to match the aggregator +pods. The operator labels them with `app.kubernetes.io/instance: ` and +`app.kubernetes.io/component: Aggregator`. Select on both, so one aggregator is spread without +catching another in the same namespace, and without counting the event collector pod, which +shares the instance label but is labelled `component: EventCollector`. + +The constraints reach the event collector pod too, the same way `spec.affinity` and +`spec.tolerations` already do. It runs a single replica, so there is nothing there to spread. + +`spec.affinity` cannot express this. Pod anti-affinity has two forms and neither fits: +`preferredDuringSchedulingIgnoredDuringExecution` is only a scheduling score, so under a +provisioner like Karpenter, which adds nodes to fit pending pods rather than to satisfy +preferences, replicas still end up packed onto one node. `requiredDuringSchedulingIgnoredDuringExecution` +has no count, so it allows a single pod per zone and any replica above the zone count stays +Pending, which rules it out for [autoscaling](aggregator-autoscaling.md). + +Keep `whenUnsatisfiable: ScheduleAnyway`. With `DoNotSchedule` a zone outage or a scale up +past what the zones can hold leaves pods Pending rather than merely unevenly placed. + +An even spread is a precondition for zone local routing, not a way to get it. The operator +creates plain ClusterIP services, so kube-proxy picks an endpoint anywhere in the cluster and +events still cross zones until topology aware routing is turned on for the service itself. +`spec.annotations` is applied to the aggregator services, so +`service.kubernetes.io/topology-mode: Auto` can be set there. + +The field works the same way with [persistence](aggregator-persistence.md) enabled, since the +Deployment and the StatefulSet share one pod template. Keep in mind that a volume is bound to +the zone it was provisioned in, so a rescheduled replica goes back to its original zone no +matter what the constraint says. diff --git a/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml b/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml index bcafd5f..caa13b5 100644 --- a/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml +++ b/helm/charts/vector-operator/crds/observability.kaasops.io_clustervectoraggregators.yaml @@ -4412,6 +4412,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml b/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml index 0708e96..72d909c 100644 --- a/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml +++ b/helm/charts/vector-operator/crds/observability.kaasops.io_vectoraggregators.yaml @@ -4406,6 +4406,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints spreads the aggregator pods across failure domains + such as zones or nodes. Pod anti-affinity cannot express this, because the + preferred form is only a score and the required form allows just one pod per + domain, which caps the replica count. A constraint with maxSkew and + whenUnsatisfiable: ScheduleAnyway spreads evenly without ever leaving a pod + Pending. When not set the scheduler places the pods freely. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. items: diff --git a/internal/vector/aggregator/deployment.go b/internal/vector/aggregator/deployment.go index 2fdb5da..0d46fba 100644 --- a/internal/vector/aggregator/deployment.go +++ b/internal/vector/aggregator/deployment.go @@ -64,19 +64,20 @@ func (ctrl *Controller) aggregatorPodTemplateSpec() corev1.PodTemplateSpec { return corev1.PodTemplateSpec{ ObjectMeta: ctrl.objectMetaVectorAggregator(labels, annotations, ctrl.Namespace), Spec: corev1.PodSpec{ - ServiceAccountName: ctrl.getNameVectorAggregator(), - Volumes: ctrl.generateVectorAggregatorVolume(), - SecurityContext: ctrl.Spec.SecurityContext, - ImagePullSecrets: ctrl.Spec.ImagePullSecrets, - Affinity: ctrl.Spec.Affinity, - RuntimeClassName: ctrl.Spec.RuntimeClassName, - SchedulerName: ctrl.Spec.SchedulerName, - Tolerations: ctrl.Spec.Tolerations, - PriorityClassName: ctrl.Spec.PriorityClassName, - HostNetwork: ctrl.Spec.HostNetwork, - HostAliases: ctrl.Spec.HostAliases, - InitContainers: initContainers, - Containers: containers, + ServiceAccountName: ctrl.getNameVectorAggregator(), + Volumes: ctrl.generateVectorAggregatorVolume(), + SecurityContext: ctrl.Spec.SecurityContext, + ImagePullSecrets: ctrl.Spec.ImagePullSecrets, + Affinity: ctrl.Spec.Affinity, + TopologySpreadConstraints: ctrl.Spec.TopologySpreadConstraints, + RuntimeClassName: ctrl.Spec.RuntimeClassName, + SchedulerName: ctrl.Spec.SchedulerName, + Tolerations: ctrl.Spec.Tolerations, + PriorityClassName: ctrl.Spec.PriorityClassName, + HostNetwork: ctrl.Spec.HostNetwork, + HostAliases: ctrl.Spec.HostAliases, + InitContainers: initContainers, + Containers: containers, }, } } diff --git a/internal/vector/aggregator/deployment_test.go b/internal/vector/aggregator/deployment_test.go index 9c6ac67..cca15e4 100644 --- a/internal/vector/aggregator/deployment_test.go +++ b/internal/vector/aggregator/deployment_test.go @@ -4,10 +4,26 @@ import ( "testing" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" vectorv1alpha1 "github.com/kaasops/vector-operator/api/v1alpha1" ) +// zoneSpreadConstraint spreads pods evenly across zones without ever leaving one +// Pending, which is the case the field exists for. +func zoneSpreadConstraint() corev1.TopologySpreadConstraint { + return corev1.TopologySpreadConstraint{ + MaxSkew: 1, + TopologyKey: corev1.LabelTopologyZone, + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app.kubernetes.io/instance": "test"}, + }, + } +} + func TestGenerateVolumesKeepsPreUpgradeOrderWhenNotPersistent(t *testing.T) { g := NewWithT(t) @@ -23,3 +39,42 @@ func TestGenerateVolumesKeepsPreUpgradeOrderWhenNotPersistent(t *testing.T) { // pod template rolls every aggregator Deployment on operator upgrade. g.Expect(names).To(Equal([]string{"config", "data", "procfs", "sysfs"})) } + +func TestDeploymentTopologySpreadConstraints(t *testing.T) { + g := NewWithT(t) + + constraint := zoneSpreadConstraint() + ctrl := createTestController("test", "default", &vectorv1alpha1.VectorAggregatorCommon{ + Replicas: ptr.To(int32(3)), + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{constraint}, + }, false) + + deployment := ctrl.createVectorAggregatorDeployment() + + g.Expect(deployment.Spec.Template.Spec.TopologySpreadConstraints).To(Equal([]corev1.TopologySpreadConstraint{constraint})) +} + +func TestDeploymentTopologySpreadConstraintsUnset(t *testing.T) { + g := NewWithT(t) + + ctrl := createTestController("test", "default", &vectorv1alpha1.VectorAggregatorCommon{}, false) + + deployment := ctrl.createVectorAggregatorDeployment() + + // An unset field must leave the pod spec untouched, so existing aggregators do + // not roll on operator upgrade. + g.Expect(deployment.Spec.Template.Spec.TopologySpreadConstraints).To(BeNil()) +} + +func TestEventCollectorInheritsTopologySpreadConstraints(t *testing.T) { + g := NewWithT(t) + + constraint := zoneSpreadConstraint() + ctrl := createTestController("test", "default", &vectorv1alpha1.VectorAggregatorCommon{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{constraint}, + }, false) + + deployment := ctrl.createEventCollectorDeployment() + + g.Expect(deployment.Spec.Template.Spec.TopologySpreadConstraints).To(Equal([]corev1.TopologySpreadConstraint{constraint})) +} diff --git a/internal/vector/aggregator/event_collector.go b/internal/vector/aggregator/event_collector.go index ae81a58..7ce4f83 100644 --- a/internal/vector/aggregator/event_collector.go +++ b/internal/vector/aggregator/event_collector.go @@ -136,18 +136,19 @@ func (ctrl *Controller) createEventCollectorDeployment() *appsv1.Deployment { Template: corev1.PodTemplateSpec{ ObjectMeta: ctrl.objectMetaVectorAggregator(labels, annotations, ctrl.Namespace), Spec: corev1.PodSpec{ - ServiceAccountName: ctrl.Name + "-event-collector", - Volumes: ctrl.generateEventCollectorVolume(), - SecurityContext: ctrl.Spec.SecurityContext, - ImagePullSecrets: ctrl.Spec.ImagePullSecrets, - Affinity: ctrl.Spec.Affinity, - RuntimeClassName: ctrl.Spec.RuntimeClassName, - SchedulerName: ctrl.Spec.SchedulerName, - Tolerations: ctrl.Spec.Tolerations, - PriorityClassName: ctrl.Spec.PriorityClassName, - HostNetwork: ctrl.Spec.HostNetwork, - HostAliases: ctrl.Spec.HostAliases, - Containers: containers, + ServiceAccountName: ctrl.Name + "-event-collector", + Volumes: ctrl.generateEventCollectorVolume(), + SecurityContext: ctrl.Spec.SecurityContext, + ImagePullSecrets: ctrl.Spec.ImagePullSecrets, + Affinity: ctrl.Spec.Affinity, + TopologySpreadConstraints: ctrl.Spec.TopologySpreadConstraints, + RuntimeClassName: ctrl.Spec.RuntimeClassName, + SchedulerName: ctrl.Spec.SchedulerName, + Tolerations: ctrl.Spec.Tolerations, + PriorityClassName: ctrl.Spec.PriorityClassName, + HostNetwork: ctrl.Spec.HostNetwork, + HostAliases: ctrl.Spec.HostAliases, + Containers: containers, }, }, }, diff --git a/internal/vector/aggregator/statefulset_test.go b/internal/vector/aggregator/statefulset_test.go index 749617b..e9cb181 100644 --- a/internal/vector/aggregator/statefulset_test.go +++ b/internal/vector/aggregator/statefulset_test.go @@ -68,6 +68,20 @@ func TestStatefulSetReplicasNilWhenAutoscaling(t *testing.T) { g.Expect(sts.Spec.Replicas).To(BeNil(), "replicas must be nil so the operator does not fight the HPA") } +func TestStatefulSetTopologySpreadConstraints(t *testing.T) { + g := NewWithT(t) + + constraint := zoneSpreadConstraint() + spec := persistentSpec() + spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{constraint} + ctrl := createTestController("test", "default", spec, false) + + sts := ctrl.createVectorAggregatorStatefulSet() + + // Both workloads share aggregatorPodTemplateSpec, so the persistent path spreads too. + g.Expect(sts.Spec.Template.Spec.TopologySpreadConstraints).To(Equal([]corev1.TopologySpreadConstraint{constraint})) +} + func TestVolumeClaimTemplatesEscapeHatch(t *testing.T) { g := NewWithT(t)