diff --git a/api/v1beta1/machine_types.go b/api/v1beta1/machine_types.go index fccc584fe5a4..94b840ef07a0 100644 --- a/api/v1beta1/machine_types.go +++ b/api/v1beta1/machine_types.go @@ -66,10 +66,10 @@ const ( // * KCP adds its own pre-terminate hook on all Machines it controls. This is done to ensure it can later remove // the etcd member right before Machine termination (i.e. before InfraMachine deletion). // * Starting with Kubernetes v1.31 the KCP pre-terminate hook will wait for all other pre-terminate hooks to finish to - // ensure it runs last (thus ensuring that kubelet is still working while other pre-terminate hooks run). This is only done - // for v1.31 or above because the kubeadm ControlPlaneKubeletLocalMode was introduced with kubeadm 1.31. This feature configures - // the kubelet to communicate with the local apiserver. Only because of that the kubelet immediately starts failing after the etcd - // member is removed. We need the ControlPlaneKubeletLocalMode feature with 1.31 to adhere to the kubelet skew policy. + // ensure it runs last (thus ensuring that kubelet is still working while other pre-terminate hooks run). This is done + // for v1.31 or above because the kubeadm ControlPlaneKubeletLocalMode was introduced with kubeadm 1.31 (graduated to GA in 1.36). + // This feature configures the kubelet to communicate with the local apiserver. Only because of that the kubelet immediately + // starts failing after the etcd member is removed. We need the ControlPlaneKubeletLocalMode feature with 1.31+ to adhere to the kubelet skew policy. PreTerminateDeleteHookAnnotationPrefix = "pre-terminate.delete.hook.machine.cluster.x-k8s.io" // MachineCertificatesExpiryDateAnnotation annotation specifies the expiry date of the machine certificates in RFC3339 format. diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go index 048df78ff59f..3cfc0067a552 100644 --- a/controlplane/kubeadm/internal/workload_cluster.go +++ b/controlplane/kubeadm/internal/workload_cluster.go @@ -81,6 +81,12 @@ var ( // the spec.clusterIP field selector that is only implemented in kube-apiserver >= 1.31.0). minKubernetesVersionControlPlaneKubeletLocalMode = semver.MustParse("1.31.0") + // droppedKubernetesVersionControlPlaneKubeletLocalMode is the version from which + // we will drop the ControlPlaneKubeletLocalMode kubeadm feature gate. + // Starting with Kubernetes 1.36, this feature graduated to GA and the feature gate + // is no longer needed (and will be removed in future K8s versions). + droppedKubernetesVersionControlPlaneKubeletLocalMode = semver.MustParse("1.36.0") + // ErrControlPlaneMinNodes signals that a cluster doesn't meet the minimum required nodes // to remove an etcd member. ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported") @@ -199,7 +205,10 @@ const ( // DefaultFeatureGates defaults the feature gates field. func DefaultFeatureGates(kubeadmConfigSpec *bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) { - if kubernetesVersion.LT(minKubernetesVersionControlPlaneKubeletLocalMode) { + // Only set ControlPlaneKubeletLocalMode for Kubernetes versions 1.31 <= version < 1.36 + // For K8s < 1.31: feature gate doesn't exist + // For K8s >= 1.36: feature graduated to GA and gate does not exist anymore + if kubernetesVersion.LT(minKubernetesVersionControlPlaneKubeletLocalMode) || kubernetesVersion.GTE(droppedKubernetesVersionControlPlaneKubeletLocalMode) { return } diff --git a/controlplane/kubeadm/internal/workload_cluster_test.go b/controlplane/kubeadm/internal/workload_cluster_test.go index 7ff30cbd0da6..2532544f3f0b 100644 --- a/controlplane/kubeadm/internal/workload_cluster_test.go +++ b/controlplane/kubeadm/internal/workload_cluster_test.go @@ -1261,6 +1261,36 @@ func TestUpdateFeatureGatesInKubeadmConfigMap(t *testing.T) { }, }, }, + { + name: "it should not add ControlPlaneKubeletLocalMode feature gate for 1.36.0 (GA)", + clusterConfigurationData: utilyaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration`), + kubernetesVersion: semver.MustParse("1.36.0"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: nil, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: nil, + }, + }, + { + name: "it should not add ControlPlaneKubeletLocalMode feature gate for 1.36.0 even with other feature gates", + clusterConfigurationData: utilyaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration`), + kubernetesVersion: semver.MustParse("1.36.0"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, + }, } for _, tt := range tests { diff --git a/docs/book/src/reference/versions.md b/docs/book/src/reference/versions.md index bf4890c2988a..178eea9838ac 100644 --- a/docs/book/src/reference/versions.md +++ b/docs/book/src/reference/versions.md @@ -144,6 +144,11 @@ The Kubeadm Control Plane requires the Kubeadm Bootstrap Provider. #### Kubernetes version specific notes +**1.36**: + +* Kubeadm Bootstrap provider: + * The `ControlPlaneKubeletLocalMode` feature gate graduated to GA and is enabled by default. CAPI will no longer set this feature gate explicitly for Kubernetes 1.36+. For Kubernetes versions 1.31-1.35, CAPI continues to set it automatically to ensure kubelet continues working during control plane upgrades. + **1.31**: * All providers: diff --git a/test/e2e/kcp_adoption.go b/test/e2e/kcp_adoption.go index 932abb5fc317..4ab7f7af8f9e 100644 --- a/test/e2e/kcp_adoption.go +++ b/test/e2e/kcp_adoption.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "bytes" "context" "fmt" "os" @@ -24,6 +25,7 @@ import ( "strings" "time" + "github.com/blang/semver/v4" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -38,6 +40,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/version" ) // KCPAdoptionSpecInput is the input for KCPAdoptionSpec. @@ -142,6 +145,12 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu }) Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") + v, err := semver.ParseTolerant(input.E2EConfig.GetVariable(KubernetesVersion)) + Expect(err).ToNot(HaveOccurred()) + if version.Compare(v, semver.MustParse("1.36.0"), version.WithoutPreReleases()) >= 0 { + workloadClusterTemplate = bytes.Replace(workloadClusterTemplate, []byte("featureGates:\n ControlPlaneKubeletLocalMode: true"), []byte(""), 1) + } + By("Applying the cluster template yaml to the cluster with the 'initial' selector") selector := labels.NewSelector().Add(must(labels.NewRequirement("kcp-adoption.step1", selection.Exists, nil))) Expect(input.BootstrapClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate, framework.WithLabelSelector(selector))).ShouldNot(HaveOccurred())