From 664191b81fa13cd6a71fdc698d0b0534a0120cf1 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Tue, 25 Oct 2022 12:49:49 +0530 Subject: [PATCH 01/13] [WIP] PCP-22: Boostrap and CAPI changes --- .../controllers/kubeadmconfig_controller.go | 22 +++- .../internal/controllers/controller.go | 16 ++- .../kubeadm/internal/controllers/helpers.go | 1 + .../kubeadm/internal/controllers/status.go | 10 +- .../cluster/cluster_controller_phases.go | 44 +++++-- util/kubeconfig/kubeconfig.go | 69 ++++++++++ util/secret/certificates.go | 122 +++++++++++++++++- util/secret/consts.go | 1 + util/secret/secret.go | 4 +- 9 files changed, 265 insertions(+), 24 deletions(-) diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 0a5fd52b2cdb..23a819e19a4c 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "log" "strconv" "time" @@ -262,10 +263,13 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } + //TODO: PCP-22 check (annotation to skip handleClusterNotInitialized and go for join ) + //how to make this condition true for new cluster as kubeadm cluster is already initialized // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. - if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { - return r.handleClusterNotInitialized(ctx, scope) - } + log.Info("TESTING... skip handx``leClusterNotInitialized and push cluster for join") + //if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { + // return r.handleClusterNotInitialized(ctx, scope) + //} // Every other case it's a join scenario // Nb. in this case ClusterConfiguration and InitConfiguration should not be defined by users, but in case of misconfigurations, CABPK simply ignore them @@ -281,10 +285,12 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques // it's a control plane join if configOwner.IsControlPlaneMachine() { + log.Info("TESTING.... joinControlplane") return r.joinControlplane(ctx, scope) } // It's a worker join + log.Info("TESTING.... It's a worker join") return r.joinWorker(ctx, scope) } @@ -342,6 +348,9 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex // initialize the DataSecretAvailableCondition if missing. // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing // using the DataSecretGeneratedFailedReason + + scope.Info("TESTING.... In handleClusterNotInitialized") + if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason { conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "") } @@ -424,6 +433,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex return ctrl.Result{}, err } + scope.Info("TESTING.... LookupOrGenerate new certificates") certificates := secret.NewCertificatesForInitialControlPlane(scope.Config.Spec.ClusterConfiguration) err = certificates.LookupOrGenerate( ctx, @@ -495,6 +505,10 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex } func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) { + + scope.Info("TESTING.... joinWorker") + log.Println("TESTING.... joinWorker") + certificates := secret.NewCertificatesForWorker(scope.Config.Spec.JoinConfiguration.CACertPath) err := certificates.Lookup( ctx, @@ -600,6 +614,8 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S scope.Config.Spec.JoinConfiguration.ControlPlane = &bootstrapv1.JoinControlPlane{} } + scope.Info("TESTING.... NewControlPlaneJoinCerts") + log.Println("TESTING.... joinControlplane") certificates := secret.NewControlPlaneJoinCerts(scope.Config.Spec.ClusterConfiguration) err := certificates.Lookup( ctx, diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 081c57de3026..7bc232284b77 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -261,10 +261,15 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * if config.ClusterConfiguration == nil { config.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{} } + + //TODO: PCP-22 lookup or generate ca, sa, etcd certificates and key certificates := secret.NewCertificatesForInitialControlPlane(config.ClusterConfiguration) + //for _, certificate := range certificates { + // log.Info("TESTING.... lookup or generate ca, sa, etcd certificates and key: ", certificate) + //} controllerRef := metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("KubeadmControlPlane")) if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { - log.Error(err, "unable to lookup or create cluster certificates") + log.Error(err, "TESTING.... unable to lookup or create cluster certificates") conditions.MarkFalse(kcp, controlplanev1.CertificatesAvailableCondition, controlplanev1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) return ctrl.Result{}, err } @@ -276,6 +281,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * return ctrl.Result{}, nil } + //TODO: PCP-22 adopt kubeconfig instead of generating new // Generate Cluster Kubeconfig if needed if result, err := r.reconcileKubeconfig(ctx, cluster, kcp); !result.IsZero() || err != nil { if err != nil { @@ -352,6 +358,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * desiredReplicas := int(*kcp.Spec.Replicas) switch { + //TODO: PCP-22 skip creating new control plane // We are creating the first replica case numMachines < desiredReplicas && numMachines == 0: // Create new Machine w/ init @@ -359,7 +366,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * conditions.MarkFalse(controlPlane.KCP, controlplanev1.AvailableCondition, controlplanev1.WaitingForKubeadmInitReason, clusterv1.ConditionSeverityInfo, "") return r.initializeControlPlane(ctx, cluster, kcp, controlPlane) // We are scaling up - case numMachines < desiredReplicas && numMachines > 0: + case numMachines < desiredReplicas && numMachines >= 0: // Create a new Machine w/ join log.Info("Scaling up control plane", "Desired", desiredReplicas, "Existing", numMachines) return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) @@ -528,6 +535,11 @@ func (r *KubeadmControlPlaneReconciler) reconcileEtcdMembers(ctx context.Context log := ctrl.LoggerFrom(ctx, "cluster", controlPlane.Cluster.Name) // If etcd is not managed by KCP this is a no-op. + if true { + //TODO: PCP-22 + return ctrl.Result{}, nil + } + if !controlPlane.IsEtcdManaged() { return ctrl.Result{}, nil } diff --git a/controlplane/kubeadm/internal/controllers/helpers.go b/controlplane/kubeadm/internal/controllers/helpers.go index 31034141ca9f..d9c6ee767baa 100644 --- a/controlplane/kubeadm/internal/controllers/helpers.go +++ b/controlplane/kubeadm/internal/controllers/helpers.go @@ -75,6 +75,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, // check if the kubeconfig secret was created by v1alpha2 controllers, and thus it has the Cluster as the owner instead of KCP; // if yes, adopt it. + //TODO: PCP-22 Need to inject original Kubeconfig instead of generating new if util.IsOwnedByObject(configSecret, cluster) && !util.IsControlledBy(configSecret, kcp) { if err := r.adoptKubeconfigSecret(ctx, cluster, configSecret, controllerOwnerRef); err != nil { return ctrl.Result{}, err diff --git a/controlplane/kubeadm/internal/controllers/status.go b/controlplane/kubeadm/internal/controllers/status.go index 1617827ff3e5..ee27bab1f3f5 100644 --- a/controlplane/kubeadm/internal/controllers/status.go +++ b/controlplane/kubeadm/internal/controllers/status.go @@ -106,11 +106,13 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c kcp.Status.ReadyReplicas = status.ReadyNodes kcp.Status.UnavailableReplicas = replicas - status.ReadyNodes + //TODO: PCP-22 Initialized should be true to join new node to cluster or else it will try to init // This only gets initialized once and does not change if the kubeadm config map goes away. - if status.HasKubeadmConfig { - kcp.Status.Initialized = true - conditions.MarkTrue(kcp, controlplanev1.AvailableCondition) - } + //if status.HasKubeadmConfig { + log.Info("TESTING.... set kcp.Status.Initialized to true") + kcp.Status.Initialized = true + conditions.MarkTrue(kcp, controlplanev1.AvailableCondition) + //} if kcp.Status.ReadyReplicas > 0 { kcp.Status.Ready = true diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index 589c8a0a9db6..88e43b6a9f65 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -245,6 +245,9 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster if err != nil { return ctrl.Result{}, err } + + //TODO: PCP-22 + conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) if initialized { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) } else { @@ -265,23 +268,44 @@ func (r *Reconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1 // Do not generate the Kubeconfig if there is a ControlPlaneRef, since the Control Plane provider is // responsible for the management of the Kubeconfig. We continue to manage it here only for backward // compatibility when a Control Plane provider is not in use. + + log.Info("TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef", "cluster.Spec.ControlPlaneRef", cluster.Spec.ControlPlaneRef) + //TODO: PCP-22 comment this to let secret generation for now, ControlPlaneRef is present already if cluster.Spec.ControlPlaneRef != nil { return ctrl.Result{}, nil } _, err := secret.Get(ctx, r.Client, util.ObjectKey(cluster), secret.Kubeconfig) - switch { - case apierrors.IsNotFound(err): - if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil { - if err == kubeconfig.ErrDependentCertificateNotFound { - log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA) - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - return ctrl.Result{}, err + if err != nil { + log.Info("TESTING.... error getting kubeconfig", "err", err) + } + + // TODO: PCP-22 read kubeconfig secrets from kube-system namespace + log.Error(nil, "TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef") + if err := kubeconfig.ReadSecret(ctx, r.Client, cluster); err != nil { + if err == kubeconfig.ErrDependentCertificateNotFound { + log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + if err == kubeconfig.ErrAlreadyExists { + log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - case err != nil: - return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace) + return ctrl.Result{}, err } + //switch { + //case apierrors.IsNotFound(err): + // if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil { + // if err == kubeconfig.ErrDependentCertificateNotFound { + // log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA) + // return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + // } + // return ctrl.Result{}, err + // } + //case err != nil: + // return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace) + //} + return ctrl.Result{}, nil } diff --git a/util/kubeconfig/kubeconfig.go b/util/kubeconfig/kubeconfig.go index 2c8872bbb723..6e5332c0171e 100644 --- a/util/kubeconfig/kubeconfig.go +++ b/util/kubeconfig/kubeconfig.go @@ -22,6 +22,7 @@ import ( "crypto" "crypto/x509" "fmt" + "log" "time" "github.com/pkg/errors" @@ -41,6 +42,7 @@ import ( var ( // ErrDependentCertificateNotFound signals that a CA secret could not be found. ErrDependentCertificateNotFound = errors.New("could not find secret ca") + ErrAlreadyExists = errors.New("secrets \"t-cluster-kubeconfig\" already exists") ) // FromSecret fetches the Kubeconfig for a Cluster. @@ -107,6 +109,72 @@ func CreateSecret(ctx context.Context, c client.Client, cluster *clusterv1.Clust }) } +// ReadSecret reads the Kubeconfig secret from kube-system +func ReadSecret(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) error { + log.Println("TESTING.... IN ReadSecret") + + name := util.ObjectKey(cluster) + return ReadSecretWithOwner(ctx, c, name, cluster.Spec.ControlPlaneEndpoint.String(), metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + }) +} + +// ReadSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. +func ReadSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { + //server := fmt.Sprintf("https://%s", endpoint) + //out, err := ReadExistingSecret(ctx, c, clusterName, server) + //if err != nil { + // return err + //} + log.Println("TESTING.... IN ReadSecretWithOwner") + + //clusterName := util.ObjectKey(clusterName) + configSecret, err := secret.GetFromNamespacedName(ctx, c, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: clusterName.Name}, secret.Kubeconfig) + if err != nil { + log.Println("TESTING....", "error in getting kubeconfig: ", err) + return err + } + + data, err := toKubeconfigBytes(configSecret) + if err != nil { + log.Println("TESTING....", "error in parsing kubeconfig: ", err) + return err + } + //if err := ReadExistingSecret(ctx, r.Client, configSecret); err != nil { + // return ctrl.Result{}, errors.Wrap(err, "failed to regenerate kubeconfig") + //} + + return c.Create(ctx, GenerateSecretWithOwner(clusterName, data, owner)) +} + +// ReadExistingSecret creates and stores a new Kubeconfig in the given secret. +func ReadExistingSecret(ctx context.Context, c client.Client, configSecret *corev1.Secret) error { + clusterName, _, err := secret.ParseSecretName(configSecret.Name) + if err != nil { + return errors.Wrap(err, "failed to parse secret name") + } + data, err := toKubeconfigBytes(configSecret) + if err != nil { + return err + } + + config, err := clientcmd.Load(data) + if err != nil { + return errors.Wrap(err, "failed to convert kubeconfig Secret into a clientcmdapi.Config") + } + endpoint := config.Clusters[clusterName].Server + key := client.ObjectKey{Name: clusterName, Namespace: configSecret.Namespace} + out, err := generateKubeconfig(ctx, c, key, endpoint) + if err != nil { + return err + } + configSecret.Data[secret.KubeconfigDataName] = out + return c.Update(ctx, configSecret) +} + // CreateSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. func CreateSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { server := fmt.Sprintf("https://%s", endpoint) @@ -237,6 +305,7 @@ func generateKubeconfig(ctx context.Context, c client.Client, clusterName client } func toKubeconfigBytes(out *corev1.Secret) ([]byte, error) { + //data, ok := out.Data[secret.KubeconfigDataName2] data, ok := out.Data[secret.KubeconfigDataName] if !ok { return nil, errors.Errorf("missing key %q in secret data", secret.KubeconfigDataName) diff --git a/util/secret/certificates.go b/util/secret/certificates.go index e3aadfbc7a37..06169e293d7b 100644 --- a/util/secret/certificates.go +++ b/util/secret/certificates.go @@ -24,6 +24,8 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" + "fmt" + "log" "math/big" "path/filepath" "strings" @@ -207,12 +209,75 @@ func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clus return errors.WithStack(err) } // If a user has a badly formatted secret it will prevent the cluster from working. + //kp, err := secretToKeyPair(s, certificate.Purpose) kp, err := secretToKeyPair(s) if err != nil { return err } certificate.KeyPair = kp } + + //err := c.LookupKubeadm(ctx, ctrlclient, clusterName) + //if err != nil { + // log.Println("TESTING.... error:", err) + // return err + //} + return nil +} + +// LookupKubeadm looks up each certificate from secrets and populates the certificate with the secret data. +func (c Certificates) LookupKubeadm(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey) error { + // Look up each certificate as a secret and populate the certificate/key + + fmt.Println("TESTING.... LookupKubeadm") + for _, certificate := range c { + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: Name(clusterName.Name, certificate.Purpose), + Namespace: clusterName.Namespace, + } + + //key := client.ObjectKey{ + // Name: "kubeadm-certs", + // Namespace: "kube-system", + //} + + if err := ctrlclient.Get(ctx, key, s); err != nil { + if apierrors.IsNotFound(err) { + if certificate.External { + //log.Println("TESTING.... ERROR external certificate not found: ", certificate.Purpose) + //fmt.Println("TESTING.... ERROR external certificate not found: ", certificate.Purpose) + return errors.WithMessage(err, "external certificate not found") + } + + //fmt.Println("TESTING..... err:", err) + continue + } + //log.Println("TESTING.... ERROR: ", certificate.Purpose) + //fmt.Println("TESTING.... ERROR: ", certificate.Purpose) + return errors.WithStack(err) + } + + // If a user has a badly formatted secret it will prevent the cluster from working. + //log.Println("TESTING.... secretToKeyPair for certificate.Purpose", certificate.Purpose) + //fmt.Println("TESTING.... secretToKeyPair for certificate.Purpose", certificate.Purpose) + + //for k, _ := range s.Data { + // log.Println("TESTING.... s.Data", k) + //} + //for k, _ := range s.StringData { + // log.Println("TESTING.... s.StringData", k) + //} + //kp, err := secretToKeyPair(s, certificate.Purpose) + + kp, err := secretToKeyPair(s) + if err != nil { + return err + } + + certificate.KeyPair = kp + //certificate.Generated = true + } return nil } @@ -220,6 +285,7 @@ func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clus func (c Certificates) EnsureAllExist() error { for _, certificate := range c { if certificate.KeyPair == nil { + log.Println("TESTING....", certificate) return ErrMissingCertificate } if len(certificate.KeyPair.Cert) == 0 { @@ -238,6 +304,9 @@ func (c Certificates) EnsureAllExist() error { func (c Certificates) Generate() error { for _, certificate := range c { if certificate.KeyPair == nil { + //TODO: Read existing certificate and create required secrets + log.Println("TESTING.... Certificate not present generate new for KeyPair", certificate.KeyFile) + fmt.Println("TESTING.... Certificate not present generate new for KeyPair", certificate.KeyFile) err := certificate.Generate() if err != nil { return err @@ -268,10 +337,10 @@ func (c Certificates) LookupOrGenerate(ctx context.Context, ctrlclient client.Cl return err } - // Generate the certificates that don't exist - if err := c.Generate(); err != nil { - return err - } + //// Generate the certificates that don't exist + //if err := c.Generate(); err != nil { + // return err + //} // Save any certificates that have been generated return c.SaveGenerated(ctx, ctrlclient, clusterName, owner) @@ -415,6 +484,51 @@ func secretToKeyPair(s *corev1.Secret) (*certs.KeyPair, error) { }, nil } +//func secretToKeyPair(s *corev1.Secret, purpose Purpose) (*certs.KeyPair, error) { +// +// var c, key []byte +// var exists bool +// var dataName, keyName string +// if purpose == ServiceAccount { +// dataName = string(purpose) + ".pub" +// keyName = string(purpose) + ".key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// +// } else if purpose == EtcdCA { +// dataName = "etcd-ca.crt" +// keyName = "etcd-ca.key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// +// } else if purpose == FrontProxyCA { +// dataName = "front-proxy-ca.crt" +// keyName = "front-proxy-ca.key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// +// } else { +// dataName = string(purpose) + ".crt" +// keyName = string(purpose) + ".key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// } +// +// c, exists = s.Data[dataName] +// if !exists { +// return nil, errors.Errorf("missing data for key %s", dataName) +// } +// //fmt.Println("TESTING.... c", c) +// // In some cases (external etcd) it's ok if the etcd.key does not exist. +// // TODO: some other function should ensure that the certificates we need exist. +// key, exists = s.Data[keyName] +// if !exists { +// key = []byte("") +// } +// //fmt.Println("TESTING.... key", key) +// +// return &certs.KeyPair{ +// Cert: c, +// Key: key, +// }, nil +//} + func generateCACert() (*certs.KeyPair, error) { x509Cert, privKey, err := newCertificateAuthority() if err != nil { diff --git a/util/secret/consts.go b/util/secret/consts.go index ccfe02ba2ab3..7ecd498cdfe2 100644 --- a/util/secret/consts.go +++ b/util/secret/consts.go @@ -22,6 +22,7 @@ type Purpose string const ( // KubeconfigDataName is the key used to store a Kubeconfig in the secret's data field. KubeconfigDataName = "value" + //KubeconfigDataName2 = "admin.conf" // TLSKeyDataName is the key used to store a TLS private key in the secret's data field. TLSKeyDataName = "tls.key" diff --git a/util/secret/secret.go b/util/secret/secret.go index 5af9fd9f5057..4face40a8529 100644 --- a/util/secret/secret.go +++ b/util/secret/secret.go @@ -38,7 +38,9 @@ func GetFromNamespacedName(ctx context.Context, c client.Reader, clusterName cli secret := &corev1.Secret{} secretKey := client.ObjectKey{ Namespace: clusterName.Namespace, - Name: Name(clusterName.Name, purpose), + //TODO: PCP-22 + //Name: string(purpose), + Name: Name(clusterName.Name, purpose), } if err := c.Get(ctx, secretKey, secret); err != nil { From 8af45cd68b86d8177fd0dd6fd3263c07158a52fd Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Wed, 9 Nov 2022 11:00:22 +0530 Subject: [PATCH 02/13] Minor changes --- controlplane/kubeadm/internal/controllers/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 7bc232284b77..3bae9ba39e8f 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -366,7 +366,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * conditions.MarkFalse(controlPlane.KCP, controlplanev1.AvailableCondition, controlplanev1.WaitingForKubeadmInitReason, clusterv1.ConditionSeverityInfo, "") return r.initializeControlPlane(ctx, cluster, kcp, controlPlane) // We are scaling up - case numMachines < desiredReplicas && numMachines >= 0: + case numMachines < desiredReplicas && numMachines > 0: // Create a new Machine w/ join log.Info("Scaling up control plane", "Desired", desiredReplicas, "Existing", numMachines) return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) From b3179dc9ec405a66f54e59664e79659f2fa53815 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Thu, 1 Dec 2022 18:21:06 +0530 Subject: [PATCH 03/13] PCP-22: CAPI changes --- .../controllers/kubeadmconfig_controller.go | 9 -- .../internal/controllers/controller.go | 5 +- .../kubeadm/internal/controllers/helpers.go | 1 - .../kubeadm/internal/controllers/status.go | 13 +- .../cluster/cluster_controller_phases.go | 42 ++----- util/kubeconfig/kubeconfig.go | 69 ----------- util/secret/certificates.go | 113 ------------------ util/secret/consts.go | 1 - util/secret/secret.go | 4 +- 9 files changed, 19 insertions(+), 238 deletions(-) diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 23a819e19a4c..31811c2c8234 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -19,7 +19,6 @@ package controllers import ( "context" "fmt" - "log" "strconv" "time" @@ -349,8 +348,6 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing // using the DataSecretGeneratedFailedReason - scope.Info("TESTING.... In handleClusterNotInitialized") - if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason { conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "") } @@ -433,7 +430,6 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex return ctrl.Result{}, err } - scope.Info("TESTING.... LookupOrGenerate new certificates") certificates := secret.NewCertificatesForInitialControlPlane(scope.Config.Spec.ClusterConfiguration) err = certificates.LookupOrGenerate( ctx, @@ -506,9 +502,6 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) { - scope.Info("TESTING.... joinWorker") - log.Println("TESTING.... joinWorker") - certificates := secret.NewCertificatesForWorker(scope.Config.Spec.JoinConfiguration.CACertPath) err := certificates.Lookup( ctx, @@ -614,8 +607,6 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S scope.Config.Spec.JoinConfiguration.ControlPlane = &bootstrapv1.JoinControlPlane{} } - scope.Info("TESTING.... NewControlPlaneJoinCerts") - log.Println("TESTING.... joinControlplane") certificates := secret.NewControlPlaneJoinCerts(scope.Config.Spec.ClusterConfiguration) err := certificates.Lookup( ctx, diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 3bae9ba39e8f..1683a188e1ff 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -264,12 +264,9 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * //TODO: PCP-22 lookup or generate ca, sa, etcd certificates and key certificates := secret.NewCertificatesForInitialControlPlane(config.ClusterConfiguration) - //for _, certificate := range certificates { - // log.Info("TESTING.... lookup or generate ca, sa, etcd certificates and key: ", certificate) - //} controllerRef := metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("KubeadmControlPlane")) if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { - log.Error(err, "TESTING.... unable to lookup or create cluster certificates") + log.Error(err, "unable to lookup or create cluster certificates") conditions.MarkFalse(kcp, controlplanev1.CertificatesAvailableCondition, controlplanev1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) return ctrl.Result{}, err } diff --git a/controlplane/kubeadm/internal/controllers/helpers.go b/controlplane/kubeadm/internal/controllers/helpers.go index d9c6ee767baa..31034141ca9f 100644 --- a/controlplane/kubeadm/internal/controllers/helpers.go +++ b/controlplane/kubeadm/internal/controllers/helpers.go @@ -75,7 +75,6 @@ func (r *KubeadmControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, // check if the kubeconfig secret was created by v1alpha2 controllers, and thus it has the Cluster as the owner instead of KCP; // if yes, adopt it. - //TODO: PCP-22 Need to inject original Kubeconfig instead of generating new if util.IsOwnedByObject(configSecret, cluster) && !util.IsControlledBy(configSecret, kcp) { if err := r.adoptKubeconfigSecret(ctx, cluster, configSecret, controllerOwnerRef); err != nil { return ctrl.Result{}, err diff --git a/controlplane/kubeadm/internal/controllers/status.go b/controlplane/kubeadm/internal/controllers/status.go index ee27bab1f3f5..c95d24194866 100644 --- a/controlplane/kubeadm/internal/controllers/status.go +++ b/controlplane/kubeadm/internal/controllers/status.go @@ -105,14 +105,11 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c } kcp.Status.ReadyReplicas = status.ReadyNodes kcp.Status.UnavailableReplicas = replicas - status.ReadyNodes - - //TODO: PCP-22 Initialized should be true to join new node to cluster or else it will try to init - // This only gets initialized once and does not change if the kubeadm config map goes away. - //if status.HasKubeadmConfig { - log.Info("TESTING.... set kcp.Status.Initialized to true") - kcp.Status.Initialized = true - conditions.MarkTrue(kcp, controlplanev1.AvailableCondition) - //} + + if status.HasKubeadmConfig { + kcp.Status.Initialized = true + conditions.MarkTrue(kcp, controlplanev1.AvailableCondition) + } if kcp.Status.ReadyReplicas > 0 { kcp.Status.Ready = true diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index 88e43b6a9f65..b58d10a80af0 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -246,7 +246,8 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster return ctrl.Result{}, err } - //TODO: PCP-22 + // TODO: PCP-22 set controlPlaneInitializedCondition to true for takeOver cluster + // as CP are already initialized in existing cluster conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) if initialized { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) @@ -269,43 +270,24 @@ func (r *Reconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1 // responsible for the management of the Kubeconfig. We continue to manage it here only for backward // compatibility when a Control Plane provider is not in use. - log.Info("TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef", "cluster.Spec.ControlPlaneRef", cluster.Spec.ControlPlaneRef) - //TODO: PCP-22 comment this to let secret generation for now, ControlPlaneRef is present already if cluster.Spec.ControlPlaneRef != nil { return ctrl.Result{}, nil } _, err := secret.Get(ctx, r.Client, util.ObjectKey(cluster), secret.Kubeconfig) - if err != nil { - log.Info("TESTING.... error getting kubeconfig", "err", err) - } - // TODO: PCP-22 read kubeconfig secrets from kube-system namespace - log.Error(nil, "TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef") - if err := kubeconfig.ReadSecret(ctx, r.Client, cluster); err != nil { - if err == kubeconfig.ErrDependentCertificateNotFound { - log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA) - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - if err == kubeconfig.ErrAlreadyExists { - log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA) - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + switch { + case apierrors.IsNotFound(err): + if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil { + if err == kubeconfig.ErrDependentCertificateNotFound { + log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + return ctrl.Result{}, err } - return ctrl.Result{}, err + case err != nil: + return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace) } - //switch { - //case apierrors.IsNotFound(err): - // if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil { - // if err == kubeconfig.ErrDependentCertificateNotFound { - // log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA) - // return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - // } - // return ctrl.Result{}, err - // } - //case err != nil: - // return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace) - //} - return ctrl.Result{}, nil } diff --git a/util/kubeconfig/kubeconfig.go b/util/kubeconfig/kubeconfig.go index 6e5332c0171e..2c8872bbb723 100644 --- a/util/kubeconfig/kubeconfig.go +++ b/util/kubeconfig/kubeconfig.go @@ -22,7 +22,6 @@ import ( "crypto" "crypto/x509" "fmt" - "log" "time" "github.com/pkg/errors" @@ -42,7 +41,6 @@ import ( var ( // ErrDependentCertificateNotFound signals that a CA secret could not be found. ErrDependentCertificateNotFound = errors.New("could not find secret ca") - ErrAlreadyExists = errors.New("secrets \"t-cluster-kubeconfig\" already exists") ) // FromSecret fetches the Kubeconfig for a Cluster. @@ -109,72 +107,6 @@ func CreateSecret(ctx context.Context, c client.Client, cluster *clusterv1.Clust }) } -// ReadSecret reads the Kubeconfig secret from kube-system -func ReadSecret(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) error { - log.Println("TESTING.... IN ReadSecret") - - name := util.ObjectKey(cluster) - return ReadSecretWithOwner(ctx, c, name, cluster.Spec.ControlPlaneEndpoint.String(), metav1.OwnerReference{ - APIVersion: clusterv1.GroupVersion.String(), - Kind: "Cluster", - Name: cluster.Name, - UID: cluster.UID, - }) -} - -// ReadSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. -func ReadSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { - //server := fmt.Sprintf("https://%s", endpoint) - //out, err := ReadExistingSecret(ctx, c, clusterName, server) - //if err != nil { - // return err - //} - log.Println("TESTING.... IN ReadSecretWithOwner") - - //clusterName := util.ObjectKey(clusterName) - configSecret, err := secret.GetFromNamespacedName(ctx, c, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: clusterName.Name}, secret.Kubeconfig) - if err != nil { - log.Println("TESTING....", "error in getting kubeconfig: ", err) - return err - } - - data, err := toKubeconfigBytes(configSecret) - if err != nil { - log.Println("TESTING....", "error in parsing kubeconfig: ", err) - return err - } - //if err := ReadExistingSecret(ctx, r.Client, configSecret); err != nil { - // return ctrl.Result{}, errors.Wrap(err, "failed to regenerate kubeconfig") - //} - - return c.Create(ctx, GenerateSecretWithOwner(clusterName, data, owner)) -} - -// ReadExistingSecret creates and stores a new Kubeconfig in the given secret. -func ReadExistingSecret(ctx context.Context, c client.Client, configSecret *corev1.Secret) error { - clusterName, _, err := secret.ParseSecretName(configSecret.Name) - if err != nil { - return errors.Wrap(err, "failed to parse secret name") - } - data, err := toKubeconfigBytes(configSecret) - if err != nil { - return err - } - - config, err := clientcmd.Load(data) - if err != nil { - return errors.Wrap(err, "failed to convert kubeconfig Secret into a clientcmdapi.Config") - } - endpoint := config.Clusters[clusterName].Server - key := client.ObjectKey{Name: clusterName, Namespace: configSecret.Namespace} - out, err := generateKubeconfig(ctx, c, key, endpoint) - if err != nil { - return err - } - configSecret.Data[secret.KubeconfigDataName] = out - return c.Update(ctx, configSecret) -} - // CreateSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. func CreateSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { server := fmt.Sprintf("https://%s", endpoint) @@ -305,7 +237,6 @@ func generateKubeconfig(ctx context.Context, c client.Client, clusterName client } func toKubeconfigBytes(out *corev1.Secret) ([]byte, error) { - //data, ok := out.Data[secret.KubeconfigDataName2] data, ok := out.Data[secret.KubeconfigDataName] if !ok { return nil, errors.Errorf("missing key %q in secret data", secret.KubeconfigDataName) diff --git a/util/secret/certificates.go b/util/secret/certificates.go index 06169e293d7b..f05926c61024 100644 --- a/util/secret/certificates.go +++ b/util/secret/certificates.go @@ -24,8 +24,6 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" - "fmt" - "log" "math/big" "path/filepath" "strings" @@ -209,7 +207,6 @@ func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clus return errors.WithStack(err) } // If a user has a badly formatted secret it will prevent the cluster from working. - //kp, err := secretToKeyPair(s, certificate.Purpose) kp, err := secretToKeyPair(s) if err != nil { return err @@ -217,67 +214,6 @@ func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clus certificate.KeyPair = kp } - //err := c.LookupKubeadm(ctx, ctrlclient, clusterName) - //if err != nil { - // log.Println("TESTING.... error:", err) - // return err - //} - return nil -} - -// LookupKubeadm looks up each certificate from secrets and populates the certificate with the secret data. -func (c Certificates) LookupKubeadm(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey) error { - // Look up each certificate as a secret and populate the certificate/key - - fmt.Println("TESTING.... LookupKubeadm") - for _, certificate := range c { - s := &corev1.Secret{} - key := client.ObjectKey{ - Name: Name(clusterName.Name, certificate.Purpose), - Namespace: clusterName.Namespace, - } - - //key := client.ObjectKey{ - // Name: "kubeadm-certs", - // Namespace: "kube-system", - //} - - if err := ctrlclient.Get(ctx, key, s); err != nil { - if apierrors.IsNotFound(err) { - if certificate.External { - //log.Println("TESTING.... ERROR external certificate not found: ", certificate.Purpose) - //fmt.Println("TESTING.... ERROR external certificate not found: ", certificate.Purpose) - return errors.WithMessage(err, "external certificate not found") - } - - //fmt.Println("TESTING..... err:", err) - continue - } - //log.Println("TESTING.... ERROR: ", certificate.Purpose) - //fmt.Println("TESTING.... ERROR: ", certificate.Purpose) - return errors.WithStack(err) - } - - // If a user has a badly formatted secret it will prevent the cluster from working. - //log.Println("TESTING.... secretToKeyPair for certificate.Purpose", certificate.Purpose) - //fmt.Println("TESTING.... secretToKeyPair for certificate.Purpose", certificate.Purpose) - - //for k, _ := range s.Data { - // log.Println("TESTING.... s.Data", k) - //} - //for k, _ := range s.StringData { - // log.Println("TESTING.... s.StringData", k) - //} - //kp, err := secretToKeyPair(s, certificate.Purpose) - - kp, err := secretToKeyPair(s) - if err != nil { - return err - } - - certificate.KeyPair = kp - //certificate.Generated = true - } return nil } @@ -285,7 +221,6 @@ func (c Certificates) LookupKubeadm(ctx context.Context, ctrlclient client.Clien func (c Certificates) EnsureAllExist() error { for _, certificate := range c { if certificate.KeyPair == nil { - log.Println("TESTING....", certificate) return ErrMissingCertificate } if len(certificate.KeyPair.Cert) == 0 { @@ -304,9 +239,6 @@ func (c Certificates) EnsureAllExist() error { func (c Certificates) Generate() error { for _, certificate := range c { if certificate.KeyPair == nil { - //TODO: Read existing certificate and create required secrets - log.Println("TESTING.... Certificate not present generate new for KeyPair", certificate.KeyFile) - fmt.Println("TESTING.... Certificate not present generate new for KeyPair", certificate.KeyFile) err := certificate.Generate() if err != nil { return err @@ -484,51 +416,6 @@ func secretToKeyPair(s *corev1.Secret) (*certs.KeyPair, error) { }, nil } -//func secretToKeyPair(s *corev1.Secret, purpose Purpose) (*certs.KeyPair, error) { -// -// var c, key []byte -// var exists bool -// var dataName, keyName string -// if purpose == ServiceAccount { -// dataName = string(purpose) + ".pub" -// keyName = string(purpose) + ".key" -// log.Println("TESTING.... secret name: ", dataName, keyName) -// -// } else if purpose == EtcdCA { -// dataName = "etcd-ca.crt" -// keyName = "etcd-ca.key" -// log.Println("TESTING.... secret name: ", dataName, keyName) -// -// } else if purpose == FrontProxyCA { -// dataName = "front-proxy-ca.crt" -// keyName = "front-proxy-ca.key" -// log.Println("TESTING.... secret name: ", dataName, keyName) -// -// } else { -// dataName = string(purpose) + ".crt" -// keyName = string(purpose) + ".key" -// log.Println("TESTING.... secret name: ", dataName, keyName) -// } -// -// c, exists = s.Data[dataName] -// if !exists { -// return nil, errors.Errorf("missing data for key %s", dataName) -// } -// //fmt.Println("TESTING.... c", c) -// // In some cases (external etcd) it's ok if the etcd.key does not exist. -// // TODO: some other function should ensure that the certificates we need exist. -// key, exists = s.Data[keyName] -// if !exists { -// key = []byte("") -// } -// //fmt.Println("TESTING.... key", key) -// -// return &certs.KeyPair{ -// Cert: c, -// Key: key, -// }, nil -//} - func generateCACert() (*certs.KeyPair, error) { x509Cert, privKey, err := newCertificateAuthority() if err != nil { diff --git a/util/secret/consts.go b/util/secret/consts.go index 7ecd498cdfe2..ccfe02ba2ab3 100644 --- a/util/secret/consts.go +++ b/util/secret/consts.go @@ -22,7 +22,6 @@ type Purpose string const ( // KubeconfigDataName is the key used to store a Kubeconfig in the secret's data field. KubeconfigDataName = "value" - //KubeconfigDataName2 = "admin.conf" // TLSKeyDataName is the key used to store a TLS private key in the secret's data field. TLSKeyDataName = "tls.key" diff --git a/util/secret/secret.go b/util/secret/secret.go index 4face40a8529..5af9fd9f5057 100644 --- a/util/secret/secret.go +++ b/util/secret/secret.go @@ -38,9 +38,7 @@ func GetFromNamespacedName(ctx context.Context, c client.Reader, clusterName cli secret := &corev1.Secret{} secretKey := client.ObjectKey{ Namespace: clusterName.Namespace, - //TODO: PCP-22 - //Name: string(purpose), - Name: Name(clusterName.Name, purpose), + Name: Name(clusterName.Name, purpose), } if err := c.Get(ctx, secretKey, secret); err != nil { From a4e54d3d62bc43f342618db3833e74e02c4786d3 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Mon, 5 Dec 2022 14:47:03 +0530 Subject: [PATCH 04/13] Code cleanup --- .../internal/controllers/kubeadmconfig_controller.go | 2 -- .../kubeadm/internal/controllers/controller.go | 3 --- controlplane/kubeadm/internal/controllers/status.go | 3 ++- .../controllers/cluster/cluster_controller_phases.go | 2 -- util/secret/certificates.go | 12 +++++++----- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 31811c2c8234..bfc5313f8df2 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -347,7 +347,6 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex // initialize the DataSecretAvailableCondition if missing. // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing // using the DataSecretGeneratedFailedReason - if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason { conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "") } @@ -501,7 +500,6 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex } func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) { - certificates := secret.NewCertificatesForWorker(scope.Config.Spec.JoinConfiguration.CACertPath) err := certificates.Lookup( ctx, diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 1683a188e1ff..8d5fd09d9439 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -261,8 +261,6 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * if config.ClusterConfiguration == nil { config.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{} } - - //TODO: PCP-22 lookup or generate ca, sa, etcd certificates and key certificates := secret.NewCertificatesForInitialControlPlane(config.ClusterConfiguration) controllerRef := metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("KubeadmControlPlane")) if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { @@ -355,7 +353,6 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * desiredReplicas := int(*kcp.Spec.Replicas) switch { - //TODO: PCP-22 skip creating new control plane // We are creating the first replica case numMachines < desiredReplicas && numMachines == 0: // Create new Machine w/ init diff --git a/controlplane/kubeadm/internal/controllers/status.go b/controlplane/kubeadm/internal/controllers/status.go index c95d24194866..1617827ff3e5 100644 --- a/controlplane/kubeadm/internal/controllers/status.go +++ b/controlplane/kubeadm/internal/controllers/status.go @@ -105,7 +105,8 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c } kcp.Status.ReadyReplicas = status.ReadyNodes kcp.Status.UnavailableReplicas = replicas - status.ReadyNodes - + + // This only gets initialized once and does not change if the kubeadm config map goes away. if status.HasKubeadmConfig { kcp.Status.Initialized = true conditions.MarkTrue(kcp, controlplanev1.AvailableCondition) diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index b58d10a80af0..60b5ef1e4ebf 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -269,13 +269,11 @@ func (r *Reconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1 // Do not generate the Kubeconfig if there is a ControlPlaneRef, since the Control Plane provider is // responsible for the management of the Kubeconfig. We continue to manage it here only for backward // compatibility when a Control Plane provider is not in use. - if cluster.Spec.ControlPlaneRef != nil { return ctrl.Result{}, nil } _, err := secret.Get(ctx, r.Client, util.ObjectKey(cluster), secret.Kubeconfig) - switch { case apierrors.IsNotFound(err): if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil { diff --git a/util/secret/certificates.go b/util/secret/certificates.go index f05926c61024..c017f5ea6f9c 100644 --- a/util/secret/certificates.go +++ b/util/secret/certificates.go @@ -24,6 +24,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" + "fmt" "math/big" "path/filepath" "strings" @@ -213,7 +214,6 @@ func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clus } certificate.KeyPair = kp } - return nil } @@ -239,6 +239,7 @@ func (c Certificates) EnsureAllExist() error { func (c Certificates) Generate() error { for _, certificate := range c { if certificate.KeyPair == nil { + fmt.Println("TESTING.... Generate new certificates") err := certificate.Generate() if err != nil { return err @@ -269,10 +270,11 @@ func (c Certificates) LookupOrGenerate(ctx context.Context, ctrlclient client.Cl return err } - //// Generate the certificates that don't exist - //if err := c.Generate(); err != nil { - // return err - //} + fmt.Println("TESTING.... Generate new certificates if that don't exist") + // Generate the certificates that don't exist + if err := c.Generate(); err != nil { + return err + } // Save any certificates that have been generated return c.SaveGenerated(ctx, ctrlclient, clusterName, owner) From de2a728c2a9b38a686391a40b5904345e5608a0e Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Tue, 20 Dec 2022 17:22:14 +0530 Subject: [PATCH 05/13] Added takeOverCluster in ClusterSpec --- api/v1beta1/cluster_types.go | 4 ++++ config/crd/bases/cluster.x-k8s.io_clusters.yaml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/api/v1beta1/cluster_types.go b/api/v1beta1/cluster_types.go index d7971b85c7b6..4969142d6e2a 100644 --- a/api/v1beta1/cluster_types.go +++ b/api/v1beta1/cluster_types.go @@ -68,6 +68,10 @@ type ClusterSpec struct { // this feature is highly experimental, and parts of it might still be not implemented. // +optional Topology *Topology `json:"topology,omitempty"` + + //TODO: PCP-22, create/set cluster object for imported/takeover cluster + // TakeOverCluster represents the cluster launch is for taking over existing pre-provisioned cluster + TakeOverCluster bool `json:"takeOverCluster,omitempty"` } // Topology encapsulates the information of the managed resources. diff --git a/config/crd/bases/cluster.x-k8s.io_clusters.yaml b/config/crd/bases/cluster.x-k8s.io_clusters.yaml index 1ab8b7211315..434dc0a7ad52 100644 --- a/config/crd/bases/cluster.x-k8s.io_clusters.yaml +++ b/config/crd/bases/cluster.x-k8s.io_clusters.yaml @@ -818,6 +818,11 @@ spec: description: Paused can be used to prevent controllers from processing the Cluster and all its associated objects. type: boolean + takeOverCluster: + description: 'TODO: PCP-22, create/set cluster object for imported/takeover + cluster TakeOverCluster represents the cluster launch is for taking + over existing pre-provisioned cluster' + type: boolean topology: description: 'This encapsulates the topology for the cluster. NOTE: It is required to enable the ClusterTopology feature gate flag to From c6d97e4c8a60f2ea037ea477301c82fedfb3f2e9 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Tue, 20 Dec 2022 18:03:47 +0530 Subject: [PATCH 06/13] Added takeOverCluster in ClusterSpec --- .../internal/controllers/kubeadmconfig_controller.go | 8 ++++---- controlplane/kubeadm/internal/controllers/controller.go | 7 +------ internal/controllers/cluster/cluster_controller_phases.go | 3 +-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index bfc5313f8df2..2d7921ceb588 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -265,10 +265,10 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques //TODO: PCP-22 check (annotation to skip handleClusterNotInitialized and go for join ) //how to make this condition true for new cluster as kubeadm cluster is already initialized // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. - log.Info("TESTING... skip handx``leClusterNotInitialized and push cluster for join") - //if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { - // return r.handleClusterNotInitialized(ctx, scope) - //} + log.Info("TESTING... skip handleClusterNotInitialized and push cluster for join") + if !cluster.Spec.TakeOverCluster && !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { + return r.handleClusterNotInitialized(ctx, scope) + } // Every other case it's a join scenario // Nb. in this case ClusterConfiguration and InitConfiguration should not be defined by users, but in case of misconfigurations, CABPK simply ignore them diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 8d5fd09d9439..0073058f2e02 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -529,12 +529,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileEtcdMembers(ctx context.Context log := ctrl.LoggerFrom(ctx, "cluster", controlPlane.Cluster.Name) // If etcd is not managed by KCP this is a no-op. - if true { - //TODO: PCP-22 - return ctrl.Result{}, nil - } - - if !controlPlane.IsEtcdManaged() { + if controlPlane.Cluster.Spec.TakeOverCluster || !controlPlane.IsEtcdManaged() { return ctrl.Result{}, nil } diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index 60b5ef1e4ebf..dac4d5e77bfb 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -248,8 +248,7 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster // TODO: PCP-22 set controlPlaneInitializedCondition to true for takeOver cluster // as CP are already initialized in existing cluster - conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) - if initialized { + if cluster.Spec.TakeOverCluster || initialized { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) } else { conditions.MarkFalse(cluster, clusterv1.ControlPlaneInitializedCondition, clusterv1.WaitingForControlPlaneProviderInitializedReason, clusterv1.ConditionSeverityInfo, "Waiting for control plane provider to indicate the control plane has been initialized") From c3748c8efb88caa807bb2741262ef599fb041e29 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Wed, 21 Dec 2022 10:09:31 +0530 Subject: [PATCH 07/13] Spectro generated changes --- bootstrap/kubeadm/config/default/manager_image_patch.yaml | 2 +- config/default/manager_image_patch.yaml | 2 +- .../kubeadm/config/default/manager_image_patch.yaml | 2 +- spectro/generated/bootstrap-base.yaml | 2 +- spectro/generated/bootstrap-global.yaml | 2 +- spectro/generated/controlplane-base.yaml | 2 +- spectro/generated/controlplane-global.yaml | 2 +- spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 7 ++++++- 9 files changed, 14 insertions(+), 9 deletions(-) diff --git a/bootstrap/kubeadm/config/default/manager_image_patch.yaml b/bootstrap/kubeadm/config/default/manager_image_patch.yaml index 810a74d02471..1a8b1833dd06 100644 --- a/bootstrap/kubeadm/config/default/manager_image_patch.yaml +++ b/bootstrap/kubeadm/config/default/manager_image_patch.yaml @@ -7,5 +7,5 @@ spec: template: spec: containers: - - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:20220805 + - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:dev name: manager diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 11b1230a111c..d68596266d58 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -7,5 +7,5 @@ spec: template: spec: containers: - - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:20220805 + - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:dev name: manager diff --git a/controlplane/kubeadm/config/default/manager_image_patch.yaml b/controlplane/kubeadm/config/default/manager_image_patch.yaml index 5c278272e3dd..7fb394e94feb 100644 --- a/controlplane/kubeadm/config/default/manager_image_patch.yaml +++ b/controlplane/kubeadm/config/default/manager_image_patch.yaml @@ -7,5 +7,5 @@ spec: template: spec: containers: - - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:20220805 + - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:dev name: manager diff --git a/spectro/generated/bootstrap-base.yaml b/spectro/generated/bootstrap-base.yaml index 558d42de5217..efe4f4239c69 100644 --- a/spectro/generated/bootstrap-base.yaml +++ b/spectro/generated/bootstrap-base.yaml @@ -26,7 +26,7 @@ spec: - --bootstrap-token-ttl=${KUBEADM_BOOTSTRAP_TOKEN_TTL:=15m} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/bootstrap-global.yaml b/spectro/generated/bootstrap-global.yaml index 90de902d0bb1..22afe88f7a10 100644 --- a/spectro/generated/bootstrap-global.yaml +++ b/spectro/generated/bootstrap-global.yaml @@ -6241,7 +6241,7 @@ spec: - --bootstrap-token-ttl=${KUBEADM_BOOTSTRAP_TOKEN_TTL:=15m} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:dev imagePullPolicy: Always livenessProbe: httpGet: diff --git a/spectro/generated/controlplane-base.yaml b/spectro/generated/controlplane-base.yaml index 5cd9f6869892..ae2c22f687e0 100644 --- a/spectro/generated/controlplane-base.yaml +++ b/spectro/generated/controlplane-base.yaml @@ -25,7 +25,7 @@ spec: - --feature-gates=ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/controlplane-global.yaml b/spectro/generated/controlplane-global.yaml index 6adf5b04b649..b0d272291e92 100644 --- a/spectro/generated/controlplane-global.yaml +++ b/spectro/generated/controlplane-global.yaml @@ -6209,7 +6209,7 @@ spec: - --feature-gates=ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:dev imagePullPolicy: Always livenessProbe: httpGet: diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index 3096a97a49fd..0e468767b44b 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -25,7 +25,7 @@ spec: - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},ClusterResourceSet=${EXP_CLUSTER_RESOURCE_SET:=false},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index 28ab7d47c1fb..f70de0bd0f7b 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -2906,6 +2906,11 @@ spec: description: Paused can be used to prevent controllers from processing the Cluster and all its associated objects. type: boolean + takeOverCluster: + description: 'TODO: PCP-22, create/set cluster object for imported/takeover + cluster TakeOverCluster represents the cluster launch is for taking + over existing pre-provisioned cluster' + type: boolean topology: description: 'This encapsulates the topology for the cluster. NOTE: It is required to enable the ClusterTopology feature gate flag to @@ -9646,7 +9651,7 @@ spec: - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},ClusterResourceSet=${EXP_CLUSTER_RESOURCE_SET:=false},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:dev imagePullPolicy: Always livenessProbe: httpGet: From dc530f9dc60b350369163d684a940c000d8dc68e Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe <33931378+AmitSahastra@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:16:26 +0530 Subject: [PATCH 08/13] Update cluster_types.go --- api/v1beta1/cluster_types.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/v1beta1/cluster_types.go b/api/v1beta1/cluster_types.go index 4969142d6e2a..932b017f2d9a 100644 --- a/api/v1beta1/cluster_types.go +++ b/api/v1beta1/cluster_types.go @@ -69,7 +69,6 @@ type ClusterSpec struct { // +optional Topology *Topology `json:"topology,omitempty"` - //TODO: PCP-22, create/set cluster object for imported/takeover cluster // TakeOverCluster represents the cluster launch is for taking over existing pre-provisioned cluster TakeOverCluster bool `json:"takeOverCluster,omitempty"` } From ffdc41fbdf0ecc91906fce2c43e23ef2e78aba89 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe <33931378+AmitSahastra@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:17:26 +0530 Subject: [PATCH 09/13] Update kubeadmconfig_controller.go --- .../kubeadm/internal/controllers/kubeadmconfig_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 2d7921ceb588..a95b8d69893b 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -262,8 +262,8 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } - //TODO: PCP-22 check (annotation to skip handleClusterNotInitialized and go for join ) - //how to make this condition true for new cluster as kubeadm cluster is already initialized + // TODO: PCP-22 check (annotation to skip handleClusterNotInitialized and go for join ) + // how to make this condition true for new cluster as kubeadm cluster is already initialized // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. log.Info("TESTING... skip handleClusterNotInitialized and push cluster for join") if !cluster.Spec.TakeOverCluster && !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { From 0352ddf40576726c3d9bd013d7312956fd299683 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe <33931378+AmitSahastra@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:18:08 +0530 Subject: [PATCH 10/13] Update controller.go --- controlplane/kubeadm/internal/controllers/controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 0073058f2e02..efb332c92d5b 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -276,7 +276,6 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * return ctrl.Result{}, nil } - //TODO: PCP-22 adopt kubeconfig instead of generating new // Generate Cluster Kubeconfig if needed if result, err := r.reconcileKubeconfig(ctx, cluster, kcp); !result.IsZero() || err != nil { if err != nil { From 4949425325025249c7949d598a47cee7f98956d6 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe <33931378+AmitSahastra@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:36:05 +0530 Subject: [PATCH 11/13] Update kubeadm_control_plane_webhook.go --- .../v1beta1/kubeadm_control_plane_webhook.go | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go index 2cf470fa97f0..a03ed4ed40e5 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go @@ -111,21 +111,21 @@ func (in *KubeadmControlPlane) ValidateCreate() error { const ( spec = "spec" kubeadmConfigSpec = "kubeadmConfigSpec" - clusterConfiguration = "clusterConfiguration" - initConfiguration = "initConfiguration" - joinConfiguration = "joinConfiguration" - nodeRegistration = "nodeRegistration" - patches = "patches" - directory = "directory" - preKubeadmCommands = "preKubeadmCommands" - postKubeadmCommands = "postKubeadmCommands" - files = "files" - users = "users" - apiServer = "apiServer" - controllerManager = "controllerManager" - scheduler = "scheduler" - ntp = "ntp" - ignition = "ignition" + //clusterConfiguration = "clusterConfiguration" + //initConfiguration = "initConfiguration" + //joinConfiguration = "joinConfiguration" + //nodeRegistration = "nodeRegistration" + //patches = "patches" + //directory = "directory" + //preKubeadmCommands = "preKubeadmCommands" + //postKubeadmCommands = "postKubeadmCommands" + //files = "files" + //users = "users" + //apiServer = "apiServer" + //controllerManager = "controllerManager" + //scheduler = "scheduler" + //ntp = "ntp" + //ignition = "ignition" ) // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. From e75f4e5a577e78ef842f30fecf72e7d277828c06 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Wed, 21 Dec 2022 14:05:23 +0530 Subject: [PATCH 12/13] Added takeOverCluster in Annotations --- api/v1beta1/cluster_types.go | 3 -- api/v1beta1/common_types.go | 3 ++ .../controllers/kubeadmconfig_controller.go | 3 +- .../crd/bases/cluster.x-k8s.io_clusters.yaml | 5 --- .../v1beta1/kubeadm_control_plane_webhook.go | 34 +++++++++---------- .../internal/controllers/controller.go | 2 +- .../cluster/cluster_controller_phases.go | 2 +- spectro/generated/core-global.yaml | 5 --- util/annotations/helpers.go | 5 +++ 9 files changed, 29 insertions(+), 33 deletions(-) diff --git a/api/v1beta1/cluster_types.go b/api/v1beta1/cluster_types.go index 932b017f2d9a..d7971b85c7b6 100644 --- a/api/v1beta1/cluster_types.go +++ b/api/v1beta1/cluster_types.go @@ -68,9 +68,6 @@ type ClusterSpec struct { // this feature is highly experimental, and parts of it might still be not implemented. // +optional Topology *Topology `json:"topology,omitempty"` - - // TakeOverCluster represents the cluster launch is for taking over existing pre-provisioned cluster - TakeOverCluster bool `json:"takeOverCluster,omitempty"` } // Topology encapsulates the information of the managed resources. diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index f9d89ef75125..ca94e2a42431 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -117,6 +117,9 @@ const ( // An external controller must fulfill the contract of the InfraCluster resource. // External infrastructure providers should ensure that the annotation, once set, cannot be removed. ManagedByAnnotation = "cluster.x-k8s.io/managed-by" + + // TakeOverCluster is the label used to mark the nodes that run on takeover-cluster instances. + TakeOverCluster = "cluster.x-k8s.io/takeover-cluster" ) const ( diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index a95b8d69893b..3c7f96c0c5ec 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -266,7 +266,8 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques // how to make this condition true for new cluster as kubeadm cluster is already initialized // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. log.Info("TESTING... skip handleClusterNotInitialized and push cluster for join") - if !cluster.Spec.TakeOverCluster && !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { + + if !annotations.IsTakeOverCluster(cluster.GetObjectMeta()) && !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { return r.handleClusterNotInitialized(ctx, scope) } diff --git a/config/crd/bases/cluster.x-k8s.io_clusters.yaml b/config/crd/bases/cluster.x-k8s.io_clusters.yaml index 434dc0a7ad52..1ab8b7211315 100644 --- a/config/crd/bases/cluster.x-k8s.io_clusters.yaml +++ b/config/crd/bases/cluster.x-k8s.io_clusters.yaml @@ -818,11 +818,6 @@ spec: description: Paused can be used to prevent controllers from processing the Cluster and all its associated objects. type: boolean - takeOverCluster: - description: 'TODO: PCP-22, create/set cluster object for imported/takeover - cluster TakeOverCluster represents the cluster launch is for taking - over existing pre-provisioned cluster' - type: boolean topology: description: 'This encapsulates the topology for the cluster. NOTE: It is required to enable the ClusterTopology feature gate flag to diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go index a03ed4ed40e5..4aa04e104333 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go @@ -109,23 +109,23 @@ func (in *KubeadmControlPlane) ValidateCreate() error { } const ( - spec = "spec" - kubeadmConfigSpec = "kubeadmConfigSpec" - //clusterConfiguration = "clusterConfiguration" - //initConfiguration = "initConfiguration" - //joinConfiguration = "joinConfiguration" - //nodeRegistration = "nodeRegistration" - //patches = "patches" - //directory = "directory" - //preKubeadmCommands = "preKubeadmCommands" - //postKubeadmCommands = "postKubeadmCommands" - //files = "files" - //users = "users" - //apiServer = "apiServer" - //controllerManager = "controllerManager" - //scheduler = "scheduler" - //ntp = "ntp" - //ignition = "ignition" + spec = "spec" + kubeadmConfigSpec = "kubeadmConfigSpec" + // clusterConfiguration = "clusterConfiguration" + // initConfiguration = "initConfiguration" + // joinConfiguration = "joinConfiguration" + // nodeRegistration = "nodeRegistration" + // patches = "patches" + // directory = "directory" + // preKubeadmCommands = "preKubeadmCommands" + // postKubeadmCommands = "postKubeadmCommands" + // files = "files" + // users = "users" + // apiServer = "apiServer" + // controllerManager = "controllerManager" + // scheduler = "scheduler" + // ntp = "ntp" + // ignition = "ignition" ) // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index efb332c92d5b..759273e5b0ff 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -528,7 +528,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileEtcdMembers(ctx context.Context log := ctrl.LoggerFrom(ctx, "cluster", controlPlane.Cluster.Name) // If etcd is not managed by KCP this is a no-op. - if controlPlane.Cluster.Spec.TakeOverCluster || !controlPlane.IsEtcdManaged() { + if annotations.IsTakeOverCluster(controlPlane.Cluster.GetObjectMeta()) || !controlPlane.IsEtcdManaged() { return ctrl.Result{}, nil } diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index dac4d5e77bfb..d18d9574fad4 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -248,7 +248,7 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster // TODO: PCP-22 set controlPlaneInitializedCondition to true for takeOver cluster // as CP are already initialized in existing cluster - if cluster.Spec.TakeOverCluster || initialized { + if annotations.IsTakeOverCluster(cluster.GetObjectMeta()) || initialized { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) } else { conditions.MarkFalse(cluster, clusterv1.ControlPlaneInitializedCondition, clusterv1.WaitingForControlPlaneProviderInitializedReason, clusterv1.ConditionSeverityInfo, "Waiting for control plane provider to indicate the control plane has been initialized") diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index f70de0bd0f7b..1d40fe5793aa 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -2906,11 +2906,6 @@ spec: description: Paused can be used to prevent controllers from processing the Cluster and all its associated objects. type: boolean - takeOverCluster: - description: 'TODO: PCP-22, create/set cluster object for imported/takeover - cluster TakeOverCluster represents the cluster launch is for taking - over existing pre-provisioned cluster' - type: boolean topology: description: 'This encapsulates the topology for the cluster. NOTE: It is required to enable the ClusterTopology feature gate flag to diff --git a/util/annotations/helpers.go b/util/annotations/helpers.go index de07f7a55b92..cc6b77888012 100644 --- a/util/annotations/helpers.go +++ b/util/annotations/helpers.go @@ -38,6 +38,11 @@ func IsExternallyManaged(o metav1.Object) bool { return hasAnnotation(o, clusterv1.ManagedByAnnotation) } +// IsTakeOverCluster returns true if the object has the `managed-by` annotation. +func IsTakeOverCluster(o metav1.Object) bool { + return hasAnnotation(o, clusterv1.TakeOverCluster) +} + // HasPaused returns true if the object has the `paused` annotation. func HasPaused(o metav1.Object) bool { return hasAnnotation(o, clusterv1.PausedAnnotation) From ecb13524723e5656ae1041e571a330eb70bec2f8 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe <33931378+AmitSahastra@users.noreply.github.com> Date: Wed, 21 Dec 2022 16:19:33 +0530 Subject: [PATCH 13/13] Update kubeadm_control_plane_webhook.go --- .../v1beta1/kubeadm_control_plane_webhook.go | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go index 4aa04e104333..67db0d8ebffb 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go @@ -134,26 +134,26 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error { // For example, {"spec", "*"} will allow any path under "spec" to change. allowedPaths := [][]string{ {"metadata", "*"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageRepository"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageTag"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "extraArgs", "*"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageRepository"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageTag"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, "imageRepository"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, apiServer, "*"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, controllerManager, "*"}, - //{spec, kubeadmConfigSpec, clusterConfiguration, scheduler, "*"}, - //{spec, kubeadmConfigSpec, initConfiguration, nodeRegistration, "*"}, - //{spec, kubeadmConfigSpec, initConfiguration, patches, directory}, - //{spec, kubeadmConfigSpec, joinConfiguration, nodeRegistration, "*"}, - //{spec, kubeadmConfigSpec, joinConfiguration, patches, directory}, - //{spec, kubeadmConfigSpec, preKubeadmCommands}, - //{spec, kubeadmConfigSpec, postKubeadmCommands}, - //{spec, kubeadmConfigSpec, files}, - //{spec, kubeadmConfigSpec, "verbosity"}, - //{spec, kubeadmConfigSpec, users}, - //{spec, kubeadmConfigSpec, ntp, "*"}, - //{spec, kubeadmConfigSpec, ignition, "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageRepository"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageTag"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "extraArgs", "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageRepository"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageTag"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "imageRepository"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, apiServer, "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, controllerManager, "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, scheduler, "*"}, + // {spec, kubeadmConfigSpec, initConfiguration, nodeRegistration, "*"}, + // {spec, kubeadmConfigSpec, initConfiguration, patches, directory}, + // {spec, kubeadmConfigSpec, joinConfiguration, nodeRegistration, "*"}, + // {spec, kubeadmConfigSpec, joinConfiguration, patches, directory}, + // {spec, kubeadmConfigSpec, preKubeadmCommands}, + // {spec, kubeadmConfigSpec, postKubeadmCommands}, + // {spec, kubeadmConfigSpec, files}, + // {spec, kubeadmConfigSpec, "verbosity"}, + // {spec, kubeadmConfigSpec, users}, + // {spec, kubeadmConfigSpec, ntp, "*"}, + // {spec, kubeadmConfigSpec, ignition, "*"}, // allow all fields to be modified {spec, kubeadmConfigSpec, "*"}, {spec, "machineTemplate", "metadata", "*"},