diff --git a/pkg/hash/hash.go b/pkg/hash/hash.go index 5762cd3..415aad5 100644 --- a/pkg/hash/hash.go +++ b/pkg/hash/hash.go @@ -18,22 +18,7 @@ package hash import ( - "fmt" - "hash/fnv" - corev1 "k8s.io/api/core/v1" - hashutil "k8s.io/kubernetes/pkg/util/hash" -) - -const ( - // TemplateHashKey is a key for storing PodTemplateSpec's hash value in labels. - // It will be used to check whether pod's PodTemplateSpec has changed, if yes, - // we need recreate or do in-place update for the pod according to the value of UniqHash. - TemplateHashKey = "tapp_template_hash_key" - // UniqHashKey is a key for storing hash value of PodTemplateSpec(without container images) in labels. - // It will will be used to check whether pod's PodTemplateSpec hash changed and only container images - // changed, if yes, we will do in place update for the pod. - UniqHashKey = "tapp_uniq_hash_key" ) // TappHashInterface is used for generate and verify hash for tapp. @@ -51,85 +36,3 @@ type TappHashInterface interface { // HashLabels returns labels key that stores TemplateHash and UniqHash HashLabels() []string } - -func NewTappHash() TappHashInterface { - return &defaultTappHash{} -} - -type defaultTappHash struct{} - -func (th *defaultTappHash) SetTemplateHash(template *corev1.PodTemplateSpec) bool { - expected := generateTemplateHash(template) - hash := th.GetTemplateHash(template.Labels) - if hash != expected { - if template.Labels == nil { - template.Labels = make(map[string]string) - } - template.Labels[TemplateHashKey] = expected - return true - } else { - return false - } -} - -func (th *defaultTappHash) GetTemplateHash(labels map[string]string) string { - return labels[TemplateHashKey] -} - -func (th *defaultTappHash) SetUniqHash(template *corev1.PodTemplateSpec) bool { - expected := generateUniqHash(*template) - hash := th.GetUniqHash(template.Labels) - if hash != expected { - if template.Labels == nil { - template.Labels = make(map[string]string) - } - template.Labels[UniqHashKey] = expected - return true - } else { - return false - } -} - -func (th *defaultTappHash) GetUniqHash(labels map[string]string) string { - return labels[UniqHashKey] -} - -func (th *defaultTappHash) HashLabels() []string { - return []string{TemplateHashKey, UniqHashKey} -} - -func generateHash(template interface{}) uint64 { - hasher := fnv.New64() - hashutil.DeepHashObject(hasher, template) - return hasher.Sum64() -} - -func generateTemplateHash(template *corev1.PodTemplateSpec) string { - meta := template.ObjectMeta.DeepCopy() - delete(meta.Labels, TemplateHashKey) - delete(meta.Labels, UniqHashKey) - return fmt.Sprintf("%d", generateHash(corev1.PodTemplateSpec{ - ObjectMeta: *meta, - Spec: template.Spec, - })) -} - -func generateUniqHash(template corev1.PodTemplateSpec) string { - if template.Spec.InitContainers != nil { - var newContainers []corev1.Container - for _, container := range template.Spec.InitContainers { - container.Image = "" - newContainers = append(newContainers, container) - } - template.Spec.InitContainers = newContainers - } - - var newContainers []corev1.Container - for _, container := range template.Spec.Containers { - container.Image = "" - newContainers = append(newContainers, container) - } - template.Spec.Containers = newContainers - - return fmt.Sprintf("%d", generateHash(template.Spec)) -} diff --git a/pkg/hash/v1/hash.go b/pkg/hash/v1/hash.go new file mode 100644 index 0000000..54c9649 --- /dev/null +++ b/pkg/hash/v1/hash.go @@ -0,0 +1,121 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package v1 + +import ( + "fmt" + "hash/fnv" + + "tkestack.io/tapp/pkg/hash" + + corev1 "k8s.io/api/core/v1" + hashutil "k8s.io/kubernetes/pkg/util/hash" +) + +const ( + // TemplateHashKey is a key for storing PodTemplateSpec's hash value in labels. + // It will be used to check whether pod's PodTemplateSpec has changed, if yes, + // we need recreate or do in-place update for the pod according to the value of UniqHash. + TemplateHashKey = "tapp_template_hash_key" + // UniqHashKey is a key for storing hash value of PodTemplateSpec(without container images) in labels. + // It will will be used to check whether pod's PodTemplateSpec hash changed and only container images + // changed, if yes, we will do in place update for the pod. + UniqHashKey = "tapp_uniq_hash_key" +) + +func NewTappHash() hash.TappHashInterface { + return &defaultTappHash{} +} + +type defaultTappHash struct{} + +func (th *defaultTappHash) SetTemplateHash(template *corev1.PodTemplateSpec) bool { + expected := generateTemplateHash(template) + hash := th.GetTemplateHash(template.Labels) + if hash != expected { + if template.Labels == nil { + template.Labels = make(map[string]string) + } + template.Labels[TemplateHashKey] = expected + return true + } else { + return false + } +} + +func (th *defaultTappHash) GetTemplateHash(labels map[string]string) string { + return labels[TemplateHashKey] +} + +func (th *defaultTappHash) SetUniqHash(template *corev1.PodTemplateSpec) bool { + expected := generateUniqHash(*template) + hash := th.GetUniqHash(template.Labels) + if hash != expected { + if template.Labels == nil { + template.Labels = make(map[string]string) + } + template.Labels[UniqHashKey] = expected + return true + } else { + return false + } +} + +func (th *defaultTappHash) GetUniqHash(labels map[string]string) string { + return labels[UniqHashKey] +} + +func (th *defaultTappHash) HashLabels() []string { + return []string{TemplateHashKey, UniqHashKey} +} + +func generateHash(template interface{}) uint64 { + hasher := fnv.New64() + hashutil.DeepHashObject(hasher, template) + return hasher.Sum64() +} + +func generateTemplateHash(template *corev1.PodTemplateSpec) string { + meta := template.ObjectMeta.DeepCopy() + delete(meta.Labels, TemplateHashKey) + delete(meta.Labels, UniqHashKey) + return fmt.Sprintf("%d", generateHash(corev1.PodTemplateSpec{ + ObjectMeta: *meta, + Spec: template.Spec, + })) +} + +func generateUniqHash(template corev1.PodTemplateSpec) string { + if template.Spec.InitContainers != nil { + var newContainers []corev1.Container + for _, container := range template.Spec.InitContainers { + container.Image = "" + newContainers = append(newContainers, container) + } + template.Spec.InitContainers = newContainers + } + + var newContainers []corev1.Container + for _, container := range template.Spec.Containers { + container.Image = "" + newContainers = append(newContainers, container) + } + template.Spec.Containers = newContainers + + return fmt.Sprintf("%d", generateHash(template.Spec)) +} diff --git a/pkg/hash/hash_test.go b/pkg/hash/v1/hash_test.go similarity index 76% rename from pkg/hash/hash_test.go rename to pkg/hash/v1/hash_test.go index afc3c17..8be0037 100644 --- a/pkg/hash/hash_test.go +++ b/pkg/hash/v1/hash_test.go @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package hash +package v1 import ( "testing" @@ -34,6 +34,11 @@ func TestSetTemplateHash(t *testing.T) { if expectedTemplateHash != realHash { t.Errorf("Failed to set template h") } + // expectedHashValue is current hash value according to current algorithm and data structures. + expectedHashValue := "16554283907952254416" + if realHash != expectedHashValue { + t.Errorf("Hash values is not expected, got: %s, expected: %s", realHash, expectedHashValue) + } } func TestSetUniqHash(t *testing.T) { @@ -46,6 +51,11 @@ func TestSetUniqHash(t *testing.T) { if expectedUniqHash != realHash { t.Errorf("Failed to set uniq h") } + // expectedHashValue is current hash value according to current algorithm and data structures. + expectedHashValue := "9455405163903490883" + if realHash != expectedHashValue { + t.Errorf("Hash values is not expected, got: %s, expected: %s", realHash, expectedHashValue) + } } func createPodTemplate() corev1.PodTemplateSpec { diff --git a/pkg/hash/v2/hash.go b/pkg/hash/v2/hash.go new file mode 100644 index 0000000..0fb8ed8 --- /dev/null +++ b/pkg/hash/v2/hash.go @@ -0,0 +1,131 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package v2 + +import ( + "encoding/json" + "fmt" + "hash/fnv" + + "tkestack.io/tapp/pkg/hash" + + corev1 "k8s.io/api/core/v1" + hashutil "k8s.io/kubernetes/pkg/util/hash" +) + +const ( + // TemplateHashKeyV2 is a key for storing PodTemplateSpec's hash value in labels. + // It will be used to check whether pod's PodTemplateSpec has changed, if yes, + // we need recreate or do in-place update for the pod according to the value of UniqHash. + TemplateHashKeyV2 = "tapp_template_hash_key_v2" + // UniqHashKeyV2 is a key for storing hash value of PodTemplateSpec(without container images) in labels. + // It will will be used to check whether pod's PodTemplateSpec hash changed and only container images + // changed, if yes, we will do in place update for the pod. + UniqHashKeyV2 = "tapp_uniq_hash_key_v2" +) + +func NewTappHash() hash.TappHashInterface { + return &defaultTappHashV2{} +} + +type defaultTappHashV2 struct{} + +func (th *defaultTappHashV2) SetTemplateHash(template *corev1.PodTemplateSpec) bool { + expected := generateTemplateHash(template) + h := th.GetTemplateHash(template.Labels) + if h != expected { + if template.Labels == nil { + template.Labels = make(map[string]string) + } + template.Labels[TemplateHashKeyV2] = expected + return true + } else { + return false + } +} + +func (th *defaultTappHashV2) GetTemplateHash(labels map[string]string) string { + return labels[TemplateHashKeyV2] +} + +func (th *defaultTappHashV2) SetUniqHash(template *corev1.PodTemplateSpec) bool { + expected := generateUniqHash(*template) + h := th.GetUniqHash(template.Labels) + if h != expected { + if template.Labels == nil { + template.Labels = make(map[string]string) + } + template.Labels[UniqHashKeyV2] = expected + return true + } else { + return false + } +} + +func (th *defaultTappHashV2) GetUniqHash(labels map[string]string) string { + return labels[UniqHashKeyV2] +} + +func (th *defaultTappHashV2) HashLabels() []string { + return []string{TemplateHashKeyV2, UniqHashKeyV2} +} + +func generateHash(template interface{}) uint64 { + // Omit nil or empty field when calculating hash value + // Please see https://github.com/kubernetes/kubernetes/issues/53644 + content, _ := json.Marshal(template) + hasher := fnv.New64() + hashutil.DeepHashObject(hasher, content) + return hasher.Sum64() +} + +func generateTemplateHash(template *corev1.PodTemplateSpec) string { + meta := template.ObjectMeta.DeepCopy() + delete(meta.Labels, TemplateHashKeyV2) + delete(meta.Labels, UniqHashKeyV2) + return fmt.Sprintf("%d", generateHash(corev1.PodTemplateSpec{ + ObjectMeta: *meta, + Spec: template.Spec, + })) +} + +func generateUniqHash(template corev1.PodTemplateSpec) string { + if template.Spec.InitContainers != nil { + var newContainers []corev1.Container + for _, container := range template.Spec.InitContainers { + container.Image = "" + newContainers = append(newContainers, container) + } + template.Spec.InitContainers = newContainers + } + + var newContainers []corev1.Container + for _, container := range template.Spec.Containers { + container.Image = "" + newContainers = append(newContainers, container) + } + template.Spec.Containers = newContainers + + meta := template.ObjectMeta.DeepCopy() + delete(meta.Labels, TemplateHashKeyV2) + delete(meta.Labels, UniqHashKeyV2) + return fmt.Sprintf("%d", generateHash(corev1.PodTemplateSpec{ + ObjectMeta: *meta, + Spec: template.Spec, + })) +} diff --git a/pkg/hash/v2/hash_test.go b/pkg/hash/v2/hash_test.go new file mode 100644 index 0000000..3f6c700 --- /dev/null +++ b/pkg/hash/v2/hash_test.go @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package v2 + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSetTemplateHash(t *testing.T) { + h := NewTappHash() + + template := createPodTemplate() + expectedTemplateHash := generateTemplateHash(&template) + h.SetTemplateHash(&template) + realHash := h.GetTemplateHash(template.Labels) + if expectedTemplateHash != realHash { + t.Errorf("Failed to set template h") + } + // expectedHashValue is current hash value according to current algorithm and data structures. + expectedHashValue := "16703310259724598646" + if realHash != expectedHashValue { + t.Errorf("Hash values is not expected, got: %s, expected: %s", realHash, expectedHashValue) + } +} + +func TestSetUniqHash(t *testing.T) { + h := NewTappHash() + + template := createPodTemplate() + expectedUniqHash := generateUniqHash(template) + h.SetUniqHash(&template) + realHash := h.GetUniqHash(template.Labels) + if expectedUniqHash != realHash { + t.Errorf("Failed to set uniq h") + } + // expectedHashValue is current hash value according to current algorithm and data structures. + expectedHashValue := "2691283627662810275" + if realHash != expectedHashValue { + t.Errorf("Hash values is not expected, got: %s, expected: %s", realHash, expectedHashValue) + } +} + +func createPodTemplate() corev1.PodTemplateSpec { + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"test": "hello"}, + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + DNSPolicy: corev1.DNSClusterFirst, + Containers: []corev1.Container{{Name: "abc", Image: "image", ImagePullPolicy: "IfNotPresent"}}, + }, + } +} diff --git a/pkg/tapp/controller.go b/pkg/tapp/controller.go index 933be25..1a95f97 100644 --- a/pkg/tapp/controller.go +++ b/pkg/tapp/controller.go @@ -31,6 +31,8 @@ import ( informers "tkestack.io/tapp/pkg/client/informers/externalversions" listers "tkestack.io/tapp/pkg/client/listers/tappcontroller/v1" "tkestack.io/tapp/pkg/hash" + hashv1 "tkestack.io/tapp/pkg/hash/v1" + hashv2 "tkestack.io/tapp/pkg/hash/v2" "tkestack.io/tapp/pkg/util" corev1 "k8s.io/api/core/v1" @@ -138,7 +140,6 @@ func NewController( tappclient: tappclientset, tappLister: tappInformer.Lister(), tappsSynced: tappInformer.Informer().HasSynced, - tappHash: hash.NewTappHash(), workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "TApps"), recorder: recorder, syncer: InstanceSyncer{ @@ -512,8 +513,9 @@ func (c *Controller) removeUnusedTemplate(tapp *tappv1.TApp) error { // updateTemplateHash will generate and update templates hash if needed. func (c *Controller) updateTemplateHash(tapp *tappv1.TApp) error { updateHash := func(template *corev1.PodTemplateSpec) { - if c.tappHash.SetTemplateHash(template) { - c.tappHash.SetUniqHash(template) + tappHash := c.getTappHasher(template) + if tappHash.SetTemplateHash(template) { + tappHash.SetUniqHash(template) } } @@ -543,7 +545,7 @@ func (c *Controller) setLabelSelector(tapp *tappv1.TApp) error { for k, v := range labels { tappLabels[k] = v } - for _, label := range c.tappHash.HashLabels() { + for _, label := range util.AllHashLabels() { delete(tappLabels, label) } @@ -877,29 +879,41 @@ func (c *Controller) needForceDelete(tapp *tappv1.TApp, pod *corev1.Pod) bool { return isPodDying(pod) && pod.Status.Reason == NodeUnreachablePodReason && tapp.Spec.ForceDeletePod } -func (c *Controller) isTemplateHashChanged(tapp *tappv1.TApp, podId string, pod *corev1.Pod) bool { - hash := c.tappHash.GetTemplateHash(pod.Labels) +func (c *Controller) getTappHasher(template *corev1.PodTemplateSpec) hash.TappHashInterface { + if template.Labels[hashv1.TemplateHashKey] != "" { + return hashv1.NewTappHash() + } else { + return hashv2.NewTappHash() + } +} +func (c *Controller) isTemplateHashChanged(tapp *tappv1.TApp, podId string, pod *corev1.Pod) bool { template, err := getPodTemplate(&tapp.Spec, podId) if err != nil { klog.Errorf("Failed to get pod template for %s from tapp %s", getPodFullName(pod), util.GetTAppFullName(tapp)) return true } - expected := c.tappHash.GetTemplateHash(template.Labels) - return hash != expected + + tappHash := c.getTappHasher(template) + expected := tappHash.GetTemplateHash(template.Labels) + got := tappHash.GetTemplateHash(pod.Labels) + + return got != expected } func (c *Controller) isUniqHashChanged(tapp *tappv1.TApp, podId string, pod *corev1.Pod) bool { - hash := c.tappHash.GetUniqHash(pod.Labels) template, err := getPodTemplate(&tapp.Spec, podId) if err != nil { klog.Errorf("Failed to get pod template for %s from tapp %s", getPodFullName(pod), util.GetTAppFullName(tapp)) return true } - expected := c.tappHash.GetUniqHash(template.Labels) - return hash != expected + tappHash := c.getTappHasher(template) + expected := tappHash.GetUniqHash(template.Labels) + got := tappHash.GetUniqHash(pod.Labels) + + return got != expected } func getInstanceStatus(tapp *tappv1.TApp, pods []*corev1.Pod) map[string]tappv1.InstanceStatus { diff --git a/pkg/tapp/instance_test.go b/pkg/tapp/instance_test.go index 1bec4a6..c69e421 100644 --- a/pkg/tapp/instance_test.go +++ b/pkg/tapp/instance_test.go @@ -20,7 +20,7 @@ package tapp import ( "testing" - "tkestack.io/tapp/pkg/hash" + hashv1 "tkestack.io/tapp/pkg/hash/v1" "tkestack.io/tapp/pkg/testutil" corev1 "k8s.io/api/core/v1" @@ -38,7 +38,7 @@ func TestMergePod(t *testing.T) { newPod := pod.DeepCopy() newPod.Spec.Containers[0].Image = "image.new" newPod.Spec.Containers[0].Name = "name.new" - newPod.Labels[hash.TemplateHashKey] = "tappHashKey.new" + newPod.Labels[hashv1.TemplateHashKey] = "tappHashKey.new" mergePod(pod, newPod) if pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image { t.Errorf("pod image not updated") @@ -46,7 +46,7 @@ func TestMergePod(t *testing.T) { if pod.Spec.Containers[0].Name == newPod.Spec.Containers[0].Name { t.Errorf("pod name updated") } - if pod.Labels[hash.TemplateHashKey] != newPod.Labels[hash.TemplateHashKey] { + if pod.Labels[hashv1.TemplateHashKey] != newPod.Labels[hashv1.TemplateHashKey] { t.Errorf("TAppHashKey not updated") } diff --git a/pkg/tapp/tapp_test.go b/pkg/tapp/tapp_test.go index e8f6359..7eed140 100644 --- a/pkg/tapp/tapp_test.go +++ b/pkg/tapp/tapp_test.go @@ -25,7 +25,6 @@ import ( "time" tappv1 "tkestack.io/tapp/pkg/apis/tappcontroller/v1" - "tkestack.io/tapp/pkg/hash" "tkestack.io/tapp/pkg/testutil" corev1 "k8s.io/api/core/v1" @@ -37,7 +36,6 @@ func newFakeTAppController() (*Controller, *fakeInstanceClient) { return &Controller{ kubeclient: nil, podStoreSynced: func() bool { return true }, - tappHash: hash.NewTappHash(), syncer: InstanceSyncer{InstanceClient: fakeClient}, }, fakeClient } diff --git a/pkg/testutil/utils.go b/pkg/testutil/utils.go index 7891015..b550a22 100644 --- a/pkg/testutil/utils.go +++ b/pkg/testutil/utils.go @@ -22,7 +22,7 @@ import ( "strconv" v1 "tkestack.io/tapp/pkg/apis/tappcontroller/v1" - "tkestack.io/tapp/pkg/hash" + hashv1 "tkestack.io/tapp/pkg/hash/v1" "tkestack.io/tapp/pkg/util" corev1 "k8s.io/api/core/v1" @@ -35,7 +35,7 @@ const ( FakeLabelValue = "fake_tapp_label_value" ) -var tappHash = hash.NewTappHash() +var tappHash = hashv1.NewTappHash() func AddPodTemplate(tapp *v1.TApp, templateName string, template *corev1.PodTemplateSpec) error { template = template.DeepCopy() diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 206c422..8c2bdb7 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -21,7 +21,8 @@ import ( "fmt" v1 "tkestack.io/tapp/pkg/apis/tappcontroller/v1" - "tkestack.io/tapp/pkg/hash" + hashv1 "tkestack.io/tapp/pkg/hash/v1" + hashv2 "tkestack.io/tapp/pkg/hash/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -40,14 +41,21 @@ func GenerateTAppSelector(labels map[string]string) *metav1.LabelSelector { for k, v := range labels { tappLabels[k] = v } - delete(tappLabels, hash.TemplateHashKey) - delete(tappLabels, hash.UniqHashKey) + for _, label := range AllHashLabels() { + delete(tappLabels, label) + } return &metav1.LabelSelector{ MatchLabels: tappLabels, } } +func AllHashLabels() []string { + l1 := hashv1.NewTappHash().HashLabels() + l2 := hashv2.NewTappHash().HashLabels() + return append(l1, l2...) +} + func GetPodFromTemplate(template *corev1.PodTemplateSpec, parentObject runtime.Object, controllerRef *metav1.OwnerReference) (*corev1.Pod, error) { desiredLabels := getPodsLabelSet(template)