CM-1225: Apply cluster TLS profile to trust-manager, operator metrics, and operand HTTPS metrics

bindata/cert-manager-deployment/cainjector/cert-manager-cainjector-deployment.yaml

@@ -21,6 +21,7 @@ spec:
       annotations:
         prometheus.io/path: /metrics
         prometheus.io/port: "9402"
+        prometheus.io/scheme: https
         prometheus.io/scrape: "true"
       labels:
         app: cainjector

bindata/cert-manager-deployment/controller/cert-manager-deployment.yaml

@@ -21,6 +21,7 @@ spec:
       annotations:
         prometheus.io/path: /metrics
         prometheus.io/port: "9402"
+        prometheus.io/scheme: https
         prometheus.io/scrape: "true"
       labels:
         app: cert-manager

bindata/cert-manager-deployment/controller/cert-manager-metrics-dynamic-serving-rb.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: cert-manager
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: cert-manager
+    app.kubernetes.io/name: cert-manager
+    app.kubernetes.io/version: v1.20.3
+  name: cert-manager-metrics-dynamic-serving
+  namespace: cert-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cert-manager-metrics-dynamic-serving
+subjects:
+  - kind: ServiceAccount
+    name: cert-manager
+    namespace: cert-manager
+  - kind: ServiceAccount
+    name: cert-manager-webhook
+    namespace: cert-manager
+  - kind: ServiceAccount
+    name: cert-manager-cainjector
+    namespace: cert-manager

bindata/cert-manager-deployment/controller/cert-manager-metrics-dynamic-serving-role.yaml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: cert-manager
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: cert-manager
+    app.kubernetes.io/name: cert-manager
+    app.kubernetes.io/version: v1.20.3
+  name: cert-manager-metrics-dynamic-serving
+  namespace: cert-manager
+rules:
+  - apiGroups:
+      - ""
+    resourceNames:
+      - cert-manager-metrics-ca
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create

bindata/cert-manager-deployment/webhook/cert-manager-webhook-deployment.yaml

@@ -21,6 +21,7 @@ spec:
       annotations:
         prometheus.io/path: /metrics
         prometheus.io/port: "9402"
+        prometheus.io/scheme: https
         prometheus.io/scrape: "true"
       labels:
         app: webhook

bundle/manifests/cert-manager-operator-controller-manager-metrics-service_v1_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: cert-manager-operator-serving-cert
   creationTimestamp: null
   labels:
     app.kubernetes.io/created-by: cert-manager-operator

bundle/manifests/cert-manager-operator.clusterserviceversion.yaml

@@ -284,7 +284,7 @@ metadata:
     features.operators.openshift.io/disconnected: "true"
     features.operators.openshift.io/fips-compliant: "true"
     features.operators.openshift.io/proxy-aware: "true"
-    features.operators.openshift.io/tls-profiles: "false"
+    features.operators.openshift.io/tls-profiles: "true"
     features.operators.openshift.io/token-auth-aws: "true"
     features.operators.openshift.io/token-auth-azure: "true"
     features.operators.openshift.io/token-auth-gcp: "true"
@@ -848,6 +848,9 @@ spec:
                 volumeMounts:
                 - mountPath: /tmp
                   name: tmp
+                - mountPath: /var/run/secrets/serving-cert
+                  name: serving-cert
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
                 seccompProfile:
@@ -857,6 +860,10 @@ spec:
               volumes:
               - emptyDir: {}
                 name: tmp
+              - name: serving-cert
+                secret:
+                  optional: true
+                  secretName: cert-manager-operator-serving-cert
       permissions:
       - rules:
         - apiGroups:

config/manager/manager.yaml

@@ -123,8 +123,15 @@ spec:
           volumeMounts:
             - name: tmp
               mountPath: /tmp
+            - name: serving-cert
+              mountPath: /var/run/secrets/serving-cert
+              readOnly: true
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
         - name: tmp
           emptyDir: {}
+        - name: serving-cert
+          secret:
+            secretName: cert-manager-operator-serving-cert
+            optional: true

config/manifests/bases/cert-manager-operator.clusterserviceversion.yaml

@@ -14,7 +14,7 @@ metadata:
     features.operators.openshift.io/disconnected: "true"
     features.operators.openshift.io/fips-compliant: "true"
     features.operators.openshift.io/proxy-aware: "true"
-    features.operators.openshift.io/tls-profiles: "false"
+    features.operators.openshift.io/tls-profiles: "true"
     features.operators.openshift.io/token-auth-aws: "true"
     features.operators.openshift.io/token-auth-azure: "true"
     features.operators.openshift.io/token-auth-gcp: "true"

config/rbac/auth_proxy_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: cert-manager-operator-serving-cert
   labels:
     control-plane: controller-manager
     app.kubernetes.io/name: service

go.mod

@@ -16,6 +16,7 @@ require (
 	k8s.io/api v0.35.2
 	k8s.io/apiextensions-apiserver v0.35.2
 	k8s.io/apimachinery v0.35.2
+	k8s.io/apiserver v0.35.2
 	k8s.io/client-go v0.35.2
 	k8s.io/component-base v0.35.2
 	k8s.io/klog/v2 v2.140.0
@@ -123,7 +124,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiserver v0.35.2 // indirect
 	k8s.io/component-helpers v0.35.2 // indirect
 	k8s.io/controller-manager v0.35.2 // indirect
 	k8s.io/kms v0.35.2 // indirect

pkg/cmd/operator/cmd.go

@@ -2,23 +2,89 @@ package operator
 
 import (
 	"context"
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/clock"
 
 	"github.com/openshift/cert-manager-operator/pkg/operator"
+	"github.com/openshift/cert-manager-operator/pkg/tlsprofile"
 	"github.com/openshift/cert-manager-operator/pkg/version"
 	"github.com/openshift/library-go/pkg/controller/controllercmd"
-	"github.com/spf13/cobra"
-	"k8s.io/utils/clock"
+	"github.com/openshift/library-go/pkg/controller/fileobserver"
+	"github.com/openshift/library-go/pkg/operator/events"
+	"github.com/openshift/library-go/pkg/serviceability"
 )
 
 func NewOperator() *cobra.Command {
-	cmd := controllercmd.NewControllerCommandConfig(
+	cc := controllercmd.NewControllerCommandConfig(
 		"cert-manager-operator",
 		version.Get(),
 		operator.RunOperator,
 		clock.RealClock{},
-	).NewCommandWithContext(context.TODO())
+	)
+
+	cmd := cc.NewCommandWithContext(context.TODO())
 	cmd.Use = "start"
 	cmd.Short = "Start the cert-manager Operator"
+
+	// Replace the default Run so we can apply the cluster TLS profile to the
+	// metrics serving config before the HTTPS listener is created.
+	cmd.Run = func(cmd *cobra.Command, args []string) {
+		rand.Seed(time.Now().UTC().UnixNano())
+		logs.InitLogs()
+		defer logs.FlushLogs()
+		defer serviceability.BehaviorOnPanic(os.Getenv("OPENSHIFT_ON_PANIC"), version.Get())()
+		defer serviceability.Profile(os.Getenv("OPENSHIFT_PROFILE")).Stop()
+		serviceability.StartProfiler()
+
+		shutdownCtx, cancel := context.WithCancel(context.Background())
+		shutdownHandler := server.SetupSignalHandler()
+		go func() {
+			defer cancel()
+			<-shutdownHandler
+			klog.Infof("Received SIGTERM or SIGINT signal, shutting down controller.")
+		}()
+
+		ctx, terminate := context.WithCancel(shutdownCtx)
+		defer terminate()
+
+		terminateOnFiles, err := cmd.Flags().GetStringArray("terminate-on-files")
+		if err != nil {
+			klog.Fatal(err)
+		}
+		if len(terminateOnFiles) > 0 {
+			obs, err := fileobserver.NewObserver(10 * time.Second)
+			if err != nil {
+				klog.Fatal(err)
+			}
+			files := map[string][]byte{}
+			for _, fn := range terminateOnFiles {
+				fileBytes, err := os.ReadFile(fn)
+				if err != nil {
+					klog.Warningf("Unable to read initial content of %q: %v", fn, err)
+					continue
+				}
+				files[fn] = fileBytes
+			}
+			obs.AddReactor(func(filename string, action fileobserver.ActionType) error {
+				klog.Infof("exiting because %q changed", filename)
+				terminate()
+				return nil
+			}, files, terminateOnFiles...)
+			go obs.Run(shutdownHandler)
+		}
+
+		if err := startControllerWithClusterTLS(ctx, cc, cmd); err != nil {
+			klog.Fatal(err)
+		}
+	}
+
 	cmd.Flags().StringVar(&operator.TrustedCAConfigMapName, "trusted-ca-configmap", "", "The name of the config map containing TLS CA(s) which should be trusted by the controller's containers. PEM encoded file under \"ca-bundle.crt\" key is expected.")
 	cmd.Flags().StringVar(&operator.CloudCredentialSecret, "cloud-credentials-secret", "", "The name of the secret containing cloud credentials for authenticating using cert-manager ambient credentials mode.")
 
@@ -34,3 +100,86 @@ These features provide early access to upcoming product features,
 enabling customers to test functionality and provide feedback during the development process.`)
 	return cmd
 }
+
+func startControllerWithClusterTLS(ctx context.Context, c *controllercmd.ControllerCommandConfig, cmd *cobra.Command) error {
+	unstructuredConfig, config, configContent, err := c.Config()
+	if err != nil {
+		return err
+	}
+
+	startingFileContent, observedFiles, err := c.AddDefaultRotationToConfig(config, configContent)
+	if err != nil {
+		return err
+	}
+
+	listen, err := cmd.Flags().GetString("listen")
+	if err != nil {
+		return err
+	}
+	if len(listen) != 0 {
+		config.ServingInfo.BindAddress = listen
+	}
+
+	kubeConfigFile, err := cmd.Flags().GetString("kubeconfig")
+	if err != nil {
+		return err
+	}
+	namespace, err := cmd.Flags().GetString("namespace")
+	if err != nil {
+		return err
+	}
+
+	if !c.DisableServing {
+		restConfig, err := tlsprofile.RESTConfigFromKubeConfig(kubeConfigFile)
+		if err != nil {
+			klog.Warningf("unable to build rest config for cluster TLS profile lookup; using Controllercmd default TLS settings: %v", err)
+		} else {
+			lookupCtx, cancelLookup := context.WithTimeout(ctx, 30*time.Second)
+			err := tlsprofile.ApplyClusterProfileToHTTPServingInfo(lookupCtx, restConfig, &config.ServingInfo)
+			cancelLookup()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	exitOnChangeReactorCh := make(chan struct{})
+	controllerCtx, cancel := context.WithCancel(ctx)
+	go func() {
+		select {
+		case <-exitOnChangeReactorCh:
+			cancel()
+		case <-ctx.Done():
+			cancel()
+		}
+	}()
+
+	config.LeaderElection.Disable = c.DisableLeaderElection
+	config.LeaderElection.LeaseDuration = c.LeaseDuration
+	config.LeaderElection.RenewDeadline = c.RenewDeadline
+	config.LeaderElection.RetryPeriod = c.RetryPeriod
+
+	builder := controllercmd.NewController("cert-manager-operator", operator.RunOperator, clock.RealClock{}).
+		WithKubeConfigFile(kubeConfigFile, nil).
+		WithComponentNamespace(namespace).
+		WithLeaderElection(config.LeaderElection, namespace, "cert-manager-operator-lock").
+		WithVersion(version.Get()).
+		WithEventRecorderOptions(events.RecommendedClusterSingletonCorrelatorOptions()).
+		WithRestartOnChange(exitOnChangeReactorCh, startingFileContent, observedFiles...)
+
+	if !c.DisableServing {
+		builder = builder.WithServer(config.ServingInfo, config.Authentication, config.Authorization)
+		if c.EnableHTTP2 {
+			builder = builder.WithHTTP2()
+		}
+		if c.SkipInClusterAuthenticationLookup {
+			builder = builder.WithSkipInClusterAuthenticationLookup()
+		}
+	}
+
+	if c.TopologyDetector != nil {
+		builder = builder.WithTopologyDetector(c.TopologyDetector)
+	}
+
+	return builder.Run(controllerCtx, unstructuredConfig)
+}

pkg/controller/certmanager/cert_manager_controller_deployment.go

@@ -47,6 +47,8 @@ var (
 		"cert-manager-deployment/controller/cert-manager-tokenrequest-rb.yaml",
 		"cert-manager-deployment/controller/cert-manager-tokenrequest-role.yaml",
 		"cert-manager-deployment/controller/cert-manager-view-cr.yaml",
+		"cert-manager-deployment/controller/cert-manager-metrics-dynamic-serving-role.yaml",
+		"cert-manager-deployment/controller/cert-manager-metrics-dynamic-serving-rb.yaml",
 		"cert-manager-deployment/cert-manager/cert-manager-controller-approve-cert-manager-io-cr.yaml",
 		"cert-manager-deployment/cert-manager/cert-manager-controller-approve-cert-manager-io-crb.yaml",
 		"cert-manager-deployment/cert-manager/cert-manager-controller-certificatesigningrequests-cr.yaml",