feat(controller): make namespace clusterrole configurable

Controllers/Sync.cs

@@ -1,4 +1,5 @@
 using ConplementAG.CopsController.Services;
+using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
 using Serilog;
 using System;
@@ -11,6 +12,13 @@ namespace ConplementAG.CopsController.Controllers
     [ApiController]
     public class SyncController : ControllerBase
     {
+        private readonly IConfiguration _configuration;
+
+        public SyncController(IConfiguration configuration)
+        {
+            _configuration = configuration;
+        }
+
         [HttpGet]
         public ActionResult<string> Get()
         {
@@ -25,7 +33,8 @@ public IActionResult Post([FromBody]JObject value)
                 Log.Verbose("REQUEST===============================" + value.ToString(Newtonsoft.Json.Formatting.Indented));
 
                 var copsResource = CopsResourceFactory.Create(value);
-                var k8sResources = K8sResourceFactory.Create(copsResource);
+                var namespaceAdminRole = _configuration.GetValue("CopsController:NamespaceAdminRole", "cluster-admin");
+                var k8sResources = K8sResourceFactory.Create(copsResource, namespaceAdminRole);
 
                 JObject response = JObject.FromObject(
                     new

Models/K8sRoleBinding.cs

@@ -22,8 +22,9 @@ public class K8sRoleBinding
         [JsonProperty("roleRef")]
         public K8sRoleRef RoleRef { get; set; }
 
-        public static K8sRoleBinding NamespaceFullAccess(string namespacename, 
-            ICollection<string> users, ICollection<CopsAdminServiceAccountSpec> serviceAccounts)
+        public static K8sRoleBinding NamespaceFullAccess(string namespacename,
+            ICollection<string> users, ICollection<CopsAdminServiceAccountSpec> serviceAccounts,
+            string roleName = "cluster-admin")
         {
             if (string.IsNullOrEmpty(namespacename))
             {
@@ -52,11 +53,7 @@ public static K8sRoleBinding NamespaceFullAccess(string namespacename,
                 Kind = "RoleBinding",
                 ApiVersion = "rbac.authorization.k8s.io/v1",
                 Metadata = new K8sMetadata { Name = $"copsnamespace-user", Namespace = namespacename },
-                // The in-built clusterrole cluster-admin allows access to all resources (wildcard),
-                // so that we can use that clusterrole instead of writing our own which is far more brittle.
-                // Since we scope the cluster-admin to a namespace using RoleBinding (in contrast to ClusterRoleBinding)
-                // this is a good approach.
-                RoleRef = new K8sRoleRef("ClusterRole", "cluster-admin", "rbac.authorization.k8s.io")
+                RoleRef = new K8sRoleRef("ClusterRole", roleName, "rbac.authorization.k8s.io")
             };
 
             var subjects = users.ToList()

Services/K8sResourceFactory.cs

@@ -7,22 +7,23 @@ namespace ConplementAG.CopsController.Services
 {
     public class K8sResourceFactory
     {
-        public static IList<object> Create(CopsResource resource)
+        public static IList<object> Create(CopsResource resource, string namespaceAdminRole = "cluster-admin")
         {
             var source = Convert.ChangeType(resource, resource.GetType());
 
             var method = typeof(K8sResourceFactory).GetMethod("Create", BindingFlags.NonPublic | BindingFlags.Static);
-            return (IList<object>)method.Invoke(null, new[] { source });
+            return (IList<object>)method.Invoke(null, new[] { source, namespaceAdminRole });
         }
 
         // Method used by reflection call
-        private static IList<object> Create(CopsNamespace copsNamespace)
+        private static IList<object> Create(CopsNamespace copsNamespace, string namespaceAdminRole)
         {
             return new List<object>
             {
                 new K8sNamespace(copsNamespace.Metadata.Name, copsNamespace.Spec.Project?.Name, copsNamespace.Spec.Project?.CostCenter),
                 K8sRoleBinding.NamespaceFullAccess(copsNamespace.Metadata.Name, copsNamespace.Spec.NamespaceAdminUsers,
-                    copsNamespace.Spec.NamespaceAdminServiceAccounts ??  new List<CopsAdminServiceAccountSpec>().ToArray()),
+                    copsNamespace.Spec.NamespaceAdminServiceAccounts ??  new List<CopsAdminServiceAccountSpec>().ToArray(),
+                    namespaceAdminRole),
                 K8sClusterRoleBinding.CopsNamespaceEditBinding(copsNamespace.Metadata.Name, copsNamespace.Spec.NamespaceAdminUsers,
                     copsNamespace.Spec.NamespaceAdminServiceAccounts ??  new List<CopsAdminServiceAccountSpec>().ToArray()),
                 K8sClusterRole.CopsNamespaceEdit(copsNamespace.Metadata.Name),

appsettings.json

@@ -4,5 +4,8 @@
       "Default": "Information"
     }
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  "CopsController": {
+    "NamespaceAdminRole": "cluster-admin"
+  }
 }

deployment/cops-controller/templates/03-deployment.yaml

@@ -31,4 +31,6 @@ spec:
           env:
           - name: Serilog__MinimumLevel
             value: Error
+          - name: CopsController__NamespaceAdminRole
+            value: {{ .Values.copsController.namespaceAdminRole }}
 

deployment/cops-controller/values.yaml

@@ -1,3 +1,5 @@
 image:
   repository: conplementag/cops-controller
-  tag: 1.10.1 # x-release-please-version
+  tag: 1.10.1 # x-release-please-version
+copsController:
+  namespaceAdminRole: cluster-admin

deployment/crds/copsnamespace.crd.yaml

@@ -103,7 +103,7 @@ spec:
                 type: object
                 required: 
                 - name
-                - costcenter
+                - costCenter
                 properties:
                     name:
                         type: string

run_tests.sh

@@ -6,11 +6,12 @@ programname=$0
 UNIVERSAL_TEST_IDENTIFIED="cops-controller-component-tests"
 
 function usage {
-    echo "usage: $programname [--install-helm-and-cops-controller] [-r repository] [-t tag]"
+    echo "usage: $programname [--install-helm-and-cops-controller] [-r repository] [-t tag] [-a namespaceAdminRole]"
     echo "  MAKE SURE YOU SPECIFY THE ARGUMENTS IN THE EXACT ORDER AS BELOW, THIS SCRIPT DOES NOT SUPPORT OUT OF ORDER ARGUMENTS!"
-    echo "  --install-helm-and-cops-controller (optional) use to install global helm in the cluster and to deploy the cops controller specified by the -r and -t arguments. Make sure you have Helm > 2.16 if you use this option and you are running on k8s > 1.16"
-    echo "  -r repository   (optional) cops controller image repository. Should be accessible from the cluster (e.g. remote registry or local one shared with the cluster)."
-    echo "  -t tag          (optional) cops controller image tag."
+    echo "  --install-helm-and-cops-controller (optional) install metacontroller and the cops controller specified by the -r and -t arguments. Requires Helm 3 and kubectl."
+    echo "  -r repository           (optional) cops controller image repository. Should be accessible from the cluster (e.g. remote registry or local one shared with the cluster)."
+    echo "  -t tag                  (optional) cops controller image tag."
+    echo "  -a namespaceAdminRole   (optional) ClusterRole to bind for namespace admins. Defaults to devops-namespace-admin ."
     echo " "
     echo "Prerequisites: "
     echo "   To run the tests, you need a running k8s cluster and docker engine"
@@ -83,6 +84,7 @@ function cleanup {
 installController="no"
 repository=""
 tag=""
+namespaceAdminRole="devops-namespace-admin"
 
 if [ -n "$1" ]; then # if any argument specified
     # all parameters mandatory now, in correct order
@@ -92,10 +94,13 @@ if [ -n "$1" ]; then # if any argument specified
         installController="yes"
         repository=$3
         tag=$5
+        if [ "$6" == "-a" ] && [ -n "$7" ]; then
+            namespaceAdminRole=$7
+        fi
     fi
 fi
 
 # register the cleanup function for all signal types (emulating finally block)
 trap cleanup EXIT ERR INT TERM
 
-. ./tests/tests.sh "$installController" "$repository" "$tag"
+. ./tests/tests.sh "$installController" "$repository" "$tag" "$namespaceAdminRole"

tests/1-empire-cns.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: {{NAMESPACE}}
@@ -10,4 +10,7 @@ spec:
   - Second.User@conplement.de
   namespaceAdminServiceAccounts:
   - serviceAccount: {{SERVICE_ACCOUNT}}
-    namespace: {{SERVICE_ACCOUNT_NAMESPACE}}
+    namespace: {{SERVICE_ACCOUNT_NAMESPACE}}
+  project:
+    name: empire
+    costCenter: "66"

tests/2-updated-cns.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: {{NAMESPACE}}
@@ -13,4 +13,7 @@ spec:
   - serviceAccount: {{SERVICE_ACCOUNT}}
     namespace: {{SERVICE_ACCOUNT_NAMESPACE}}
   - serviceAccount: {{ADDITIONAL_SERVICE_ACCOUNT}}
-    namespace: {{ADDITIONAL_SERVICE_ACCOUNT_NAMESPACE}}
+    namespace: {{ADDITIONAL_SERVICE_ACCOUNT_NAMESPACE}}
+  project:
+    name: empire
+    costCenter: "66"

tests/invalid-definitions/1.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-invalid-ns-1-without-spec

tests/invalid-definitions/2.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-invalid-ns-2-without-users

tests/invalid-definitions/3.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-invalid-ns-3-only-with-sas

tests/invalid-definitions/4.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-invalid-ns-4-with-invalid-sa-formats-1

tests/invalid-definitions/5.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-invalid-ns-5-with-invalid-sa-formats-2

tests/invalid-definitions/6.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-invalid-ns-6-with-invalid-sa-formats-3

tests/invalid-definitions/7.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-invalid-ns-7-with-invalid-sa-formats-4

tests/tests.sh

@@ -21,6 +21,7 @@ function logTestStarted {
 serviceAccountsNamespace="default"
 darthVaderAccount="cops-controller-darth-vader"
 kyloRenAccount="cops-controller-kylo-ren"
+namespaceAdminRole="${4:-devops-namespace-admin}"
 
 #########################################################################
 #                           Arrange helpers                             #
@@ -29,25 +30,33 @@ function setupCluster {
     installController=$1
     repository=$2
     tag=$3
+    role=$4
 
     if [ $installController == "yes" ]; then
-        kubectl -n kube-system create serviceaccount tiller --dry-run=true -o yaml | kubectl apply -f -
-        kubectl create clusterrolebinding tiller --clusterrole=cluster-admin --serviceaccount=kube-system:tiller --dry-run=true -o yaml | kubectl apply -f -
-        helm init --service-account tiller --wait --upgrade
-
-        # ensure metacontroller dependency in the cluster
-        metacontrollerVersion="v0.4.0"
-        kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/metacontroller/${metacontrollerVersion}/manifests/metacontroller-namespace.yaml"
-        kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/metacontroller/${metacontrollerVersion}/manifests/metacontroller-rbac.yaml"
-        kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/metacontroller/${metacontrollerVersion}/manifests/metacontroller.yaml"
+        # ensure metacontroller dependency in the cluster (v0.4.0 used v1beta1 CRDs removed in k8s 1.22+;
+        # project moved to metacontroller/metacontroller and is now distributed via Helm)
+        helm upgrade --install metacontroller oci://ghcr.io/metacontroller/metacontroller-helm \
+            --version=4.15.0 \
+            --namespace metacontroller \
+            --create-namespace \
+            --skip-crds \
+            --wait
 
         # install cops controller via the local chart
         copsControllerNamespace="coreops-cops-controller-component-test"
         kubectl apply -f deployment/crds
+        kubectl create namespace $copsControllerNamespace --dry-run=client -o yaml | kubectl apply -f -
+
+        # uninstall cops-controller from any namespace it may already occupy (cluster-scoped resources
+        # like ClusterRoles cannot be owned by two releases simultaneously)
+        for ns in $(helm list -A -o json 2>/dev/null | python3 -c "import json,sys; [print(r['namespace']) for r in json.load(sys.stdin) if r['name']=='cops-controller']" 2>/dev/null); do
+            helm uninstall cops-controller --namespace $ns --wait 2>/dev/null || true
+        done
 
-        helm upgrade --install --wait --timeout 60 --namespace $copsControllerNamespace \
+        helm upgrade --install --wait --timeout 60s --namespace $copsControllerNamespace \
             --set image.repository=$repository \
             --set image.tag=$tag \
+            --set copsController.namespaceAdminRole=$role \
             cops-controller deployment/cops-controller
     fi
 }
@@ -62,12 +71,11 @@ function setupServiceAccount {
     testAccountNamespace=$2
 
     # create account
-    kubectl create serviceaccount $testAccount -n $testAccountNamespace --dry-run=true -o yaml | kubectl apply -f -
+    kubectl create serviceaccount $testAccount -n $testAccountNamespace --dry-run=client -o yaml | kubectl apply -f -
 
-    # extract the secret, set into kubeconfig
-    serviceAccountSecret=$(kubectl get serviceaccount $testAccount -n $testAccountNamespace -o jsonpath={.secrets[0].name})
-    token=$(kubectl get secret $serviceAccountSecret -n $testAccountNamespace -o jsonpath={.data.token} | base64 --decode)
-    kubectl config set-credentials $testAccount --token=$token    
+    # create a token for the service account (works with k8s 1.24+ which no longer auto-creates SA secrets)
+    token=$(kubectl create token $testAccount -n $testAccountNamespace)
+    kubectl config set-credentials $testAccount --token=$token
 
     # create the new kubeconfig context
     kubectl config set-context $testAccount --user=$testAccount --cluster=$(kubectl config view --minify -o jsonpath={.clusters[0].name})
@@ -199,6 +207,13 @@ function test_shouldDeployEmpireCnsWithValidRbac {
     ensureAccessToNamespace $namespaceName
     ensureAllResourcesAreSupported $namespaceName
 
+    kubectl config use-context $INITIAL_CONTEXT
+    roleRefName=$(kubectl get rolebinding copsnamespace-user -n $namespaceName -o jsonpath='{.roleRef.name}')
+    if [ "$roleRefName" != "$namespaceAdminRole" ]; then
+        fail "Expected rolebinding roleRef to be $namespaceAdminRole but got $roleRefName"
+    fi
+    success "RoleBinding roleRef is $namespaceAdminRole as expected"
+
     # no access for other accounts
     kubectl config use-context $kyloRenAccount
 
@@ -249,7 +264,7 @@ function test_shouldUpdateEmpireCnsWithAdditionalRbac {
 #                              Test runner                              #
 #########################################################################
 
-setupCluster "$1" "$2" "$3"
+setupCluster "$1" "$2" "$3" "$namespaceAdminRole"
 
 setupTheEmpireServiceAccounts
 

tests/valid-definitions/1.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-valid-ns-1-typical-with-non-existing-sa
@@ -12,4 +12,7 @@ spec:
   - serviceAccount: default # should work with existing sa
     namespace: default
   - serviceAccount: notyetthere # but also with non-existing
-    namespace: default
+    namespace: default
+  project:
+    name: test-project-1
+    costCenter: "42"

tests/valid-definitions/2.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-valid-ns-2-only-users
@@ -7,4 +7,7 @@ metadata:
 spec:
   namespaceAdminUsers: # should work without service account which are optional
   - Test.User@conplement.de
-  - Second.User@conplement.de
+  - Second.User@conplement.de
+  project:
+    name: test-project-2
+    costCenter: "42"

tests/valid-definitions/3.yaml

@@ -1,4 +1,4 @@
-apiVersion: coreops.conplement.cloud/v1
+apiVersion: coreops.conplement.cloud/v2
 kind: CopsNamespace
 metadata:
   name: test-valid-ns-3-lot-of-users-and-sa
@@ -37,4 +37,7 @@ spec:
   - serviceAccount: account4
     namespace: custom
   - serviceAccount: account5
-    namespace: custom
+    namespace: custom
+  project:
+    name: test-project-3
+    costCenter: "42"