Feat/phase 9 docs adrs

k8s/overlays/staging/kustomization.yaml

@@ -17,11 +17,12 @@ patches:
   - path: patch-ingress.yaml
   - path: patch-blue-deployment.yaml
   - path: patch-green-deployment.yaml
+  - path: patch-resourcequota.yaml
 
 images:
   - name: backend-blue
     newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend
-    newTag: latest
+    newTag: sha-5137706
   - name: backend-green
     newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend
-    newTag: latest
+    newTag: sha-5137706

k8s/overlays/staging/patch-resourcequota.yaml

@@ -0,0 +1,16 @@
+# Istio sidecars add ~2 CPU limit per pod; rolling updates briefly double pods.
+# Raise the namespace quota so blue/green + postgres/redis (all with sidecars)
+# fit. Mirrors the dev overlay patch. Requests stay low, so actual scheduling
+# fits the node capacity.
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: namespace-quota
+spec:
+  hard:
+    requests.cpu: "8"
+    requests.memory: 16Gi
+    limits.cpu: "24"
+    limits.memory: 48Gi
+    pods: "40"
+    persistentvolumeclaims: "4"

monitoring/grafana/dashboards/app-feature-flags.json

@@ -14,7 +14,7 @@
         "name": "namespace",
         "type": "query",
         "datasource": "Prometheus",
-        "query": "label_values(istio_requests_total{destination_workload=\"backend\"}, destination_workload_namespace)",
+        "query": "label_values(istio_requests_total{destination_canonical_service=\"backend\"}, destination_workload_namespace)",
         "refresh": 2,
         "includeAll": true
       }
@@ -28,7 +28,7 @@
       "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
       "targets": [
         {
-          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",destination_workload_namespace=~\"$namespace\"}[5m]))",
+          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",destination_workload_namespace=~\"$namespace\"}[5m]))",
           "legendFormat": "total"
         }
       ]
@@ -41,7 +41,7 @@
       "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
       "targets": [
         {
-          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[5m])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[5m])), 1)",
+          "expr": "(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",response_code=~\"5..\"}[5m])) or vector(0)) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[5m])), 1)",
           "legendFormat": "error ratio"
         }
       ]
@@ -54,37 +54,33 @@
       "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=\"backend\"}[5m])) by (le))",
+          "expr": "histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_canonical_service=\"backend\"}[5m])) by (le))",
           "legendFormat": "p95"
         }
       ]
     },
     {
-      "title": "Redis cache hit ratio",
+      "title": "Requests by response code",
       "type": "timeseries",
       "datasource": "Prometheus",
-      "fieldConfig": { "defaults": { "unit": "percentunit" } },
+      "fieldConfig": { "defaults": { "unit": "reqps" } },
       "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
       "targets": [
         {
-          "expr": "sum(rate(flag_cache_hits_total[5m])) / clamp_min(sum(rate(flag_cache_hits_total[5m])) + sum(rate(flag_cache_misses_total[5m])), 1)",
-          "legendFormat": "hit ratio"
+          "expr": "sum by (response_code) (rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[5m]))",
+          "legendFormat": "{{response_code}}"
         }
       ]
     },
     {
-      "title": "HPA replicas",
+      "title": "Backend replicas (blue/green)",
       "type": "timeseries",
       "datasource": "Prometheus",
       "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
       "targets": [
         {
-          "expr": "kube_horizontalpodautoscaler_status_current_replicas{horizontalpodautoscaler=\"backend\"}",
-          "legendFormat": "{{namespace}} current"
-        },
-        {
-          "expr": "kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler=\"backend\"}",
-          "legendFormat": "{{namespace}} max"
+          "expr": "sum by (namespace, deployment) (kube_deployment_status_replicas{deployment=~\"backend.*\"})",
+          "legendFormat": "{{namespace}}/{{deployment}}"
         }
       ]
     }

monitoring/grafana/dashboards/slo-error-budget.json

@@ -26,7 +26,7 @@
       "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 },
       "targets": [
         {
-          "expr": "1 - (sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[30d])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[30d])), 1))",
+          "expr": "1 - ((sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",response_code=~\"5..\"}[30d])) or vector(0)) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[30d])), 1))",
           "legendFormat": "availability"
         }
       ]
@@ -53,7 +53,7 @@
       "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 },
       "targets": [
         {
-          "expr": "1 - ((sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[30d])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[30d])), 1)) / 0.001)",
+          "expr": "1 - (((sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",response_code=~\"5..\"}[30d])) or vector(0)) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[30d])), 1)) / 0.001)",
           "legendFormat": "budget remaining"
         }
       ]

monitoring/kustomization.yaml

@@ -6,5 +6,6 @@ kind: Kustomization
 
 resources:
   - prometheus/servicemonitor-backend.yaml
+  - prometheus/podmonitor-istio.yaml
   - prometheus/prometheusrules.yaml
   - grafana/dashboards

monitoring/prometheus/podmonitor-istio.yaml

@@ -0,0 +1,40 @@
+# Scrape Istio sidecar (Envoy) stats on port 15020 so mesh metrics
+# (istio_requests_total, istio_request_duration_milliseconds_bucket) reach
+# Prometheus. These power the SLO rules and custom dashboards. Standard Istio
+# PodMonitor adapted for kube-prometheus-stack (release label for selection).
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: envoy-stats
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  selector:
+    matchExpressions:
+      - { key: istio-prometheus-ignore, operator: DoesNotExist }
+  namespaceSelector:
+    any: true
+  jobLabel: envoy-stats
+  podMetricsEndpoints:
+    - path: /stats/prometheus
+      interval: 15s
+      relabelings:
+        - action: keep
+          sourceLabels: [__meta_kubernetes_pod_container_name]
+          regex: "istio-proxy"
+        - action: keep
+          sourceLabels: [__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape]
+        - action: replace
+          regex: (\d+);((([0-9]+?)(\.|$)){4})
+          replacement: $2:15020
+          sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip]
+          targetLabel: __address__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - sourceLabels: [__meta_kubernetes_namespace]
+          action: replace
+          targetLabel: namespace
+        - sourceLabels: [__meta_kubernetes_pod_name]
+          action: replace
+          targetLabel: pod_name

monitoring/prometheus/prometheusrules.yaml

@@ -21,32 +21,32 @@ spec:
         # Error ratio (5xx / total) over multiple windows for burn-rate alerting.
         - record: backend:request_errors:ratio_rate5m
           expr: |
-            sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[5m]))
+            (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[5m])) or vector(0))
             /
-            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[5m])), 1)
+            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[5m])), 1)
         - record: backend:request_errors:ratio_rate30m
           expr: |
-            sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[30m]))
+            (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[30m])) or vector(0))
             /
-            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[30m])), 1)
+            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[30m])), 1)
         - record: backend:request_errors:ratio_rate1h
           expr: |
-            sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[1h]))
+            (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[1h])) or vector(0))
             /
-            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[1h])), 1)
+            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[1h])), 1)
         - record: backend:request_errors:ratio_rate6h
           expr: |
-            sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[6h]))
+            (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[6h])) or vector(0))
             /
-            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[6h])), 1)
+            clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[6h])), 1)
         # Availability SLI (non-5xx ratio).
         - record: backend:availability:ratio_rate5m
           expr: 1 - backend:request_errors:ratio_rate5m
         # P95 latency (ms) for cached flag reads.
         - record: backend:request_latency_p95_ms:5m
           expr: |
             histogram_quantile(0.95,
-              sum(rate(istio_request_duration_milliseconds_bucket{reporter="destination",destination_workload="backend"}[5m])) by (le))
+              sum(rate(istio_request_duration_milliseconds_bucket{reporter="destination",destination_canonical_service="backend"}[5m])) by (le))
 
     - name: backend.slo.burnrate
       rules: