diff --git a/images/phase-1/11-eks-nodes-staging.png b/images/phase-1/11-eks-nodes-staging.png new file mode 100644 index 0000000..f497f18 Binary files /dev/null and b/images/phase-1/11-eks-nodes-staging.png differ diff --git a/images/phase-1/12-eks-cluster-staging.png b/images/phase-1/12-eks-cluster-staging.png new file mode 100644 index 0000000..200fd32 Binary files /dev/null and b/images/phase-1/12-eks-cluster-staging.png differ diff --git a/images/phase-3/08-eks-app.png b/images/phase-3/08-eks-app.png new file mode 100644 index 0000000..03e04f2 Binary files /dev/null and b/images/phase-3/08-eks-app.png differ diff --git a/images/phase-3/09-audit-endpoint.png b/images/phase-3/09-audit-endpoint.png new file mode 100644 index 0000000..5264133 Binary files /dev/null and b/images/phase-3/09-audit-endpoint.png differ diff --git a/images/phase-5/01-argocd-sync.png b/images/phase-5/01-argocd-sync.png new file mode 100644 index 0000000..96aa5f2 Binary files /dev/null and b/images/phase-5/01-argocd-sync.png differ diff --git a/images/phase-5/03-blue-green-virtualservice.png b/images/phase-5/03-blue-green-virtualservice.png new file mode 100644 index 0000000..6506a72 Binary files /dev/null and b/images/phase-5/03-blue-green-virtualservice.png differ diff --git a/images/phase-6/01-kyverno-enforced.png b/images/phase-6/01-kyverno-enforced.png new file mode 100644 index 0000000..0864ac8 Binary files /dev/null and b/images/phase-6/01-kyverno-enforced.png differ diff --git a/images/phase-6/02-networkpolicies-rba.png b/images/phase-6/02-networkpolicies-rba.png new file mode 100644 index 0000000..3d5c47c Binary files /dev/null and b/images/phase-6/02-networkpolicies-rba.png differ diff --git a/images/phase-6/03-pod-securitycontext.png b/images/phase-6/03-pod-securitycontext.png new file mode 100644 index 0000000..dfadf34 Binary files /dev/null and b/images/phase-6/03-pod-securitycontext.png differ diff --git a/images/phase-7/01-grafana-dashboard.png b/images/phase-7/01-grafana-dashboard.png new file mode 100644 index 0000000..3c7507a Binary files /dev/null and b/images/phase-7/01-grafana-dashboard.png differ diff --git a/images/phase-7/02-prometheus-targets.png b/images/phase-7/02-prometheus-targets.png new file mode 100644 index 0000000..bb77e51 Binary files /dev/null and b/images/phase-7/02-prometheus-targets.png differ diff --git a/images/phase-7/03-prometheus-alerts.png b/images/phase-7/03-prometheus-alerts.png new file mode 100644 index 0000000..63b1712 Binary files /dev/null and b/images/phase-7/03-prometheus-alerts.png differ diff --git a/images/phase-8/01-pod-recover.png b/images/phase-8/01-pod-recover.png new file mode 100644 index 0000000..de1cd6c Binary files /dev/null and b/images/phase-8/01-pod-recover.png differ diff --git a/k8s/overlays/staging/kustomization.yaml b/k8s/overlays/staging/kustomization.yaml index 2b0b172..5cfe1f7 100644 --- a/k8s/overlays/staging/kustomization.yaml +++ b/k8s/overlays/staging/kustomization.yaml @@ -17,11 +17,12 @@ patches: - path: patch-ingress.yaml - path: patch-blue-deployment.yaml - path: patch-green-deployment.yaml + - path: patch-resourcequota.yaml images: - name: backend-blue newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend - newTag: latest + newTag: sha-5137706 - name: backend-green newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend - newTag: latest + newTag: sha-5137706 diff --git a/k8s/overlays/staging/patch-resourcequota.yaml b/k8s/overlays/staging/patch-resourcequota.yaml new file mode 100644 index 0000000..fc6c8d5 --- /dev/null +++ b/k8s/overlays/staging/patch-resourcequota.yaml @@ -0,0 +1,16 @@ +# Istio sidecars add ~2 CPU limit per pod; rolling updates briefly double pods. +# Raise the namespace quota so blue/green + postgres/redis (all with sidecars) +# fit. Mirrors the dev overlay patch. Requests stay low, so actual scheduling +# fits the node capacity. +apiVersion: v1 +kind: ResourceQuota +metadata: + name: namespace-quota +spec: + hard: + requests.cpu: "8" + requests.memory: 16Gi + limits.cpu: "24" + limits.memory: 48Gi + pods: "40" + persistentvolumeclaims: "4" diff --git a/monitoring/grafana/dashboards/app-feature-flags.json b/monitoring/grafana/dashboards/app-feature-flags.json index 345ac05..0d5d5cd 100644 --- a/monitoring/grafana/dashboards/app-feature-flags.json +++ b/monitoring/grafana/dashboards/app-feature-flags.json @@ -14,7 +14,7 @@ "name": "namespace", "type": "query", "datasource": "Prometheus", - "query": "label_values(istio_requests_total{destination_workload=\"backend\"}, destination_workload_namespace)", + "query": "label_values(istio_requests_total{destination_canonical_service=\"backend\"}, destination_workload_namespace)", "refresh": 2, "includeAll": true } @@ -28,7 +28,7 @@ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "targets": [ { - "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",destination_workload_namespace=~\"$namespace\"}[5m]))", + "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",destination_workload_namespace=~\"$namespace\"}[5m]))", "legendFormat": "total" } ] @@ -41,7 +41,7 @@ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "targets": [ { - "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[5m])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[5m])), 1)", + "expr": "(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",response_code=~\"5..\"}[5m])) or vector(0)) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[5m])), 1)", "legendFormat": "error ratio" } ] @@ -54,37 +54,33 @@ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=\"backend\"}[5m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_canonical_service=\"backend\"}[5m])) by (le))", "legendFormat": "p95" } ] }, { - "title": "Redis cache hit ratio", + "title": "Requests by response code", "type": "timeseries", "datasource": "Prometheus", - "fieldConfig": { "defaults": { "unit": "percentunit" } }, + "fieldConfig": { "defaults": { "unit": "reqps" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "targets": [ { - "expr": "sum(rate(flag_cache_hits_total[5m])) / clamp_min(sum(rate(flag_cache_hits_total[5m])) + sum(rate(flag_cache_misses_total[5m])), 1)", - "legendFormat": "hit ratio" + "expr": "sum by (response_code) (rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[5m]))", + "legendFormat": "{{response_code}}" } ] }, { - "title": "HPA replicas", + "title": "Backend replicas (blue/green)", "type": "timeseries", "datasource": "Prometheus", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "targets": [ { - "expr": "kube_horizontalpodautoscaler_status_current_replicas{horizontalpodautoscaler=\"backend\"}", - "legendFormat": "{{namespace}} current" - }, - { - "expr": "kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler=\"backend\"}", - "legendFormat": "{{namespace}} max" + "expr": "sum by (namespace, deployment) (kube_deployment_status_replicas{deployment=~\"backend.*\"})", + "legendFormat": "{{namespace}}/{{deployment}}" } ] } diff --git a/monitoring/grafana/dashboards/slo-error-budget.json b/monitoring/grafana/dashboards/slo-error-budget.json index f9634a8..6988de8 100644 --- a/monitoring/grafana/dashboards/slo-error-budget.json +++ b/monitoring/grafana/dashboards/slo-error-budget.json @@ -26,7 +26,7 @@ "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, "targets": [ { - "expr": "1 - (sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[30d])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[30d])), 1))", + "expr": "1 - ((sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",response_code=~\"5..\"}[30d])) or vector(0)) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[30d])), 1))", "legendFormat": "availability" } ] @@ -53,7 +53,7 @@ "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, "targets": [ { - "expr": "1 - ((sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[30d])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[30d])), 1)) / 0.001)", + "expr": "1 - (((sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\",response_code=~\"5..\"}[30d])) or vector(0)) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_canonical_service=\"backend\"}[30d])), 1)) / 0.001)", "legendFormat": "budget remaining" } ] diff --git a/monitoring/kustomization.yaml b/monitoring/kustomization.yaml index b5d55bd..51eb82f 100644 --- a/monitoring/kustomization.yaml +++ b/monitoring/kustomization.yaml @@ -6,5 +6,6 @@ kind: Kustomization resources: - prometheus/servicemonitor-backend.yaml + - prometheus/podmonitor-istio.yaml - prometheus/prometheusrules.yaml - grafana/dashboards diff --git a/monitoring/prometheus/podmonitor-istio.yaml b/monitoring/prometheus/podmonitor-istio.yaml new file mode 100644 index 0000000..834b8d7 --- /dev/null +++ b/monitoring/prometheus/podmonitor-istio.yaml @@ -0,0 +1,40 @@ +# Scrape Istio sidecar (Envoy) stats on port 15020 so mesh metrics +# (istio_requests_total, istio_request_duration_milliseconds_bucket) reach +# Prometheus. These power the SLO rules and custom dashboards. Standard Istio +# PodMonitor adapted for kube-prometheus-stack (release label for selection). +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: envoy-stats + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + selector: + matchExpressions: + - { key: istio-prometheus-ignore, operator: DoesNotExist } + namespaceSelector: + any: true + jobLabel: envoy-stats + podMetricsEndpoints: + - path: /stats/prometheus + interval: 15s + relabelings: + - action: keep + sourceLabels: [__meta_kubernetes_pod_container_name] + regex: "istio-proxy" + - action: keep + sourceLabels: [__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape] + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:15020 + sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip] + targetLabel: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - sourceLabels: [__meta_kubernetes_namespace] + action: replace + targetLabel: namespace + - sourceLabels: [__meta_kubernetes_pod_name] + action: replace + targetLabel: pod_name diff --git a/monitoring/prometheus/prometheusrules.yaml b/monitoring/prometheus/prometheusrules.yaml index cf6dece..40aff56 100644 --- a/monitoring/prometheus/prometheusrules.yaml +++ b/monitoring/prometheus/prometheusrules.yaml @@ -21,24 +21,24 @@ spec: # Error ratio (5xx / total) over multiple windows for burn-rate alerting. - record: backend:request_errors:ratio_rate5m expr: | - sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[5m])) + (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[5m])) or vector(0)) / - clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[5m])), 1) + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[5m])), 1) - record: backend:request_errors:ratio_rate30m expr: | - sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[30m])) + (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[30m])) or vector(0)) / - clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[30m])), 1) + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[30m])), 1) - record: backend:request_errors:ratio_rate1h expr: | - sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[1h])) + (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[1h])) or vector(0)) / - clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[1h])), 1) + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[1h])), 1) - record: backend:request_errors:ratio_rate6h expr: | - sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[6h])) + (sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend",response_code=~"5.."}[6h])) or vector(0)) / - clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[6h])), 1) + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_canonical_service="backend"}[6h])), 1) # Availability SLI (non-5xx ratio). - record: backend:availability:ratio_rate5m expr: 1 - backend:request_errors:ratio_rate5m @@ -46,7 +46,7 @@ spec: - record: backend:request_latency_p95_ms:5m expr: | histogram_quantile(0.95, - sum(rate(istio_request_duration_milliseconds_bucket{reporter="destination",destination_workload="backend"}[5m])) by (le)) + sum(rate(istio_request_duration_milliseconds_bucket{reporter="destination",destination_canonical_service="backend"}[5m])) by (le)) - name: backend.slo.burnrate rules: