From 21b748fd2fd81acb7a2949d1d0d8b33dc5983c0c Mon Sep 17 00:00:00 2001 From: mck21 Date: Thu, 18 Jun 2026 16:56:31 +0200 Subject: [PATCH 1/3] docs: mark production runtime off; staging is the only working pipeline Reframe production from "deferred to final PLAN phase" to "off for now, next steps TBD". Production stays a Git mirror of staging (blue/green overlay) with no build, push, or deploy. Co-Authored-By: Claude Opus 4.8 --- docs/cd-production-promotion.md | 45 +++++-- docs/showcase-staging-only.md | 112 ++++++++++++++++++ images/phase-5/README.md | 4 +- .../environments/production/README.md | 40 +++++++ k8s/README.md | 20 ++-- k8s/argocd/install-notes.md | 6 +- k8s/overlays/production/active-color.yaml | 8 ++ k8s/overlays/production/kustomization.yaml | 20 +++- ...oyment.yaml => patch-blue-deployment.yaml} | 7 +- .../production/patch-delete-backend.yaml | 5 + k8s/overlays/production/patch-delete-hpa.yaml | 5 + .../production/patch-green-deployment.yaml | 22 ++++ 12 files changed, 266 insertions(+), 28 deletions(-) create mode 100644 docs/showcase-staging-only.md create mode 100644 infrastructure/environments/production/README.md create mode 100644 k8s/overlays/production/active-color.yaml rename k8s/overlays/production/{patch-deployment.yaml => patch-blue-deployment.yaml} (70%) create mode 100644 k8s/overlays/production/patch-delete-backend.yaml create mode 100644 k8s/overlays/production/patch-delete-hpa.yaml create mode 100644 k8s/overlays/production/patch-green-deployment.yaml diff --git a/docs/cd-production-promotion.md b/docs/cd-production-promotion.md index b0d65fe..66214d4 100644 --- a/docs/cd-production-promotion.md +++ b/docs/cd-production-promotion.md @@ -1,10 +1,12 @@ -# Production CD promotion (deferred) +# Production CD promotion (reference — not scheduled) -Production deploy is **intentionally disabled** during Phase 5 build to keep the pipeline green and AWS costs low. Staging is the sole automated CD target until the rest of the PLAN is complete. +> **Status:** Production runtime is **off** and not on the roadmap right now. Staging is the only counted pipeline. This document is a reference for *how* production could be turned on later; enabling it is an open decision, not a committed phase. -## When to enable +Production exists in **Git** as a mirror of staging (Kustomize overlays, Terraform, ArgoCD Application). **Runtime** (EKS cluster, deploy, ECR push) stays off — see [showcase-staging-only.md](showcase-staging-only.md). -After Phase 5 staging CD is stable and remaining PLAN phases are done (or when you explicitly choose to turn on multi-env). +## If/when production runtime is enabled + +There is no scheduled date. Revisit this only if we explicitly decide to go multi-environment. Prerequisite either way: staging CD proven stable. ## Steps @@ -12,11 +14,34 @@ After Phase 5 staging CD is stable and remaining PLAN phases are done (or when y | Variable | Value | |----------|-------| -| `ENABLE_PROD_ECR` | `true` — restores production ECR push on `main` in CI | -| `ENABLE_PROD_CD` | `true` — enable when the promote job is wired (future) | +| `ENABLE_PROD_CD` | `true` — enable promote job when wired | | `PRODUCTION_HEALTH_URL` | `https://flags.example.com/health` | -### 2. Bootstrap production EKS platform +### 2. Re-enable production ECR push in CI + +Add this step back to [`.github/workflows/ci.yaml`](../.github/workflows/ci.yaml) after `Push to staging ECR`: + +```yaml + - name: Push to production ECR + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: app/backend + push: true + tags: | + ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:${{ steps.tags.outputs.sha_tag }} + ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:latest + cache-from: type=gha + cache-to: type=gha,mode=max +``` + +Or gate behind `vars.ENABLE_PROD_ECR == 'true'` if you prefer opt-in. + +### 3. Apply production Terraform + +See [infrastructure/environments/production/README.md](../infrastructure/environments/production/README.md). + +### 4. Bootstrap production EKS platform Repeat the staging bootstrap from [k8s/argocd/install-notes.md](../k8s/argocd/install-notes.md) on the production cluster: @@ -25,13 +50,13 @@ Repeat the staging bootstrap from [k8s/argocd/install-notes.md](../k8s/argocd/in - ArgoCD Helm install - `backend-secrets` in namespace `production` -### 3. Register ArgoCD Application +### 5. Register ArgoCD Application ```bash kubectl apply -f k8s/argocd/application-production.yaml ``` -### 4. Add promote job to CD workflow +### 6. Add promote job to CD workflow Add a job (manual or automatic) after staging public health check succeeds: @@ -50,7 +75,7 @@ promote-production: # workflow_dispatch for manual approval is recommended first ``` -### 5. Blue/green on production (optional) +### 7. Blue/green on production (optional) Reuse `scripts/blue-green-switch.sh` and production overlay changes mirroring staging if zero-downtime promotion is required on production. diff --git a/docs/showcase-staging-only.md b/docs/showcase-staging-only.md new file mode 100644 index 0000000..0452120 --- /dev/null +++ b/docs/showcase-staging-only.md @@ -0,0 +1,112 @@ +# Showcase — staging-only runtime + +This portfolio project uses a **Git vs runtime** split for production: + +| Layer | Staging | Production | +|-------|---------|------------| +| **Git** (manifests, Terraform) | Active source | **Mirror** — kept aligned with staging | +| **Runtime** (AWS, EKS, CD) | **Lifted** for showcase | **Off** — not lifted; next steps undecided | + +Staging is the only environment with a working pipeline right now. You only need **staging** running for green CI + CD and Phase 5 screenshots. + +--- + +## Before your first push (one-time) + +### 1. Terraform — staging only + +```bash +cd infrastructure/environments/staging +terraform init +terraform apply +``` + +Do **not** run `terraform apply` in `infrastructure/environments/production/`. Production is off; that code stays valid in Git as a mirror only — see [infrastructure/environments/production/README.md](../infrastructure/environments/production/README.md). + +### 2. Configure kubectl + +```bash +aws eks update-kubeconfig \ + --name mck21-devsecops-staging-eks \ + --region us-east-1 + +kubectl get nodes # all Ready +``` + +### 3. Bootstrap EKS staging (cluster platform) + +Follow [k8s/argocd/install-notes.md](../k8s/argocd/install-notes.md) steps 1–8: + +- Namespaces, Istio, Ingress NGINX, cert-manager, `backend-secrets` +- ArgoCD Helm + repo connection +- `kubectl apply -f k8s/argocd/application-staging.yaml` + +Do **not** apply `k8s/argocd/application-production.yaml` — production runtime is off. See [cd-production-promotion.md](cd-production-promotion.md) only if production is ever turned on. + +### 4. GitHub repository variables + +| Variable | Value | +|----------|-------| +| `AWS_ACCOUNT_ID` | `125156866917` | +| `AWS_REGION` | `us-east-1` | +| `PROJECT_NAME` | `mck21-devsecops` | +| `EKS_CLUSTER_NAME` | `mck21-devsecops-staging-eks` | +| `STAGING_HEALTH_URL` | Leave empty until Ingress works, or `http(s):///health` | + +Optional: secret `CD_BOT_TOKEN` if branch protection blocks `GITHUB_TOKEN` GitOps pushes. + +### 5. Branch protection + +Allow GitHub Actions to push `[skip ci]` GitOps commits to `main`, or use `CD_BOT_TOKEN`. + +--- + +## Trigger pipeline + +```bash +git checkout main && git pull +git commit --allow-empty -m "chore: trigger staging cd validation" +git push origin main +``` + +Expected flow: + +1. **CI** — lint, test, build, Trivy, push image to **staging ECR only** +2. **CD** — bump idle color tag → ArgoCD sync → health check → traffic switch + +Monitor: + +```bash +gh run list --workflow=CI --limit 3 +gh run list --workflow=CD --limit 3 +``` + +--- + +## Keeping production in Git aligned + +When you change [k8s/overlays/staging/](../k8s/overlays/staging/), review [k8s/overlays/production/](../k8s/overlays/production/) in the same PR: + +- Same blue/green structure and sync waves +- Production-specific diffs: replicas (3), resources, ingress host (`flags.example.com`), ECR repo name + +No cluster deploy required for production YAML updates during showcase. + +--- + +## What production would need if ever turned on + +Production runtime is off and not scheduled. If we later decide to enable it, see [cd-production-promotion.md](cd-production-promotion.md): + +- `terraform apply` production +- Bootstrap production EKS +- Re-enable production ECR push in CI +- CD promote job + `application-production.yaml` + +--- + +## Phase 5 screenshots + +After CI + CD are green, capture evidence per [images/phase-5/README.md](../images/phase-5/README.md). + +Troubleshooting: [TROUBLESHOOTING.md § EKS staging CD](../TROUBLESHOOTING.md#eks-staging-cd-phase-5). diff --git a/images/phase-5/README.md b/images/phase-5/README.md index 7fd1c69..55891a3 100644 --- a/images/phase-5/README.md +++ b/images/phase-5/README.md @@ -1,5 +1,7 @@ # Phase 5 — CD & GitOps evidence +**Prerequisite:** Complete [docs/showcase-staging-only.md](../docs/showcase-staging-only.md) (terraform apply staging only, bootstrap EKS, push to `main`, CI + CD green). + Capture screenshots after the staging CD pipeline runs end-to-end on EKS. ## Checklist @@ -29,4 +31,4 @@ curl -f "$STAGING_HEALTH_URL" ## Notes - Dev (Minikube) CD is manual during Phase 5 — no screenshot required for phase closure. -- Production ArgoCD screenshots deferred until [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) is activated. +- Production exists in Git as a staging mirror only — runtime is **off** and not scheduled, so no production screenshots. Reference if ever turned on: [docs/cd-production-promotion.md](../docs/cd-production-promotion.md). diff --git a/infrastructure/environments/production/README.md b/infrastructure/environments/production/README.md new file mode 100644 index 0000000..7bce5d9 --- /dev/null +++ b/infrastructure/environments/production/README.md @@ -0,0 +1,40 @@ +# Production environment (Terraform) + +Valid Terraform for the production AWS stack — **Git mirror** of [../staging/](../staging/) with production sizing and a separate VPC CIDR. + +## Status: off (current) + +Production runtime is **off** and not scheduled. Staging is the only environment with a working pipeline. This directory is a Git mirror only. + +**Do not run `terraform apply` here.** + +- No production EKS cluster required for green CI/CD +- No AWS cost from production infra +- Code remains in the repo to demonstrate multi-environment IaC design + +When [staging](../staging/) modules or variables change, keep this directory aligned (same modules, production-specific values). + +## Differences from staging + +| Setting | Staging | Production | +|---------|---------|------------| +| VPC CIDR | `10.0.0.0/16` | `10.1.0.0/16` | +| Node type | `t3.medium` | `t3.large` | +| Node min | 1 | 2 | +| ECR retention | 10 images | 20 images | +| GitHub OIDC | Created in staging apply | Uses account-level provider | + +## When to apply + +There is no scheduled date. Apply only if we explicitly decide to enable production runtime — see [docs/cd-production-promotion.md](../../docs/cd-production-promotion.md). + +```bash +cd infrastructure/environments/production +terraform init +terraform plan +terraform apply # only when promoting to production runtime +``` + +## State + +Backend key: `production/terraform.tfstate` in bucket `devsecops-tfstate-125156866917`. diff --git a/k8s/README.md b/k8s/README.md index d03d097..36745fe 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -12,8 +12,8 @@ k8s/ ├── overlays/ │ ├── dev/ # Minikube — deploy manually │ ├── staging/ # EKS — GitOps CD target (Phase 5) -│ └── production/ # EKS — deferred CD -├── blue-green/ # Istio blue/green (included in staging overlay) +│ └── production/ # EKS — Git mirror of staging (runtime OFF) +├── blue-green/ # Istio blue/green (staging + production overlays) ├── argocd/ # Application CRDs ├── platform/ # EKS platform manifests (cert-manager issuer) └── namespaces/ # Platform namespaces @@ -24,10 +24,10 @@ k8s/ | Path | Status | Notes | |------|--------|-------| | `overlays/dev/` | **Live** — manual Minikube | Full stack: backend, postgres, redis, HPA | -| `overlays/staging/` | **Live** — ArgoCD + CD | Blue/green, ECR images, sync waves | -| `overlays/production/` | Scaffolding | CD deferred — [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) | -| `blue-green/` | Wired in staging | Traffic switch via `scripts/blue-green-switch.sh` | -| `argocd/` | Staging Application active | Bootstrap: [argocd/install-notes.md](argocd/install-notes.md) | +| `overlays/staging/` | **Live runtime** — ArgoCD + CD | Blue/green, ECR, sync waves | +| `overlays/production/` | **Off** — Git mirror, no runtime deploy | Same structure as staging; HA patches, `flags.example.com` | +| `blue-green/` | Wired in staging + production overlays | Traffic switch via `scripts/blue-green-switch.sh` | +| `argocd/` | Staging Application on cluster | Bootstrap: [argocd/install-notes.md](argocd/install-notes.md) | ## Deploy to Minikube (dev) @@ -57,7 +57,7 @@ Add to `/etc/hosts`: `127.0.0.1 flags.dev.local` and run `minikube tunnel`. ## Staging (EKS) — GitOps CD -**One-time bootstrap:** [k8s/argocd/install-notes.md](argocd/install-notes.md) +**Full runbook:** [docs/showcase-staging-only.md](../docs/showcase-staging-only.md) · **Bootstrap:** [k8s/argocd/install-notes.md](argocd/install-notes.md) After bootstrap, merge to `main` triggers: @@ -75,9 +75,11 @@ kubectl get application feature-flags-staging -n argocd GitHub variables: `EKS_CLUSTER_NAME`, `STAGING_HEALTH_URL`. -## Production +## Production (Git mirror — off) -Overlay and ArgoCD Application exist in Git. Automatic deploy disabled until [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) is activated. +[`k8s/overlays/production/`](overlays/production/) mirrors staging (blue/green, sync waves, production ECR). Production runtime is **off** and not scheduled — staging is the only working pipeline. No `terraform apply production`, no `application-production.yaml` on cluster. Reference if it's ever turned on: [docs/cd-production-promotion.md](../docs/cd-production-promotion.md). + +When staging overlay changes, update production overlay in the same PR to keep the mirror aligned. ## Troubleshooting diff --git a/k8s/argocd/install-notes.md b/k8s/argocd/install-notes.md index 411879a..eacdbe3 100644 --- a/k8s/argocd/install-notes.md +++ b/k8s/argocd/install-notes.md @@ -120,7 +120,7 @@ kubectl apply -k k8s/overlays/staging --dry-run=server kubectl apply -f k8s/argocd/application-staging.yaml ``` -Do **not** apply `application-production.yaml` until production CD is enabled (end of PLAN). +Do **not** apply `application-production.yaml` — production runtime is off and not scheduled. Staging is the only working pipeline. ### 9. GitHub Actions CD variables @@ -133,6 +133,6 @@ Set repository variables in GitHub → Settings → Variables: Ensure branch protection allows GitHub Actions to push GitOps commits to `main`, or configure `CD_BOT_TOKEN` secret. -## Production (EKS) +## Production (EKS) — off -ArgoCD and app registration deferred until multi-env promotion is enabled. See [docs/cd-production-promotion.md](../../docs/cd-production-promotion.md). +ArgoCD and app registration are **off** and not scheduled; production stays a Git mirror only. Reference if it's ever turned on: [docs/cd-production-promotion.md](../../docs/cd-production-promotion.md). diff --git a/k8s/overlays/production/active-color.yaml b/k8s/overlays/production/active-color.yaml new file mode 100644 index 0000000..30b154c --- /dev/null +++ b/k8s/overlays/production/active-color.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cd-active-color + annotations: + argocd.argoproj.io/sync-wave: "0" +data: + ACTIVE_COLOR: blue diff --git a/k8s/overlays/production/kustomization.yaml b/k8s/overlays/production/kustomization.yaml index 3bd0737..9921e4f 100644 --- a/k8s/overlays/production/kustomization.yaml +++ b/k8s/overlays/production/kustomization.yaml @@ -1,16 +1,30 @@ +# Git mirror of k8s/overlays/staging/ — keep aligned when staging changes. +# Production runtime is OFF (not deployed); staging is the only working pipeline. +# Turning it on is an undecided next step; see docs/cd-production-promotion.md apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: production +commonAnnotations: + argocd.argoproj.io/sync-wave: "0" + resources: - ../../base + - ../../blue-green + - active-color.yaml patches: - - path: patch-deployment.yaml + - path: patch-delete-backend.yaml + - path: patch-delete-hpa.yaml - path: patch-ingress.yaml + - path: patch-blue-deployment.yaml + - path: patch-green-deployment.yaml images: - - name: backend - newName: .dkr.ecr.us-east-1.amazonaws.com/devsecops-production-backend + - name: backend-blue + newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-production-backend + newTag: latest + - name: backend-green + newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-production-backend newTag: latest diff --git a/k8s/overlays/production/patch-deployment.yaml b/k8s/overlays/production/patch-blue-deployment.yaml similarity index 70% rename from k8s/overlays/production/patch-deployment.yaml rename to k8s/overlays/production/patch-blue-deployment.yaml index 57c89ce..28f5c50 100644 --- a/k8s/overlays/production/patch-deployment.yaml +++ b/k8s/overlays/production/patch-blue-deployment.yaml @@ -1,10 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: backend + name: backend-blue spec: replicas: 3 template: + metadata: + annotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true spec: containers: - name: backend @@ -16,4 +20,3 @@ spec: limits: cpu: "1" memory: 1Gi - # securityContext hardening applied in Phase 6 diff --git a/k8s/overlays/production/patch-delete-backend.yaml b/k8s/overlays/production/patch-delete-backend.yaml new file mode 100644 index 0000000..dedb15b --- /dev/null +++ b/k8s/overlays/production/patch-delete-backend.yaml @@ -0,0 +1,5 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +$patch: delete diff --git a/k8s/overlays/production/patch-delete-hpa.yaml b/k8s/overlays/production/patch-delete-hpa.yaml new file mode 100644 index 0000000..4a677fb --- /dev/null +++ b/k8s/overlays/production/patch-delete-hpa.yaml @@ -0,0 +1,5 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: backend-hpa +$patch: delete diff --git a/k8s/overlays/production/patch-green-deployment.yaml b/k8s/overlays/production/patch-green-deployment.yaml new file mode 100644 index 0000000..25fac79 --- /dev/null +++ b/k8s/overlays/production/patch-green-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend-green +spec: + replicas: 3 + template: + metadata: + annotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true + spec: + containers: + - name: backend + imagePullPolicy: Always + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi From 001f189d2023e037f2a207062471788427ced7a3 Mon Sep 17 00:00:00 2001 From: mck21 Date: Thu, 18 Jun 2026 16:57:04 +0200 Subject: [PATCH 2/3] feat: add phase 6 security hardening (kyverno, cosign, netpol, rbac, eso) - securityContext hardening on backend deployments (non-root, read-only rootfs + /tmp, drop ALL caps, seccomp, no SA token automount) - Pod Security Standards labels (baseline enforce, restricted audit) - k8s/security: least-privilege RBAC + default-deny NetworkPolicies - k8s/kyverno: 6 ClusterPolicies incl. Cosign image verification - k8s/external-secrets: ESO + AWS Secrets Manager (staging/prod) - ci.yaml: keyless Cosign signing of pushed images - ArgoCD apps for security baseline + kyverno policies - add .yamllint config referenced by CI Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yaml | 22 +- .yamllint | 20 ++ k8s/argocd/application-kyverno-policies.yaml | 24 ++ k8s/argocd/application-platform-security.yaml | 24 ++ k8s/base/cert-manager/cluster-issuer-dev.yaml | 2 +- k8s/base/deployment.yaml | 20 ++ k8s/blue-green/deployment-blue.yaml | 20 ++ k8s/blue-green/deployment-green.yaml | 20 ++ k8s/external-secrets/install-notes.md | 40 +++ k8s/external-secrets/production.yaml | 49 ++++ k8s/external-secrets/staging.yaml | 51 ++++ k8s/kyverno/install-notes.md | 54 +++++ k8s/kyverno/policies/disallow-latest-tag.yaml | 46 ++++ k8s/kyverno/policies/disallow-privileged.yaml | 34 +++ k8s/kyverno/policies/kustomization.yaml | 10 + k8s/kyverno/policies/require-non-root.yaml | 54 +++++ .../policies/require-resource-limits.yaml | 34 +++ .../policies/require-securitycontext.yaml | 65 +++++ .../policies/verify-images-cosign.yaml | 38 +++ k8s/namespaces/namespaces.yaml | 13 +- k8s/security/kustomization.yaml | 8 + k8s/security/network-policies.yaml | 228 ++++++++++++++++++ k8s/security/rbac.yaml | 129 ++++++++++ 23 files changed, 992 insertions(+), 13 deletions(-) create mode 100644 .yamllint create mode 100644 k8s/argocd/application-kyverno-policies.yaml create mode 100644 k8s/argocd/application-platform-security.yaml create mode 100644 k8s/external-secrets/install-notes.md create mode 100644 k8s/external-secrets/production.yaml create mode 100644 k8s/external-secrets/staging.yaml create mode 100644 k8s/kyverno/install-notes.md create mode 100644 k8s/kyverno/policies/disallow-latest-tag.yaml create mode 100644 k8s/kyverno/policies/disallow-privileged.yaml create mode 100644 k8s/kyverno/policies/kustomization.yaml create mode 100644 k8s/kyverno/policies/require-non-root.yaml create mode 100644 k8s/kyverno/policies/require-resource-limits.yaml create mode 100644 k8s/kyverno/policies/require-securitycontext.yaml create mode 100644 k8s/kyverno/policies/verify-images-cosign.yaml create mode 100644 k8s/security/kustomization.yaml create mode 100644 k8s/security/network-policies.yaml create mode 100644 k8s/security/rbac.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 579eedf..aeb4f98 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -137,6 +137,7 @@ jobs: echo "sha_tag=sha-${{ github.sha }}" >> "$GITHUB_OUTPUT" - name: Push to staging ECR + id: push uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: app/backend @@ -147,14 +148,13 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Push to production ECR - if: github.ref == 'refs/heads/main' && vars.ENABLE_PROD_ECR == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 - with: - context: app/backend - push: true - tags: | - ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:${{ steps.tags.outputs.sha_tag }} - ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:latest - cache-from: type=gha - cache-to: type=gha,mode=max + # Supply-chain: keyless Cosign signature (Sigstore/Fulcio via GitHub OIDC). + # Kyverno verify-image-signatures admits only images signed here. (Phase 6) + - name: Install Cosign + uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2 + + - name: Sign image (keyless) + env: + COSIGN_YES: "true" + run: | + cosign sign "${{ steps.tags.outputs.staging }}@${{ steps.push.outputs.digest }}" diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..60d8457 --- /dev/null +++ b/.yamllint @@ -0,0 +1,20 @@ +# Relaxed yamllint config for Kubernetes manifests and GitHub workflows. +# Kustomize/Helm-style YAML uses long lines and no document-start markers. +extends: relaxed + +rules: + line-length: disable + document-start: disable + comments: + min-spaces-from-content: 1 + comments-indentation: disable + truthy: + allowed-values: ['true', 'false', 'on'] + check-keys: false + indentation: + spaces: 2 + indent-sequences: consistent + empty-lines: + max: 2 + trailing-spaces: enable + new-line-at-end-of-file: enable diff --git a/k8s/argocd/application-kyverno-policies.yaml b/k8s/argocd/application-kyverno-policies.yaml new file mode 100644 index 0000000..953be7d --- /dev/null +++ b/k8s/argocd/application-kyverno-policies.yaml @@ -0,0 +1,24 @@ +# Kyverno ClusterPolicies. (Phase 6) +# Requires Kyverno installed via Helm first — see k8s/kyverno/install-notes.md. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kyverno-policies + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/mck21/devsecops-platform.git + targetRevision: main + path: k8s/kyverno/policies + destination: + server: https://kubernetes.default.svc + namespace: policy-system + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=false diff --git a/k8s/argocd/application-platform-security.yaml b/k8s/argocd/application-platform-security.yaml new file mode 100644 index 0000000..0baa970 --- /dev/null +++ b/k8s/argocd/application-platform-security.yaml @@ -0,0 +1,24 @@ +# Cluster-scoped security baseline: RBAC + NetworkPolicies. (Phase 6) +# Apply on each cluster's ArgoCD. Runtime is off for production during showcase. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: platform-security + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/mck21/devsecops-platform.git + targetRevision: main + path: k8s/security + destination: + server: https://kubernetes.default.svc + namespace: default + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=false diff --git a/k8s/base/cert-manager/cluster-issuer-dev.yaml b/k8s/base/cert-manager/cluster-issuer-dev.yaml index 7820a9b..56366fa 100644 --- a/k8s/base/cert-manager/cluster-issuer-dev.yaml +++ b/k8s/base/cert-manager/cluster-issuer-dev.yaml @@ -3,4 +3,4 @@ kind: ClusterIssuer metadata: name: selfsigned-issuer spec: - selfSigned: {} \ No newline at end of file + selfSigned: {} diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index 1fed506..d4cc7e8 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -18,10 +18,24 @@ spec: holdApplicationUntilProxyStarts: true spec: serviceAccountName: backend-sa + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault containers: - name: backend image: backend:latest imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL ports: - containerPort: 3000 envFrom: @@ -52,3 +66,9 @@ spec: limits: cpu: 500m memory: 512Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/k8s/blue-green/deployment-blue.yaml b/k8s/blue-green/deployment-blue.yaml index c57980e..91ca913 100644 --- a/k8s/blue-green/deployment-blue.yaml +++ b/k8s/blue-green/deployment-blue.yaml @@ -21,10 +21,24 @@ spec: version: blue spec: serviceAccountName: backend-sa + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault containers: - name: backend image: backend-blue:latest imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL ports: - containerPort: 3000 envFrom: @@ -55,3 +69,9 @@ spec: limits: cpu: 500m memory: 512Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/k8s/blue-green/deployment-green.yaml b/k8s/blue-green/deployment-green.yaml index 3642804..2795bd0 100644 --- a/k8s/blue-green/deployment-green.yaml +++ b/k8s/blue-green/deployment-green.yaml @@ -20,10 +20,24 @@ spec: version: green spec: serviceAccountName: backend-sa + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault containers: - name: backend image: backend-green:latest imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL ports: - containerPort: 3000 envFrom: @@ -54,3 +68,9 @@ spec: limits: cpu: 500m memory: 512Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/k8s/external-secrets/install-notes.md b/k8s/external-secrets/install-notes.md new file mode 100644 index 0000000..219b893 --- /dev/null +++ b/k8s/external-secrets/install-notes.md @@ -0,0 +1,40 @@ +# External Secrets Operator (ESO) — install & notes + +On EKS (staging/production), `backend-secrets` is sourced from **AWS Secrets +Manager** via ESO instead of the committed `secret.yaml` template (which is for +Minikube/dev only). + +## Install (Helm) + +```bash +helm repo add external-secrets https://charts.external-secrets.io +helm repo update + +helm install external-secrets external-secrets/external-secrets \ + --namespace external-secrets --create-namespace \ + --set installCRDs=true +``` + +## Prerequisites (Terraform) + +- IRSA roles `mck21-devsecops--external-secrets` with `secretsmanager:GetSecretValue` + scoped to `arn:aws:secretsmanager:us-east-1:125156866917:secret:mck21-devsecops//*`. +- Secret `mck21-devsecops//backend` with JSON keys `DATABASE_URL`, `REDIS_URL`. + +## Apply + +```bash +kubectl apply -f k8s/external-secrets/staging.yaml # staging +kubectl apply -f k8s/external-secrets/production.yaml # only if prod runtime is enabled +kubectl get externalsecret -n staging +``` + +ESO reconciles the `ExternalSecret` into a native `backend-secrets` Secret that +the deployment already references via `secretKeyRef`. When the AWS secret rotates, +ESO re-syncs within `refreshInterval` (1h) — see the secret-rotation resilience +test in [../../docs/resilience-testing.md](../../docs/resilience-testing.md). + +## Dev (Minikube) + +Dev keeps using the local `k8s/base/secret.yaml` template — no AWS dependency. +Do **not** apply the ESO manifests against Minikube. diff --git a/k8s/external-secrets/production.yaml b/k8s/external-secrets/production.yaml new file mode 100644 index 0000000..02d811e --- /dev/null +++ b/k8s/external-secrets/production.yaml @@ -0,0 +1,49 @@ +# External Secrets Operator — production. +# Pulls backend credentials from AWS Secrets Manager into `backend-secrets`. +# Runtime is OFF during showcase; this is the Git mirror of the staging setup. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-secrets-sa + namespace: production + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::125156866917:role/mck21-devsecops-production-external-secrets +--- +apiVersion: external-secrets.io/v1beta1 +kind: SecretStore +metadata: + name: aws-secrets-manager + namespace: production +spec: + provider: + aws: + service: SecretsManager + region: us-east-1 + auth: + jwt: + serviceAccountRef: + name: external-secrets-sa +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: backend-secrets + namespace: production +spec: + refreshInterval: 1h + secretStoreRef: + name: aws-secrets-manager + kind: SecretStore + target: + name: backend-secrets + creationPolicy: Owner + data: + - secretKey: DATABASE_URL + remoteRef: + key: mck21-devsecops/production/backend + property: DATABASE_URL + - secretKey: REDIS_URL + remoteRef: + key: mck21-devsecops/production/backend + property: REDIS_URL diff --git a/k8s/external-secrets/staging.yaml b/k8s/external-secrets/staging.yaml new file mode 100644 index 0000000..485ca75 --- /dev/null +++ b/k8s/external-secrets/staging.yaml @@ -0,0 +1,51 @@ +# External Secrets Operator — staging. +# Pulls backend credentials from AWS Secrets Manager into the `backend-secrets` +# Kubernetes Secret consumed by the deployment. Auth via IRSA (no static keys). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-secrets-sa + namespace: staging + annotations: + # IRSA role created by Terraform (external-secrets module); read-only on the + # mck21-devsecops/staging/* secret path. + eks.amazonaws.com/role-arn: arn:aws:iam::125156866917:role/mck21-devsecops-staging-external-secrets +--- +apiVersion: external-secrets.io/v1beta1 +kind: SecretStore +metadata: + name: aws-secrets-manager + namespace: staging +spec: + provider: + aws: + service: SecretsManager + region: us-east-1 + auth: + jwt: + serviceAccountRef: + name: external-secrets-sa +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: backend-secrets + namespace: staging +spec: + refreshInterval: 1h + secretStoreRef: + name: aws-secrets-manager + kind: SecretStore + target: + name: backend-secrets + creationPolicy: Owner + data: + - secretKey: DATABASE_URL + remoteRef: + key: mck21-devsecops/staging/backend + property: DATABASE_URL + - secretKey: REDIS_URL + remoteRef: + key: mck21-devsecops/staging/backend + property: REDIS_URL diff --git a/k8s/kyverno/install-notes.md b/k8s/kyverno/install-notes.md new file mode 100644 index 0000000..8e216fe --- /dev/null +++ b/k8s/kyverno/install-notes.md @@ -0,0 +1,54 @@ +# Kyverno — install & policy notes + +Policy-as-code admission control for both clusters. Kyverno runs in the +`policy-system` namespace (already created in `k8s/namespaces/namespaces.yaml`). + +## Install (Helm) + +```bash +helm repo add kyverno https://kyverno.github.io/kyverno +helm repo update + +helm install kyverno kyverno/kyverno \ + --namespace policy-system \ + --create-namespace \ + --set admissionController.replicas=1 # 3 on production for HA +``` + +## Apply policies + +```bash +kubectl apply -k k8s/kyverno/policies +kubectl get clusterpolicy +``` + +## Policies + +| Policy | Effect | Scope | +|--------|--------|-------| +| `disallow-privileged-containers` | Enforce | dev, staging, production | +| `require-resource-limits` | Enforce | dev, staging, production | +| `require-run-as-non-root` | Enforce (staging/prod), Audit (dev) | app namespaces | +| `disallow-latest-tag` | Enforce | staging, production | +| `require-securitycontext` | Enforce (prod), Audit (dev/staging) | `app: backend` pods | +| `verify-image-signatures` | Enforce | staging, production | + +> **Audit vs Enforce:** dev runs the stricter checks in `Audit` mode so local +> iteration is never blocked; production enforces. See policy `failureAction` +> overrides per rule. + +## Image signature verification + +`verify-image-signatures` uses **keyless** Cosign verification: CI signs the +image through GitHub Actions OIDC (Sigstore/Fulcio), and Kyverno checks the +signature + Rekor transparency log entry before admitting the pod. Update the +`subject`/`issuer` if the workflow path or repo changes. See +[../../docs/security.md](../../docs/security.md). + +## Demo — policy blocking a bad pod + +```bash +kubectl run bad --image=nginx:latest -n staging # blocked: latest tag + no limits +kubectl run priv --image=nginx:1.27 -n staging \ + --overrides='{"spec":{"containers":[{"name":"priv","image":"nginx:1.27","securityContext":{"privileged":true}}]}}' +``` diff --git a/k8s/kyverno/policies/disallow-latest-tag.yaml b/k8s/kyverno/policies/disallow-latest-tag.yaml new file mode 100644 index 0000000..57cfbd8 --- /dev/null +++ b/k8s/kyverno/policies/disallow-latest-tag.yaml @@ -0,0 +1,46 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: disallow-latest-tag + annotations: + policies.kyverno.io/title: Disallow Latest Image Tag + policies.kyverno.io/category: Supply Chain + policies.kyverno.io/severity: medium + policies.kyverno.io/description: >- + Images must use an explicit, immutable tag (e.g. sha-), never + :latest or an untagged reference. Enforced on staging/production where the + GitOps pipeline always pins a sha- tag. +spec: + validationFailureAction: Enforce + background: true + rules: + - name: require-image-tag + match: + any: + - resources: + kinds: + - Pod + namespaces: + - staging + - production + validate: + message: "An explicit image tag is required." + pattern: + spec: + containers: + - image: "*:*" + - name: disallow-latest-tag + match: + any: + - resources: + kinds: + - Pod + namespaces: + - staging + - production + validate: + message: "Using the :latest tag is not allowed; pin an immutable sha- tag." + pattern: + spec: + containers: + - image: "!*:latest" diff --git a/k8s/kyverno/policies/disallow-privileged.yaml b/k8s/kyverno/policies/disallow-privileged.yaml new file mode 100644 index 0000000..a075421 --- /dev/null +++ b/k8s/kyverno/policies/disallow-privileged.yaml @@ -0,0 +1,34 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: disallow-privileged-containers + annotations: + policies.kyverno.io/title: Disallow Privileged Containers + policies.kyverno.io/category: Pod Security Standards (Baseline) + policies.kyverno.io/severity: high + policies.kyverno.io/description: >- + Privileged containers share the host namespaces and devices. Block them on + app namespaces. +spec: + validationFailureAction: Enforce + background: true + rules: + - name: privileged-containers + match: + any: + - resources: + kinds: + - Pod + namespaces: + - dev + - staging + - production + validate: + message: "Privileged containers are not allowed." + pattern: + spec: + =(securityContext): + =(privileged): "false" + containers: + - =(securityContext): + =(privileged): "false" diff --git a/k8s/kyverno/policies/kustomization.yaml b/k8s/kyverno/policies/kustomization.yaml new file mode 100644 index 0000000..82d56e9 --- /dev/null +++ b/k8s/kyverno/policies/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - disallow-privileged.yaml + - require-resource-limits.yaml + - require-non-root.yaml + - disallow-latest-tag.yaml + - require-securitycontext.yaml + - verify-images-cosign.yaml diff --git a/k8s/kyverno/policies/require-non-root.yaml b/k8s/kyverno/policies/require-non-root.yaml new file mode 100644 index 0000000..8323b20 --- /dev/null +++ b/k8s/kyverno/policies/require-non-root.yaml @@ -0,0 +1,54 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: require-run-as-non-root + annotations: + policies.kyverno.io/title: Require runAsNonRoot + policies.kyverno.io/category: Pod Security Standards (Restricted) + policies.kyverno.io/severity: high + policies.kyverno.io/description: >- + Containers must not run as the root user. Enforced on staging/production, + audited on dev to allow iteration. +spec: + validationFailureAction: Enforce + background: true + rules: + - name: run-as-non-root-prod + match: + any: + - resources: + kinds: + - Pod + namespaces: + - staging + - production + validate: + message: "Containers must set runAsNonRoot: true (pod or container level)." + anyPattern: + - spec: + securityContext: + runAsNonRoot: "true" + - spec: + containers: + - securityContext: + runAsNonRoot: "true" + - name: run-as-non-root-dev-audit + match: + any: + - resources: + kinds: + - Pod + namespaces: + - dev + validate: + # Audit on dev: report violations without blocking. + failureAction: Audit + message: "Containers should set runAsNonRoot: true." + anyPattern: + - spec: + securityContext: + runAsNonRoot: "true" + - spec: + containers: + - securityContext: + runAsNonRoot: "true" diff --git a/k8s/kyverno/policies/require-resource-limits.yaml b/k8s/kyverno/policies/require-resource-limits.yaml new file mode 100644 index 0000000..3e4c15d --- /dev/null +++ b/k8s/kyverno/policies/require-resource-limits.yaml @@ -0,0 +1,34 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: require-resource-limits + annotations: + policies.kyverno.io/title: Require Resource Limits + policies.kyverno.io/category: Resource Management + policies.kyverno.io/severity: medium + policies.kyverno.io/description: >- + Every container must declare CPU and memory limits so the scheduler can + bin-pack and the LimitRange/ResourceQuota stay meaningful. +spec: + validationFailureAction: Enforce + background: true + rules: + - name: validate-resources + match: + any: + - resources: + kinds: + - Pod + namespaces: + - dev + - staging + - production + validate: + message: "CPU and memory limits are required on every container." + pattern: + spec: + containers: + - resources: + limits: + memory: "?*" + cpu: "?*" diff --git a/k8s/kyverno/policies/require-securitycontext.yaml b/k8s/kyverno/policies/require-securitycontext.yaml new file mode 100644 index 0000000..ab59729 --- /dev/null +++ b/k8s/kyverno/policies/require-securitycontext.yaml @@ -0,0 +1,65 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: require-securitycontext + annotations: + policies.kyverno.io/title: Require Hardened securityContext + policies.kyverno.io/category: Pod Security Standards (Restricted) + policies.kyverno.io/severity: medium + policies.kyverno.io/description: >- + App containers must drop ALL capabilities, disable privilege escalation and + use a read-only root filesystem. Audit on dev, Enforce on production. +spec: + validationFailureAction: Audit + background: true + rules: + - name: securitycontext-prod + match: + any: + - resources: + kinds: + - Pod + namespaces: + - production + selector: + matchLabels: + app: backend + validate: + failureAction: Enforce + message: >- + backend containers must set allowPrivilegeEscalation:false, + readOnlyRootFilesystem:true and drop ALL capabilities. + pattern: + spec: + containers: + - securityContext: + allowPrivilegeEscalation: "false" + readOnlyRootFilesystem: "true" + capabilities: + drop: + - ALL + - name: securitycontext-dev-staging-audit + match: + any: + - resources: + kinds: + - Pod + namespaces: + - dev + - staging + selector: + matchLabels: + app: backend + validate: + message: >- + backend containers should set allowPrivilegeEscalation:false, + readOnlyRootFilesystem:true and drop ALL capabilities. + pattern: + spec: + containers: + - securityContext: + allowPrivilegeEscalation: "false" + readOnlyRootFilesystem: "true" + capabilities: + drop: + - ALL diff --git a/k8s/kyverno/policies/verify-images-cosign.yaml b/k8s/kyverno/policies/verify-images-cosign.yaml new file mode 100644 index 0000000..1a40c8a --- /dev/null +++ b/k8s/kyverno/policies/verify-images-cosign.yaml @@ -0,0 +1,38 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: verify-image-signatures + annotations: + policies.kyverno.io/title: Verify Image Signatures (Cosign) + policies.kyverno.io/category: Supply Chain + policies.kyverno.io/severity: high + policies.kyverno.io/description: >- + Only allow backend images that carry a valid Cosign signature. The CI + pipeline signs images keylessly (Sigstore/Fulcio OIDC) after pushing to + ECR; this policy verifies the signature before the pod is admitted. +spec: + validationFailureAction: Enforce + background: false + webhookTimeoutSeconds: 30 + rules: + - name: verify-backend-signature + match: + any: + - resources: + kinds: + - Pod + namespaces: + - staging + - production + verifyImages: + - imageReferences: + - "125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-*-backend:*" + attestors: + - count: 1 + entries: + # Keyless verification: image signed via GitHub Actions OIDC. + - keyless: + subject: "https://github.com/mck21/devsecops-platform/.github/workflows/ci.yaml@refs/heads/main" + issuer: "https://token.actions.githubusercontent.com" + rekor: + url: https://rekor.sigstore.dev diff --git a/k8s/namespaces/namespaces.yaml b/k8s/namespaces/namespaces.yaml index e8f7c31..e23ef92 100644 --- a/k8s/namespaces/namespaces.yaml +++ b/k8s/namespaces/namespaces.yaml @@ -5,6 +5,11 @@ metadata: labels: istio-injection: enabled environment: dev + # Pod Security Standards: enforce baseline (postgres/redis run in-namespace), + # audit/warn against restricted so app workloads are held to the higher bar. + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted --- apiVersion: v1 kind: Namespace @@ -37,6 +42,9 @@ metadata: labels: istio-injection: enabled environment: staging + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted --- apiVersion: v1 kind: Namespace @@ -44,4 +52,7 @@ metadata: name: production labels: istio-injection: enabled - environment: production \ No newline at end of file + environment: production + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted diff --git a/k8s/security/kustomization.yaml b/k8s/security/kustomization.yaml new file mode 100644 index 0000000..8e4c8fd --- /dev/null +++ b/k8s/security/kustomization.yaml @@ -0,0 +1,8 @@ +# RBAC + NetworkPolicies. Applied cluster-wide (not per-overlay) via the +# platform-security ArgoCD Application or `kubectl apply -k k8s/security`. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rbac.yaml + - network-policies.yaml diff --git a/k8s/security/network-policies.yaml b/k8s/security/network-policies.yaml new file mode 100644 index 0000000..a30cc5d --- /dev/null +++ b/k8s/security/network-policies.yaml @@ -0,0 +1,228 @@ +# NetworkPolicies — default-deny ingress + explicit allows per app namespace. +# Applied to staging and production (EKS). Dev (Minikube) gets default-deny only; +# the Minikube CNI (kindnet/bridge) does not always enforce policy, so these are +# primarily a production-intent artifact validated on EKS (Calico/VPC CNI). +# +# Istio note: the sidecar proxies traffic, but NetworkPolicy still applies at L3/L4. +# DNS (53) and the Istio control plane (15012) egress must be allowed explicitly. +--- +# ---------- staging ---------- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-ingress + namespace: staging +spec: + podSelector: {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns-and-istio-egress + namespace: staging +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: {} + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Istiod (xDS) + webhook + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: istio-system + ports: + - protocol: TCP + port: 15012 + - protocol: TCP + port: 15010 + # In-namespace traffic (backend <-> postgres/redis) + - to: + - podSelector: {} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-backend-ingress + namespace: staging +spec: + podSelector: + matchLabels: + app: backend + policyTypes: + - Ingress + ingress: + # From the ingress controller + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - protocol: TCP + port: 3000 + # From within the namespace (istio-ingressgateway sidecar mesh, health) + - from: + - podSelector: {} + ports: + - protocol: TCP + port: 3000 + # Prometheus scrape of /metrics + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - protocol: TCP + port: 3000 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-data-ingress + namespace: staging +spec: + podSelector: + matchExpressions: + - key: app + operator: In + values: ["postgres", "redis"] + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: backend + ports: + - protocol: TCP + port: 5432 + - protocol: TCP + port: 6379 +--- +# ---------- production ---------- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-ingress + namespace: production +spec: + podSelector: {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns-and-istio-egress + namespace: production +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: {} + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: istio-system + ports: + - protocol: TCP + port: 15012 + - protocol: TCP + port: 15010 + - to: + - podSelector: {} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-backend-ingress + namespace: production +spec: + podSelector: + matchLabels: + app: backend + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - protocol: TCP + port: 3000 + - from: + - podSelector: {} + ports: + - protocol: TCP + port: 3000 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - protocol: TCP + port: 3000 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-data-ingress + namespace: production +spec: + podSelector: + matchExpressions: + - key: app + operator: In + values: ["postgres", "redis"] + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: backend + ports: + - protocol: TCP + port: 5432 + - protocol: TCP + port: 6379 +--- +# ---------- dev (default-deny only; CNI enforcement varies on Minikube) ---------- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-ingress + namespace: dev +spec: + podSelector: {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-same-namespace + namespace: dev +spec: + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: {} diff --git a/k8s/security/rbac.yaml b/k8s/security/rbac.yaml new file mode 100644 index 0000000..3f5cb34 --- /dev/null +++ b/k8s/security/rbac.yaml @@ -0,0 +1,129 @@ +# RBAC — least-privilege roles per namespace plus a cluster-wide read-only role. +# Subjects use groups (e.g. mapped from AWS IAM via aws-auth / EKS access entries, +# or from an OIDC provider). No real identities are committed. +--- +# Cluster-wide read-only access for on-call / auditors. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: platform-readonly + labels: + app.kubernetes.io/part-of: devsecops-platform +rules: + - apiGroups: ["", "apps", "batch", "networking.k8s.io", "autoscaling"] + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.istio.io", "security.istio.io"] + resources: ["*"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: platform-readonly + labels: + app.kubernetes.io/part-of: devsecops-platform +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: platform-readonly +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: devsecops:auditors +--- +# Developer role: manage app workloads in a namespace, read-only on RBAC/secrets. +# Re-applied per app namespace below (dev / staging / production). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: app-developer + namespace: dev + labels: + app.kubernetes.io/part-of: devsecops-platform +rules: + - apiGroups: ["", "apps", "autoscaling"] + resources: ["deployments", "replicasets", "pods", "pods/log", "services", "configmaps", "horizontalpodautoscalers"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["pods/exec", "pods/portforward"] + verbs: ["create"] + - apiGroups: ["networking.istio.io"] + resources: ["virtualservices", "destinationrules"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: app-developer + namespace: dev +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: app-developer +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: devsecops:developers +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: app-developer + namespace: staging + labels: + app.kubernetes.io/part-of: devsecops-platform +rules: + - apiGroups: ["", "apps", "autoscaling"] + resources: ["deployments", "replicasets", "pods", "pods/log", "services", "configmaps", "horizontalpodautoscalers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.istio.io"] + resources: ["virtualservices", "destinationrules"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: app-developer + namespace: staging +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: app-developer +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: devsecops:developers +--- +# Production: developers are read-only. Mutations go through GitOps (ArgoCD) only. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: app-developer + namespace: production + labels: + app.kubernetes.io/part-of: devsecops-platform +rules: + - apiGroups: ["", "apps", "autoscaling"] + resources: ["deployments", "replicasets", "pods", "pods/log", "services", "configmaps", "horizontalpodautoscalers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.istio.io"] + resources: ["virtualservices", "destinationrules"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: app-developer + namespace: production +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: app-developer +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: devsecops:developers From 986736da22be423452f66e2ddadcc7796f2ef773 Mon Sep 17 00:00:00 2001 From: mck21 Date: Thu, 18 Jun 2026 16:57:27 +0200 Subject: [PATCH 3/3] feat: add phase 7 monitoring and SRE stack - kube-prometheus-stack values (dev/eks), Loki + Promtail values - ServiceMonitor for app /metrics; SLO PrometheusRule with multi-window burn-rate alerts based on Istio mesh telemetry - custom Grafana dashboards (Feature Flag Service, SLO error budget) - docs/sre.md: SLIs, SLOs, error-budget policy - ArgoCD app for monitoring config Co-Authored-By: Claude Opus 4.8 --- docs/sre.md | 62 +++++++++ k8s/argocd/application-monitoring.yaml | 24 ++++ monitoring/README.md | 67 ++++++++++ .../grafana/dashboards/app-feature-flags.json | 92 +++++++++++++ .../grafana/dashboards/kustomization.yaml | 19 +++ .../grafana/dashboards/slo-error-budget.json | 98 ++++++++++++++ monitoring/kustomization.yaml | 10 ++ monitoring/loki/values-dev.yaml | 38 ++++++ monitoring/loki/values-eks.yaml | 38 ++++++ monitoring/prometheus/prometheusrules.yaml | 123 ++++++++++++++++++ .../prometheus/servicemonitor-backend.yaml | 24 ++++ monitoring/prometheus/values-dev.yaml | 51 ++++++++ monitoring/prometheus/values-eks.yaml | 102 +++++++++++++++ monitoring/promtail/values.yaml | 28 ++++ 14 files changed, 776 insertions(+) create mode 100644 docs/sre.md create mode 100644 k8s/argocd/application-monitoring.yaml create mode 100644 monitoring/README.md create mode 100644 monitoring/grafana/dashboards/app-feature-flags.json create mode 100644 monitoring/grafana/dashboards/kustomization.yaml create mode 100644 monitoring/grafana/dashboards/slo-error-budget.json create mode 100644 monitoring/kustomization.yaml create mode 100644 monitoring/loki/values-dev.yaml create mode 100644 monitoring/loki/values-eks.yaml create mode 100644 monitoring/prometheus/prometheusrules.yaml create mode 100644 monitoring/prometheus/servicemonitor-backend.yaml create mode 100644 monitoring/prometheus/values-dev.yaml create mode 100644 monitoring/prometheus/values-eks.yaml create mode 100644 monitoring/promtail/values.yaml diff --git a/docs/sre.md b/docs/sre.md new file mode 100644 index 0000000..f7b3241 --- /dev/null +++ b/docs/sre.md @@ -0,0 +1,62 @@ +# SRE — SLIs, SLOs & Error Budget Policy + +Reliability targets for the Feature Flag Service API. Measured from Istio mesh +telemetry (`reporter="destination"`) so they reflect the client experience. + +## Service Level Indicators (SLIs) + +| SLI | Definition | Source metric | +|-----|------------|---------------| +| Availability | % of requests returning non-5xx | `istio_requests_total` (response_code) | +| Latency | P95 of `GET /api/flags/{key}` (Redis-cached read) | `istio_request_duration_milliseconds_bucket` | +| Error rate | % of requests returning 5xx | `istio_requests_total` (response_code=~"5..") | + +## Service Level Objectives (SLOs) + +| SLO | Target | Window | +|-----|--------|--------| +| API availability | 99.9% | 30 days rolling | +| P95 latency | ≤ 300 ms | 30 days rolling | +| Error rate | < 0.1% | 30 days rolling | + +**Error budget:** 99.9% availability ⇒ **0.1%** of requests may fail per 30-day +window (~43 min of full downtime equivalent). + +## Error Budget Policy + +- **Budget healthy (>50% remaining):** normal feature velocity; deploys flow + through CI/CD freely. +- **Budget low (<50% remaining):** prioritise reliability work; non-critical + risky changes held. +- **Budget exhausted:** change freeze except reliability/security fixes until the + budget recovers over the rolling window. + +## Burn-rate alerting (multi-window, multi-burn-rate) + +Implemented in [../monitoring/prometheus/prometheusrules.yaml](../monitoring/prometheus/prometheusrules.yaml). + +| Alert | Condition | Severity | Action | +|-------|-----------|----------|--------| +| `BackendErrorBudgetFastBurn` | 14.4× budget over 1h **and** 5m | critical | Page on-call | +| `BackendErrorBudgetSlowBurn` | 6× budget over 6h **and** 30m | warning | Create ticket | +| `BackendHighErrorRate` | 5xx ratio > 5% for 5m | critical | Page | +| `BackendHighLatencyP95` | P95 > 300 ms for 10m | warning | Investigate | +| `BackendPodCrashLooping` | >3 restarts / 15m | critical | Page | +| `NodeMemoryHigh` | node mem > 85% for 10m | warning | Investigate | + +Why two burn rates: a **fast burn** (14.4×) catches acute outages within minutes; +a **slow burn** (6×) catches gradual degradation before it silently drains the +monthly budget. Each alert requires a long **and** a short window to fire, which +suppresses flapping from brief spikes. + +## Dashboards + +- **SLO — Error Budget** (`slo-error-budget`): availability vs target, budget + remaining gauge, current burn rate, multi-window error ratio. +- **Feature Flag Service** (`feature-flags-app`): request rate, error ratio, P95, + cache hit ratio, HPA replicas. + +## Reviewing the budget + +Monthly SRE review: read the SLO dashboard, classify any incidents, and decide +feature-vs-reliability priority for the next period per the policy above. diff --git a/k8s/argocd/application-monitoring.yaml b/k8s/argocd/application-monitoring.yaml new file mode 100644 index 0000000..e429b1c --- /dev/null +++ b/k8s/argocd/application-monitoring.yaml @@ -0,0 +1,24 @@ +# App ServiceMonitor + SLO PrometheusRule + custom Grafana dashboards. (Phase 7) +# Requires kube-prometheus-stack CRDs installed via Helm first (see monitoring/README.md). +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: monitoring-config + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://github.com/mck21/devsecops-platform.git + targetRevision: main + path: monitoring + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=false diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..f894b82 --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,67 @@ +# Monitoring & Observability (Phase 7) + +Prometheus + Grafana + Loki + Promtail, plus app `/metrics` scraping, SLO rules, +and curated dashboards. Works on Minikube (dev) and EKS (staging/production). + +## Stack + +| Component | Source | Dev | EKS | +|-----------|--------|-----|-----| +| Prometheus + Alertmanager + Grafana | `kube-prometheus-stack` (Helm) | 1 replica, ephemeral | HA, gp3 PVCs | +| Loki | `grafana/loki` (Helm) | SingleBinary, filesystem | SimpleScalable, S3 | +| Promtail | `grafana/promtail` (Helm) | DaemonSet | DaemonSet | +| Jaeger | via Istio addon (Phase 2) | demo | production | + +## Install order + +```bash +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update + +# 1. Metrics + dashboards (pick the right values file) +helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ + -n monitoring --create-namespace \ + -f monitoring/prometheus/values-eks.yaml # or values-dev.yaml + +# 2. Logs +helm install loki grafana/loki -n monitoring -f monitoring/loki/values-eks.yaml # or values-dev.yaml +helm install promtail grafana/promtail -n monitoring -f monitoring/promtail/values.yaml + +# 3. App ServiceMonitor + SLO rules + custom dashboards +kubectl apply -k monitoring +``` + +## What's scraped + +- **App metrics:** `ServiceMonitor/backend` scrapes `/metrics` (port `http`) in + `dev`, `staging`, `production`. +- **Mesh metrics:** Istio emits `istio_requests_total` / + `istio_request_duration_milliseconds_bucket` — the basis for SLOs. +- **Cluster metrics:** node-exporter + kube-state-metrics (bundled). + +## Dashboards + +| Dashboard | Source | UID | +|-----------|--------|-----| +| Feature Flag Service | custom (`grafana/dashboards/app-feature-flags.json`) | `feature-flags-app` | +| SLO — Error Budget | custom (`grafana/dashboards/slo-error-budget.json`) | `slo-error-budget` | +| Kubernetes Cluster | community gnetId 7249 | — | +| Istio Mesh | community gnetId 7639 | — | +| Node Exporter | community gnetId 1860 | — | + +Custom dashboards are provisioned as `grafana_dashboard`-labelled ConfigMaps and +auto-imported by the Grafana sidecar. + +## SLOs & alerts + +SLIs/SLOs and the error-budget policy live in [../docs/sre.md](../docs/sre.md). +Alerting + multi-window burn-rate rules are in +[prometheus/prometheusrules.yaml](prometheus/prometheusrules.yaml). + +## Access (dev) + +```bash +kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3001:80 +# http://localhost:3001 (admin / admin) +``` diff --git a/monitoring/grafana/dashboards/app-feature-flags.json b/monitoring/grafana/dashboards/app-feature-flags.json new file mode 100644 index 0000000..345ac05 --- /dev/null +++ b/monitoring/grafana/dashboards/app-feature-flags.json @@ -0,0 +1,92 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "graphTooltip": 1, + "title": "Feature Flag Service", + "uid": "feature-flags-app", + "tags": ["devsecops", "application"], + "timezone": "", + "schemaVersion": 39, + "time": { "from": "now-6h", "to": "now" }, + "templating": { + "list": [ + { + "name": "namespace", + "type": "query", + "datasource": "Prometheus", + "query": "label_values(istio_requests_total{destination_workload=\"backend\"}, destination_workload_namespace)", + "refresh": 2, + "includeAll": true + } + ] + }, + "panels": [ + { + "title": "Request rate (rps)", + "type": "timeseries", + "datasource": "Prometheus", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",destination_workload_namespace=~\"$namespace\"}[5m]))", + "legendFormat": "total" + } + ] + }, + { + "title": "5xx error ratio", + "type": "timeseries", + "datasource": "Prometheus", + "fieldConfig": { "defaults": { "unit": "percentunit" } }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[5m])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[5m])), 1)", + "legendFormat": "error ratio" + } + ] + }, + { + "title": "P95 latency (ms) — flag reads", + "type": "timeseries", + "datasource": "Prometheus", + "fieldConfig": { "defaults": { "unit": "ms" } }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=\"backend\"}[5m])) by (le))", + "legendFormat": "p95" + } + ] + }, + { + "title": "Redis cache hit ratio", + "type": "timeseries", + "datasource": "Prometheus", + "fieldConfig": { "defaults": { "unit": "percentunit" } }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "sum(rate(flag_cache_hits_total[5m])) / clamp_min(sum(rate(flag_cache_hits_total[5m])) + sum(rate(flag_cache_misses_total[5m])), 1)", + "legendFormat": "hit ratio" + } + ] + }, + { + "title": "HPA replicas", + "type": "timeseries", + "datasource": "Prometheus", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "kube_horizontalpodautoscaler_status_current_replicas{horizontalpodautoscaler=\"backend\"}", + "legendFormat": "{{namespace}} current" + }, + { + "expr": "kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler=\"backend\"}", + "legendFormat": "{{namespace}} max" + } + ] + } + ] +} diff --git a/monitoring/grafana/dashboards/kustomization.yaml b/monitoring/grafana/dashboards/kustomization.yaml new file mode 100644 index 0000000..80ef833 --- /dev/null +++ b/monitoring/grafana/dashboards/kustomization.yaml @@ -0,0 +1,19 @@ +# Wrap dashboard JSON as ConfigMaps the Grafana sidecar auto-imports +# (label grafana_dashboard). disableNameSuffixHash keeps stable names for GitOps. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +generatorOptions: + disableNameSuffixHash: true + labels: + grafana_dashboard: "1" + +configMapGenerator: + - name: dashboard-feature-flags + files: + - app-feature-flags.json + - name: dashboard-slo-error-budget + files: + - slo-error-budget.json diff --git a/monitoring/grafana/dashboards/slo-error-budget.json b/monitoring/grafana/dashboards/slo-error-budget.json new file mode 100644 index 0000000..f9634a8 --- /dev/null +++ b/monitoring/grafana/dashboards/slo-error-budget.json @@ -0,0 +1,98 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "title": "SLO — Availability & Error Budget", + "uid": "slo-error-budget", + "tags": ["devsecops", "sre", "slo"], + "schemaVersion": 39, + "time": { "from": "now-30d", "to": "now" }, + "panels": [ + { + "title": "Availability (30d) vs SLO 99.9%", + "type": "stat", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 0.999 } + ] + } + } + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "1 - (sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[30d])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[30d])), 1))", + "legendFormat": "availability" + } + ] + }, + { + "title": "Error budget remaining (30d)", + "type": "gauge", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 0.25 }, + { "color": "green", "value": 0.5 } + ] + } + } + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, + "targets": [ + { + "expr": "1 - ((sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\",response_code=~\"5..\"}[30d])) / clamp_min(sum(rate(istio_requests_total{reporter=\"destination\",destination_workload=\"backend\"}[30d])), 1)) / 0.001)", + "legendFormat": "budget remaining" + } + ] + }, + { + "title": "Burn rate (1h)", + "type": "stat", + "datasource": "Prometheus", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 2 }, + { "color": "red", "value": 14.4 } + ] + } + } + }, + "targets": [ + { + "expr": "backend:request_errors:ratio_rate1h / 0.001", + "legendFormat": "burn rate" + } + ] + }, + { + "title": "Error ratio over time (multi-window)", + "type": "timeseries", + "datasource": "Prometheus", + "fieldConfig": { "defaults": { "unit": "percentunit" } }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 }, + "targets": [ + { "expr": "backend:request_errors:ratio_rate5m", "legendFormat": "5m" }, + { "expr": "backend:request_errors:ratio_rate1h", "legendFormat": "1h" }, + { "expr": "backend:request_errors:ratio_rate6h", "legendFormat": "6h" } + ] + } + ] +} diff --git a/monitoring/kustomization.yaml b/monitoring/kustomization.yaml new file mode 100644 index 0000000..b5d55bd --- /dev/null +++ b/monitoring/kustomization.yaml @@ -0,0 +1,10 @@ +# Cluster-side monitoring CRs (ServiceMonitor, PrometheusRule) + Grafana dashboards. +# Requires kube-prometheus-stack (CRDs) installed via Helm first. +# Apply: kubectl apply -k monitoring +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - prometheus/servicemonitor-backend.yaml + - prometheus/prometheusrules.yaml + - grafana/dashboards diff --git a/monitoring/loki/values-dev.yaml b/monitoring/loki/values-dev.yaml new file mode 100644 index 0000000..4a2ef3f --- /dev/null +++ b/monitoring/loki/values-dev.yaml @@ -0,0 +1,38 @@ +# Loki — dev (Minikube). Single binary, filesystem storage, ephemeral. +# helm install loki grafana/loki -n monitoring -f monitoring/loki/values-dev.yaml +deploymentMode: SingleBinary + +loki: + auth_enabled: false + commonConfig: + replication_factor: 1 + storage: + type: filesystem + schemaConfig: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: loki_index_ + period: 24h + limits_config: + retention_period: 48h + +singleBinary: + replicas: 1 + persistence: + enabled: false + +# Disable the scalable components on Minikube. +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 +chunksCache: + enabled: false +resultsCache: + enabled: false diff --git a/monitoring/loki/values-eks.yaml b/monitoring/loki/values-eks.yaml new file mode 100644 index 0000000..78c80d9 --- /dev/null +++ b/monitoring/loki/values-eks.yaml @@ -0,0 +1,38 @@ +# Loki (grafana/loki Helm chart, single-binary scalable) — staging/production. +# helm install loki grafana/loki -n monitoring -f monitoring/loki/values-eks.yaml +deploymentMode: SimpleScalable + +loki: + auth_enabled: false + storage: + type: s3 + bucketNames: + chunks: mck21-devsecops-loki-chunks + ruler: mck21-devsecops-loki-ruler + s3: + region: us-east-1 + schemaConfig: + configs: + - from: "2024-01-01" + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + limits_config: + retention_period: 168h # 7 days + +backend: + replicas: 2 +read: + replicas: 2 +write: + replicas: 2 + +# IRSA for S3 access (Terraform-provisioned role). +serviceAccount: + create: true + name: loki + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::125156866917:role/mck21-devsecops-loki diff --git a/monitoring/prometheus/prometheusrules.yaml b/monitoring/prometheus/prometheusrules.yaml new file mode 100644 index 0000000..cf6dece --- /dev/null +++ b/monitoring/prometheus/prometheusrules.yaml @@ -0,0 +1,123 @@ +# Recording + alerting rules for the Feature Flag Service. +# SLIs are derived from Istio mesh telemetry (reporter="destination") so they +# measure what clients actually experience, independent of app instrumentation. +# +# SLOs (see docs/sre.md): +# Availability 99.9% -> error budget 0.1% +# P95 latency <= 300ms (GET /api/flags/{key}) +# Error rate < 0.1% +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: backend-slo-rules + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + - name: backend.sli.recording + interval: 30s + rules: + # Error ratio (5xx / total) over multiple windows for burn-rate alerting. + - record: backend:request_errors:ratio_rate5m + expr: | + sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[5m])) + / + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[5m])), 1) + - record: backend:request_errors:ratio_rate30m + expr: | + sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[30m])) + / + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[30m])), 1) + - record: backend:request_errors:ratio_rate1h + expr: | + sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[1h])) + / + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[1h])), 1) + - record: backend:request_errors:ratio_rate6h + expr: | + sum(rate(istio_requests_total{reporter="destination",destination_workload="backend",response_code=~"5.."}[6h])) + / + clamp_min(sum(rate(istio_requests_total{reporter="destination",destination_workload="backend"}[6h])), 1) + # Availability SLI (non-5xx ratio). + - record: backend:availability:ratio_rate5m + expr: 1 - backend:request_errors:ratio_rate5m + # P95 latency (ms) for cached flag reads. + - record: backend:request_latency_p95_ms:5m + expr: | + histogram_quantile(0.95, + sum(rate(istio_request_duration_milliseconds_bucket{reporter="destination",destination_workload="backend"}[5m])) by (le)) + + - name: backend.slo.burnrate + rules: + # Fast burn: 14.4x budget over 1h AND 5m windows -> page. + - alert: BackendErrorBudgetFastBurn + expr: | + backend:request_errors:ratio_rate1h > (14.4 * 0.001) + and + backend:request_errors:ratio_rate5m > (14.4 * 0.001) + for: 2m + labels: + severity: critical + slo: availability + annotations: + summary: "Backend error budget fast burn (14.4x)" + description: "5xx ratio is burning the 99.9% availability budget >14.4x over 1h and 5m." + # Slow burn: 6x budget over 6h AND 30m windows -> ticket. + - alert: BackendErrorBudgetSlowBurn + expr: | + backend:request_errors:ratio_rate6h > (6 * 0.001) + and + backend:request_errors:ratio_rate30m > (6 * 0.001) + for: 15m + labels: + severity: warning + slo: availability + annotations: + summary: "Backend error budget slow burn (6x)" + description: "5xx ratio is burning the availability budget >6x over 6h and 30m." + + - name: backend.alerts + rules: + - alert: BackendHighErrorRate + expr: backend:request_errors:ratio_rate5m > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "Backend 5xx error rate > 5%" + description: "Error rate {{ $value | humanizePercentage }} over the last 5m." + - alert: BackendHighLatencyP95 + expr: backend:request_latency_p95_ms:5m > 300 + for: 10m + labels: + severity: warning + annotations: + summary: "Backend P95 latency > 300ms" + description: "P95 latency is {{ $value }}ms (SLO 300ms)." + - alert: BackendPodCrashLooping + expr: | + increase(kube_pod_container_status_restarts_total{namespace=~"dev|staging|production",container="backend"}[15m]) > 3 + for: 5m + labels: + severity: critical + annotations: + summary: "Backend pod crash looping" + description: "Pod {{ $labels.pod }} restarted >3 times in 15m." + - alert: BackendTargetDown + expr: up{job=~".*backend.*"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Backend scrape target down" + description: "Prometheus cannot scrape backend in {{ $labels.namespace }}." + - alert: NodeMemoryHigh + expr: | + (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.85 + for: 10m + labels: + severity: warning + annotations: + summary: "Node memory > 85%" + description: "Node {{ $labels.instance }} memory usage above 85%." diff --git a/monitoring/prometheus/servicemonitor-backend.yaml b/monitoring/prometheus/servicemonitor-backend.yaml new file mode 100644 index 0000000..0bfadfb --- /dev/null +++ b/monitoring/prometheus/servicemonitor-backend.yaml @@ -0,0 +1,24 @@ +# Scrape the Feature Flag API /metrics endpoint (prom-client / @willsoto/nestjs-prometheus). +# The `release` label lets the kube-prometheus-stack Prometheus select it. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: backend + namespace: monitoring + labels: + release: kube-prometheus-stack + app: backend +spec: + selector: + matchLabels: + app: backend + namespaceSelector: + matchNames: + - dev + - staging + - production + endpoints: + - port: http + path: /metrics + interval: 15s + scrapeTimeout: 10s diff --git a/monitoring/prometheus/values-dev.yaml b/monitoring/prometheus/values-dev.yaml new file mode 100644 index 0000000..7710ec2 --- /dev/null +++ b/monitoring/prometheus/values-dev.yaml @@ -0,0 +1,51 @@ +# kube-prometheus-stack values — dev (Minikube). Single replica, ephemeral storage. +# helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ +# -n monitoring -f monitoring/prometheus/values-dev.yaml +fullnameOverride: kube-prometheus-stack + +prometheus: + prometheusSpec: + retention: 2d + replicas: 1 + serviceMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + resources: + requests: + cpu: 100m + memory: 400Mi + limits: + cpu: 500m + memory: 1Gi + # emptyDir (no storageSpec) -> ephemeral, fine for local iteration. + +alertmanager: + alertmanagerSpec: + replicas: 1 + +grafana: + replicas: 1 + adminPassword: admin # local only + service: + type: ClusterIP # access via: kubectl port-forward svc/kube-prometheus-stack-grafana 3001:80 + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: monitoring + dashboards: + default: + kubernetes-cluster: + gnetId: 7249 + revision: 1 + datasource: Prometheus + istio-mesh: + gnetId: 7639 + revision: 1 + datasource: Prometheus + +# Reduce footprint on Minikube. +nodeExporter: + enabled: true +kubeStateMetrics: + enabled: true diff --git a/monitoring/prometheus/values-eks.yaml b/monitoring/prometheus/values-eks.yaml new file mode 100644 index 0000000..273b451 --- /dev/null +++ b/monitoring/prometheus/values-eks.yaml @@ -0,0 +1,102 @@ +# kube-prometheus-stack values — staging/production (EKS). +# helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ +# -n monitoring -f monitoring/prometheus/values-eks.yaml +fullnameOverride: kube-prometheus-stack + +prometheus: + prometheusSpec: + retention: 15d + replicas: 2 + # Select ServiceMonitors/PrometheusRules from all namespaces (not just chart's). + serviceMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: gp3 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi + +alertmanager: + alertmanagerSpec: + replicas: 2 + storage: + volumeClaimTemplate: + spec: + storageClassName: gp3 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi + +grafana: + replicas: 2 + adminPassword: "" # set via existingSecret in real deploys; never commit + admin: + existingSecret: grafana-admin + service: + type: LoadBalancer + ingress: + enabled: true + ingressClassName: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + hosts: + - grafana.example.com + tls: + - secretName: grafana-tls + hosts: + - grafana.example.com + # Provision dashboards from ConfigMaps labelled grafana_dashboard (sidecar). + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: monitoring + datasources: + enabled: true + # Import curated community dashboards by Grafana.com ID. + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + dashboards: + default: + kubernetes-cluster: + gnetId: 7249 + revision: 1 + datasource: Prometheus + istio-mesh: + gnetId: 7639 + revision: 1 + datasource: Prometheus + node-exporter: + gnetId: 1860 + revision: 37 + datasource: Prometheus + +# Loki datasource is added by the loki-stack release; declare it here too so +# Explore works out of the box. +additionalDataSources: + - name: Loki + type: loki + access: proxy + url: http://loki.monitoring.svc.cluster.local:3100 diff --git a/monitoring/promtail/values.yaml b/monitoring/promtail/values.yaml new file mode 100644 index 0000000..8296471 --- /dev/null +++ b/monitoring/promtail/values.yaml @@ -0,0 +1,28 @@ +# Promtail — ships node/pod logs to Loki. Same values work on Minikube and EKS. +# helm install promtail grafana/promtail -n monitoring -f monitoring/promtail/values.yaml +config: + clients: + - url: http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push + snippets: + # Promote structured fields from the app's pino JSON logs so they can be + # correlated with traces in Grafana (request_id / trace_id). + pipelineStages: + - cri: {} + - json: + expressions: + level: level + req_id: req.id + trace_id: trace_id + - labels: + level: + +daemonset: + enabled: true + +resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi