From c465689189c6a4cbe037caaa9ce703a6b9f0cf2a Mon Sep 17 00:00:00 2001 From: mck21 Date: Mon, 8 Jun 2026 17:50:41 +0200 Subject: [PATCH 1/6] feat: add argocd sync wave annotations for deployment ordering Wire blue/green deployments into the staging overlay with sync waves, active-color ConfigMap, and separate ECR image tags per color. Co-authored-by: Cursor --- k8s/blue-green/deployment-blue.yaml | 7 ++++--- k8s/blue-green/deployment-green.yaml | 4 +++- k8s/blue-green/destinationrule.yaml | 2 ++ k8s/blue-green/virtualservice.yaml | 2 ++ k8s/overlays/staging/active-color.yaml | 8 ++++++++ k8s/overlays/staging/kustomization.yaml | 17 ++++++++++++++--- ...ployment.yaml => patch-blue-deployment.yaml} | 6 +++++- k8s/overlays/staging/patch-delete-backend.yaml | 5 +++++ k8s/overlays/staging/patch-delete-hpa.yaml | 5 +++++ .../staging/patch-green-deployment.yaml | 15 +++++++++++++++ 10 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 k8s/overlays/staging/active-color.yaml rename k8s/overlays/staging/{patch-deployment.yaml => patch-blue-deployment.yaml} (55%) create mode 100644 k8s/overlays/staging/patch-delete-backend.yaml create mode 100644 k8s/overlays/staging/patch-delete-hpa.yaml create mode 100644 k8s/overlays/staging/patch-green-deployment.yaml diff --git a/k8s/blue-green/deployment-blue.yaml b/k8s/blue-green/deployment-blue.yaml index e967720..c57980e 100644 --- a/k8s/blue-green/deployment-blue.yaml +++ b/k8s/blue-green/deployment-blue.yaml @@ -1,5 +1,4 @@ -# Blue/green skeleton — not included in dev overlay by default. -# Switch traffic via VirtualService in Phase 5 automation. +# Blue/green — wired in staging overlay; traffic switch via scripts/cd pipeline. apiVersion: apps/v1 kind: Deployment metadata: @@ -7,6 +6,8 @@ metadata: labels: app: backend version: blue + annotations: + argocd.argoproj.io/sync-wave: "1" spec: replicas: 1 selector: @@ -22,7 +23,7 @@ spec: serviceAccountName: backend-sa containers: - name: backend - image: backend:latest + image: backend-blue:latest imagePullPolicy: IfNotPresent ports: - containerPort: 3000 diff --git a/k8s/blue-green/deployment-green.yaml b/k8s/blue-green/deployment-green.yaml index 47d2da4..3642804 100644 --- a/k8s/blue-green/deployment-green.yaml +++ b/k8s/blue-green/deployment-green.yaml @@ -5,6 +5,8 @@ metadata: labels: app: backend version: green + annotations: + argocd.argoproj.io/sync-wave: "1" spec: replicas: 1 selector: @@ -20,7 +22,7 @@ spec: serviceAccountName: backend-sa containers: - name: backend - image: backend:latest + image: backend-green:latest imagePullPolicy: IfNotPresent ports: - containerPort: 3000 diff --git a/k8s/blue-green/destinationrule.yaml b/k8s/blue-green/destinationrule.yaml index d6d8617..d48cbdb 100644 --- a/k8s/blue-green/destinationrule.yaml +++ b/k8s/blue-green/destinationrule.yaml @@ -2,6 +2,8 @@ apiVersion: networking.istio.io/v1 kind: DestinationRule metadata: name: backend + annotations: + argocd.argoproj.io/sync-wave: "2" spec: host: backend subsets: diff --git a/k8s/blue-green/virtualservice.yaml b/k8s/blue-green/virtualservice.yaml index daac189..3f86252 100644 --- a/k8s/blue-green/virtualservice.yaml +++ b/k8s/blue-green/virtualservice.yaml @@ -2,6 +2,8 @@ apiVersion: networking.istio.io/v1 kind: VirtualService metadata: name: backend + annotations: + argocd.argoproj.io/sync-wave: "2" spec: hosts: - backend diff --git a/k8s/overlays/staging/active-color.yaml b/k8s/overlays/staging/active-color.yaml new file mode 100644 index 0000000..30b154c --- /dev/null +++ b/k8s/overlays/staging/active-color.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cd-active-color + annotations: + argocd.argoproj.io/sync-wave: "0" +data: + ACTIVE_COLOR: blue diff --git a/k8s/overlays/staging/kustomization.yaml b/k8s/overlays/staging/kustomization.yaml index df2bd55..2b0b172 100644 --- a/k8s/overlays/staging/kustomization.yaml +++ b/k8s/overlays/staging/kustomization.yaml @@ -3,14 +3,25 @@ kind: Kustomization namespace: staging +commonAnnotations: + argocd.argoproj.io/sync-wave: "0" + resources: - ../../base + - ../../blue-green + - active-color.yaml patches: - - path: patch-deployment.yaml + - path: patch-delete-backend.yaml + - path: patch-delete-hpa.yaml - path: patch-ingress.yaml + - path: patch-blue-deployment.yaml + - path: patch-green-deployment.yaml images: - - name: backend - newName: .dkr.ecr.us-east-1.amazonaws.com/devsecops-staging-backend + - name: backend-blue + newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend + newTag: latest + - name: backend-green + newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend newTag: latest diff --git a/k8s/overlays/staging/patch-deployment.yaml b/k8s/overlays/staging/patch-blue-deployment.yaml similarity index 55% rename from k8s/overlays/staging/patch-deployment.yaml rename to k8s/overlays/staging/patch-blue-deployment.yaml index 69d4041..13d1dd4 100644 --- a/k8s/overlays/staging/patch-deployment.yaml +++ b/k8s/overlays/staging/patch-blue-deployment.yaml @@ -1,10 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: backend + name: backend-blue spec: replicas: 2 template: + metadata: + annotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true spec: containers: - name: backend diff --git a/k8s/overlays/staging/patch-delete-backend.yaml b/k8s/overlays/staging/patch-delete-backend.yaml new file mode 100644 index 0000000..dedb15b --- /dev/null +++ b/k8s/overlays/staging/patch-delete-backend.yaml @@ -0,0 +1,5 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +$patch: delete diff --git a/k8s/overlays/staging/patch-delete-hpa.yaml b/k8s/overlays/staging/patch-delete-hpa.yaml new file mode 100644 index 0000000..4a677fb --- /dev/null +++ b/k8s/overlays/staging/patch-delete-hpa.yaml @@ -0,0 +1,5 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: backend-hpa +$patch: delete diff --git a/k8s/overlays/staging/patch-green-deployment.yaml b/k8s/overlays/staging/patch-green-deployment.yaml new file mode 100644 index 0000000..c1618bb --- /dev/null +++ b/k8s/overlays/staging/patch-green-deployment.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend-green +spec: + replicas: 2 + template: + metadata: + annotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true + spec: + containers: + - name: backend + imagePullPolicy: Always From dabf37184318bd6456986c797f9f7c5616006d8f Mon Sep 17 00:00:00 2001 From: mck21 Date: Mon, 8 Jun 2026 17:50:47 +0200 Subject: [PATCH 2/6] feat: add eks access entry for staging cicd role Allow the GitHub Actions cicd role to access the staging cluster API for rollout checks and blue/green operations. Co-authored-by: Cursor --- infrastructure/environments/staging/main.tf | 1 + infrastructure/modules/eks/main.tf | 21 +++++++++++++++++++++ infrastructure/modules/eks/variables.tf | 6 ++++++ 3 files changed, 28 insertions(+) diff --git a/infrastructure/environments/staging/main.tf b/infrastructure/environments/staging/main.tf index b1d62e9..ea5f739 100644 --- a/infrastructure/environments/staging/main.tf +++ b/infrastructure/environments/staging/main.tf @@ -60,6 +60,7 @@ module "eks" { node_desired_size = 2 node_min_size = 1 node_max_size = 4 + cicd_role_arn = module.iam.cicd_role_arn } module "ecr" { diff --git a/infrastructure/modules/eks/main.tf b/infrastructure/modules/eks/main.tf index 6a22c2b..05dec2e 100644 --- a/infrastructure/modules/eks/main.tf +++ b/infrastructure/modules/eks/main.tf @@ -79,4 +79,25 @@ resource "aws_eks_node_group" "main" { depends_on = [ aws_eks_cluster.main ] +} + +# --- CI/CD cluster access (GitHub Actions kubectl) --- +resource "aws_eks_access_entry" "cicd" { + count = var.cicd_role_arn != "" ? 1 : 0 + + cluster_name = aws_eks_cluster.main.name + principal_arn = var.cicd_role_arn + type = "STANDARD" +} + +resource "aws_eks_access_policy_association" "cicd_admin" { + count = var.cicd_role_arn != "" ? 1 : 0 + + cluster_name = aws_eks_cluster.main.name + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + principal_arn = aws_eks_access_entry.cicd[0].principal_arn + + access_scope { + type = "cluster" + } } \ No newline at end of file diff --git a/infrastructure/modules/eks/variables.tf b/infrastructure/modules/eks/variables.tf index 51a2472..1e7ded2 100644 --- a/infrastructure/modules/eks/variables.tf +++ b/infrastructure/modules/eks/variables.tf @@ -66,4 +66,10 @@ variable "node_max_size" { description = "Maximum number of worker nodes" type = number default = 4 +} + +variable "cicd_role_arn" { + description = "IAM role ARN for GitHub Actions CD (EKS access entry). Empty disables access entry." + type = string + default = "" } \ No newline at end of file From 7ac88aad98e2f34399192d5538671fb3fe0343c3 Mon Sep 17 00:00:00 2001 From: mck21 Date: Mon, 8 Jun 2026 17:50:47 +0200 Subject: [PATCH 3/6] feat: add blue-green traffic switch automation script Add GitOps bump, health check, traffic switch, and rollback scripts used by the CD pipeline. Co-authored-by: Cursor --- scripts/blue-green-health.sh | 31 +++++++++++++++++++ scripts/blue-green-switch.sh | 7 +++++ scripts/gitops-bump.sh | 7 +++++ scripts/gitops_bump.py | 55 ++++++++++++++++++++++++++++++++++ scripts/rollback.sh | 23 ++++++++++++++ scripts/traffic_switch.py | 58 ++++++++++++++++++++++++++++++++++++ 6 files changed, 181 insertions(+) create mode 100755 scripts/blue-green-health.sh create mode 100755 scripts/blue-green-switch.sh create mode 100755 scripts/gitops-bump.sh create mode 100644 scripts/gitops_bump.py create mode 100755 scripts/rollback.sh create mode 100644 scripts/traffic_switch.py diff --git a/scripts/blue-green-health.sh b/scripts/blue-green-health.sh new file mode 100755 index 0000000..d59d081 --- /dev/null +++ b/scripts/blue-green-health.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +COLOR="${1:?Usage: blue-green-health.sh [namespace]}" +NAMESPACE="${2:-staging}" +DEPLOYMENT="backend-${COLOR}" +TIMEOUT="${TIMEOUT:-300s}" + +echo "Waiting for deployment/${DEPLOYMENT} rollout in namespace ${NAMESPACE}..." +kubectl rollout status "deployment/${DEPLOYMENT}" -n "$NAMESPACE" --timeout="$TIMEOUT" + +POD="$(kubectl get pods -n "$NAMESPACE" -l "app=backend,version=${COLOR}" \ + -o jsonpath='{.items[0].metadata.name}')" + +if [[ -z "$POD" ]]; then + echo "No pod found for color ${COLOR}" >&2 + exit 1 +fi + +echo "Checking /health on pod ${POD}..." +for _ in $(seq 1 30); do + if kubectl exec -n "$NAMESPACE" "$POD" -c backend -- \ + wget -qO- http://127.0.0.1:3000/health >/dev/null 2>&1; then + echo "Health check passed for ${COLOR}" + exit 0 + fi + sleep 5 +done + +echo "Health check failed for ${COLOR}" >&2 +exit 1 diff --git a/scripts/blue-green-switch.sh b/scripts/blue-green-switch.sh new file mode 100755 index 0000000..44852a0 --- /dev/null +++ b/scripts/blue-green-switch.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TARGET_COLOR="${1:?Usage: blue-green-switch.sh }" + +python3 "$ROOT/scripts/traffic_switch.py" "$TARGET_COLOR" diff --git a/scripts/gitops-bump.sh b/scripts/gitops-bump.sh new file mode 100755 index 0000000..9a47086 --- /dev/null +++ b/scripts/gitops-bump.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SHA_TAG="${1:?Usage: gitops-bump.sh }" + +python3 "$ROOT/scripts/gitops_bump.py" "$SHA_TAG" diff --git a/scripts/gitops_bump.py b/scripts/gitops_bump.py new file mode 100644 index 0000000..446411f --- /dev/null +++ b/scripts/gitops_bump.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Update the idle color image tag in staging kustomization.yaml.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ACTIVE_COLOR_FILE = ROOT / "k8s/overlays/staging/active-color.yaml" +KUSTOMIZATION_FILE = ROOT / "k8s/overlays/staging/kustomization.yaml" + + +def read_active_color() -> str: + for line in ACTIVE_COLOR_FILE.read_text().splitlines(): + if line.strip().startswith("ACTIVE_COLOR:"): + return line.split(":", 1)[1].strip() + raise SystemExit("ACTIVE_COLOR not found in active-color.yaml") + + +def idle_color(active: str) -> str: + if active == "blue": + return "green" + if active == "green": + return "blue" + raise SystemExit(f"Invalid ACTIVE_COLOR: {active}") + + +def bump_tag(sha_tag: str) -> tuple[str, str]: + active = read_active_color() + idle = idle_color(active) + image_name = f"backend-{idle}" + + content = KUSTOMIZATION_FILE.read_text() + pattern = rf"(- name: {re.escape(image_name)}\n newName: [^\n]+\n newTag: )[^\n]+" + updated, count = re.subn(pattern, rf"\g<1>{sha_tag}", content, count=1) + if count != 1: + raise SystemExit(f"Failed to update newTag for {image_name}") + + KUSTOMIZATION_FILE.write_text(updated) + return idle, image_name + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("Usage: gitops_bump.py ") + + idle, image_name = bump_tag(sys.argv[1]) + print(f"idle_color={idle}") + print(f"image_name={image_name}") + + +if __name__ == "__main__": + main() diff --git a/scripts/rollback.sh b/scripts/rollback.sh new file mode 100755 index 0000000..7fac518 --- /dev/null +++ b/scripts/rollback.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +NAMESPACE="${NAMESPACE:-staging}" + +CURRENT="$(grep 'ACTIVE_COLOR:' "$ROOT/k8s/overlays/staging/active-color.yaml" | awk '{print $2}')" +if [[ "$CURRENT" == "blue" ]]; then + REVERT_TO="green" +else + REVERT_TO="blue" +fi + +python3 "$ROOT/scripts/traffic_switch.py" "$REVERT_TO" + +echo "Rollback: reverted VirtualService and active-color to ${REVERT_TO}" +echo "Revert the GitOps image bump commit if the idle color image must be rolled back." + +if command -v argocd >/dev/null 2>&1; then + argocd app rollback feature-flags-staging || true +fi + +kubectl rollout status "deployment/backend-${REVERT_TO}" -n "$NAMESPACE" --timeout=120s || true diff --git a/scripts/traffic_switch.py b/scripts/traffic_switch.py new file mode 100644 index 0000000..0ec8112 --- /dev/null +++ b/scripts/traffic_switch.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Switch Istio VirtualService weights and update active color in Git.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ACTIVE_COLOR_FILE = ROOT / "k8s/overlays/staging/active-color.yaml" +VIRTUALSERVICE_FILE = ROOT / "k8s/blue-green/virtualservice.yaml" + + +def switch_to(target: str) -> None: + if target not in {"blue", "green"}: + raise SystemExit("Target color must be blue or green") + + other = "green" if target == "blue" else "blue" + target_weight = 100 + other_weight = 0 + + vs = VIRTUALSERVICE_FILE.read_text() + + vs = re.sub( + rf"(subset: {target}\n weight: )\d+", + rf"\g<1>{target_weight}", + vs, + count=1, + ) + vs = re.sub( + rf"(subset: {other}\n weight: )\d+", + rf"\g<1>{other_weight}", + vs, + count=1, + ) + VIRTUALSERVICE_FILE.write_text(vs) + + active = ACTIVE_COLOR_FILE.read_text() + active = re.sub( + r"(ACTIVE_COLOR: )\w+", + rf"\g<1>{target}", + active, + count=1, + ) + ACTIVE_COLOR_FILE.write_text(active) + + print(f"active_color={target}") + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("Usage: traffic_switch.py ") + switch_to(sys.argv[1]) + + +if __name__ == "__main__": + main() From 94110860f57c9832d51eb052f2da7169fb089059 Mon Sep 17 00:00:00 2001 From: mck21 Date: Mon, 8 Jun 2026 17:50:50 +0200 Subject: [PATCH 4/6] feat: add cd workflow for gitops image tag update on main merge Trigger CD after CI on main, bump idle color tag, health check, and switch traffic. Restore CI workflow with skip-ci guard and gate production ECR push behind ENABLE_PROD_ECR. Co-authored-by: Cursor --- .github/workflows/cd.yaml | 150 +++++++++++++++++++++++++++++++++++ .github/workflows/ci.yaml | 160 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 .github/workflows/cd.yaml create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml new file mode 100644 index 0000000..d93941c --- /dev/null +++ b/.github/workflows/cd.yaml @@ -0,0 +1,150 @@ +name: CD + +on: + workflow_run: + workflows: ['CI'] + types: [completed] + branches: [main] + +concurrency: + group: cd-main + cancel-in-progress: false + +permissions: + contents: write + id-token: write + +env: + NAMESPACE: staging + OVERLAY_PATH: k8s/overlays/staging + +jobs: + gate: + name: Gate on CI success + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'success' + outputs: + sha_tag: sha-${{ github.event.workflow_run.head_sha }} + steps: + - name: Validate CI conclusion + run: echo "Deploying ${{ github.event.workflow_run.head_sha }}" + + gitops-bump: + name: GitOps image bump + needs: gate + runs-on: ubuntu-latest + outputs: + idle_color: ${{ steps.bump.outputs.idle_color }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: main + token: ${{ secrets.CD_BOT_TOKEN || github.token }} + + - name: Bump idle color image tag + id: bump + run: | + OUTPUT="$(./scripts/gitops-bump.sh "${{ needs.gate.outputs.sha_tag }}")" + eval "$OUTPUT" + echo "idle_color=${idle_color}" >> "$GITHUB_OUTPUT" + + - name: Commit and push tag bump + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "${{ env.OVERLAY_PATH }}/kustomization.yaml" + git commit -m "chore(cd): bump staging ${{ steps.bump.outputs.idle_color }} to ${{ needs.gate.outputs.sha_tag }} [skip ci]" + git push + + deploy: + name: Sync, health check, traffic switch + needs: [gate, gitops-bump] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: main + token: ${{ secrets.CD_BOT_TOKEN || github.token }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-staging-cicd-role + aws-region: ${{ vars.AWS_REGION }} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --name "${{ vars.EKS_CLUSTER_NAME }}" \ + --region "${{ vars.AWS_REGION }}" + + - name: Wait for ArgoCD sync and rollout + run: ./scripts/blue-green-health.sh "${{ needs.gitops-bump.outputs.idle_color }}" "${{ env.NAMESPACE }}" + + - name: Switch traffic to new color + run: | + ./scripts/blue-green-switch.sh "${{ needs.gitops-bump.outputs.idle_color }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add k8s/blue-green/virtualservice.yaml k8s/overlays/staging/active-color.yaml + git commit -m "chore(cd): switch staging traffic to ${{ needs.gitops-bump.outputs.idle_color }} [skip ci]" + git push + + - name: Public health check + if: ${{ vars.STAGING_HEALTH_URL != '' }} + run: | + for i in $(seq 1 30); do + if curl -fsS "${{ vars.STAGING_HEALTH_URL }}"; then + echo "Public health check passed" + exit 0 + fi + sleep 10 + done + echo "Public health check failed" >&2 + exit 1 + + rollback: + name: Rollback on failure + needs: [gate, gitops-bump, deploy] + if: failure() && needs.gate.result == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: main + fetch-depth: 0 + token: ${{ secrets.CD_BOT_TOKEN || github.token }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-staging-cicd-role + aws-region: ${{ vars.AWS_REGION }} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --name "${{ vars.EKS_CLUSTER_NAME }}" \ + --region "${{ vars.AWS_REGION }}" + + - name: Revert traffic and active color + run: | + ACTIVE="$(grep 'ACTIVE_COLOR:' k8s/overlays/staging/active-color.yaml | awk '{print $2}')" + if [[ "$ACTIVE" == "${{ needs.gitops-bump.outputs.idle_color }}" ]]; then + ./scripts/rollback.sh + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add k8s/blue-green/virtualservice.yaml k8s/overlays/staging/active-color.yaml + git diff --cached --quiet || git commit -m "chore(cd): rollback staging traffic [skip ci]" + git push || true + fi + + - name: Revert image tag bump commit + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + BUMP_SHA="$(git log -1 --grep 'bump staging' --format=%H || true)" + if [[ -n "$BUMP_SHA" ]]; then + git revert --no-edit "$BUMP_SHA" || true + git push || true + fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..579eedf --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,160 @@ +name: CI + +on: + push: + branches: ['**'] + pull_request: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # Prisma 7 loads prisma.config.ts at postinstall; no .env file in CI + DATABASE_URL: postgresql://ci:ci@localhost:5432/ci + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + + - name: Install yamllint + run: pip install yamllint + + - name: Install backend dependencies + working-directory: app/backend + run: bun install --frozen-lockfile + + - name: ESLint + working-directory: app/backend + run: bun run lint:ci + + - name: yamllint + run: | + yamllint -c .yamllint k8s/ + yamllint -c .yamllint .github/ + + - name: Terraform fmt + run: terraform fmt -check -recursive infrastructure/ + + test: + name: Unit Tests + runs-on: ubuntu-latest + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - name: Cache Bun dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.bun/install/cache + key: bun-${{ hashFiles('app/backend/bun.lock') }} + restore-keys: | + bun- + + - name: Install backend dependencies + working-directory: app/backend + run: bun install --frozen-lockfile + + - name: Jest with coverage + working-directory: app/backend + run: bun run test:ci + + - name: Upload coverage artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: coverage + path: app/backend/coverage/ + retention-days: 7 + + # SonarCloud analysis runs via the SonarCloud GitHub App (check: "SonarCloud Code Analysis"). + # A manual scanner job conflicts when Automatic Analysis is enabled on the free plan. + + build-and-push: + name: Build, Scan & Push + needs: [lint, test] + runs-on: ubuntu-latest + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: app/backend + push: false + load: true + tags: backend:ci + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Trivy image scan + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: backend:ci + format: table + severity: CRITICAL + ignore-unfixed: true + exit-code: '1' + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-staging-cicd-role + aws-region: ${{ vars.AWS_REGION }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2 + id: ecr-login + + - name: Set image tags + id: tags + run: | + REGISTRY="${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com" + STAGING="${REGISTRY}/${{ vars.PROJECT_NAME }}-staging-backend" + echo "staging=${STAGING}" >> "$GITHUB_OUTPUT" + echo "sha_tag=sha-${{ github.sha }}" >> "$GITHUB_OUTPUT" + + - name: Push to staging ECR + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: app/backend + push: true + tags: | + ${{ steps.tags.outputs.staging }}:${{ steps.tags.outputs.sha_tag }} + ${{ steps.tags.outputs.staging }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Push to production ECR + if: github.ref == 'refs/heads/main' && vars.ENABLE_PROD_ECR == 'true' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: app/backend + push: true + tags: | + ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:${{ steps.tags.outputs.sha_tag }} + ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:latest + cache-from: type=gha + cache-to: type=gha,mode=max From 8b72cc4a75f675a297bb68d7bd01886dd1631035 Mon Sep 17 00:00:00 2001 From: mck21 Date: Mon, 8 Jun 2026 17:50:56 +0200 Subject: [PATCH 5/6] docs: add eks staging bootstrap and deferred production promotion Document one-time EKS platform setup, Let's Encrypt issuer, and how to enable production CD after the rest of the PLAN is complete. Co-authored-by: Cursor --- docs/cd-production-promotion.md | 61 +++++++++ images/phase-5/README.md | 32 +++++ k8s/argocd/install-notes.md | 133 ++++++++++++++++++- k8s/istio/istio-notes.md | 27 ++-- k8s/platform/cluster-issuer-letsencrypt.yaml | 16 +++ 5 files changed, 248 insertions(+), 21 deletions(-) create mode 100644 docs/cd-production-promotion.md create mode 100644 images/phase-5/README.md create mode 100644 k8s/platform/cluster-issuer-letsencrypt.yaml diff --git a/docs/cd-production-promotion.md b/docs/cd-production-promotion.md new file mode 100644 index 0000000..b0d65fe --- /dev/null +++ b/docs/cd-production-promotion.md @@ -0,0 +1,61 @@ +# Production CD promotion (deferred) + +Production deploy is **intentionally disabled** during Phase 5 build to keep the pipeline green and AWS costs low. Staging is the sole automated CD target until the rest of the PLAN is complete. + +## When to enable + +After Phase 5 staging CD is stable and remaining PLAN phases are done (or when you explicitly choose to turn on multi-env). + +## Steps + +### 1. GitHub repository variables + +| Variable | Value | +|----------|-------| +| `ENABLE_PROD_ECR` | `true` — restores production ECR push on `main` in CI | +| `ENABLE_PROD_CD` | `true` — enable when the promote job is wired (future) | +| `PRODUCTION_HEALTH_URL` | `https://flags.example.com/health` | + +### 2. Bootstrap production EKS platform + +Repeat the staging bootstrap from [k8s/argocd/install-notes.md](../k8s/argocd/install-notes.md) on the production cluster: + +- Istio production profile +- Ingress NGINX + cert-manager + Let's Encrypt issuer +- ArgoCD Helm install +- `backend-secrets` in namespace `production` + +### 3. Register ArgoCD Application + +```bash +kubectl apply -f k8s/argocd/application-production.yaml +``` + +### 4. Add promote job to CD workflow + +Add a job (manual or automatic) after staging public health check succeeds: + +1. Read the validated `sha-` from the staging bump commit +2. Update `k8s/overlays/production/kustomization.yaml` `newTag` +3. Commit `chore(cd): promote production to sha-xxx [skip ci]` +4. Wait for ArgoCD sync on production +5. `curl -f $PRODUCTION_HEALTH_URL` + +Example trigger: + +```yaml +promote-production: + needs: deploy + if: vars.ENABLE_PROD_CD == 'true' + # workflow_dispatch for manual approval is recommended first +``` + +### 5. Blue/green on production (optional) + +Reuse `scripts/blue-green-switch.sh` and production overlay changes mirroring staging if zero-downtime promotion is required on production. + +## Rollback + +- Revert the production GitOps commit +- ArgoCD history rollback: `argocd app rollback feature-flags-production` +- Keep staging as the pre-production gate — never promote a SHA that failed staging diff --git a/images/phase-5/README.md b/images/phase-5/README.md new file mode 100644 index 0000000..7fd1c69 --- /dev/null +++ b/images/phase-5/README.md @@ -0,0 +1,32 @@ +# Phase 5 — CD & GitOps evidence + +Capture screenshots after the staging CD pipeline runs end-to-end on EKS. + +## Checklist + +| # | File | Content | +|---|------|---------| +| 1 | `01-cd-workflow-green.png` | GitHub Actions CD workflow success with GitOps commits | +| 2 | `02-argocd-staging-synced.png` | ArgoCD UI — `feature-flags-staging` Synced + Healthy | +| 3 | `03-blue-green-switch.png` | ArgoCD or terminal during traffic switch | +| 4 | `04-virtualservice-weights.png` | `kubectl get virtualservice backend -n staging -o yaml` showing 100/0 split | +| 5 | `05-rollback-complete.png` | Rollback job after simulated health failure (optional) | + +## Manual validation commands + +```bash +# After merge to main +gh run list --workflow=CD --limit 3 + +# Cluster +kubectl get pods,virtualservice -n staging +kubectl get application feature-flags-staging -n argocd + +# Health +curl -f "$STAGING_HEALTH_URL" +``` + +## Notes + +- Dev (Minikube) CD is manual during Phase 5 — no screenshot required for phase closure. +- Production ArgoCD screenshots deferred until [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) is activated. diff --git a/k8s/argocd/install-notes.md b/k8s/argocd/install-notes.md index c9f589a..411879a 100644 --- a/k8s/argocd/install-notes.md +++ b/k8s/argocd/install-notes.md @@ -1,13 +1,138 @@ # ArgoCD Installation ## Dev (Minikube) + Installed via official stable manifest: + +```bash kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml +``` Access: + +```bash kubectl port-forward svc/argocd-server -n argocd 8080:443 -URL: https://localhost:8080 -User: admin +``` + +- URL: https://localhost:8080 +- User: `admin` +- Password: `kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d` + +Register the dev app (optional, local GitOps): + +```bash +kubectl apply -f k8s/argocd/application-dev.yaml +``` + +## Staging (EKS) — Phase 5 bootstrap + +Prerequisites: `kubectl` context pointing at the staging EKS cluster (`mck21-devsecops-staging-eks`). + +### 1. Namespaces + +```bash +kubectl apply -f k8s/namespaces/namespaces.yaml +``` + +### 2. Istio (production profile) + +```bash +istioctl install --set profile=production -y +kubectl label namespace staging istio-injection=enabled --overwrite +``` + +See [k8s/istio/istio-notes.md](../istio/istio-notes.md). + +### 3. Ingress NGINX + +```bash +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx --create-namespace \ + --set controller.service.type=LoadBalancer +``` + +### 4. cert-manager + Let's Encrypt issuer + +```bash +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --create-namespace \ + --set crds.enabled=true + +kubectl apply -f k8s/platform/cluster-issuer-letsencrypt.yaml +``` + +### 5. Application secrets (never commit real values) + +```bash +kubectl apply -f k8s/base/secret.yaml -n staging +# Or create from template with env-specific DATABASE_URL: +# kubectl create secret generic backend-secrets -n staging \ +# --from-literal=DATABASE_URL='postgresql://...' +``` + +### 6. ArgoCD (Helm) + +```bash +helm repo add argo https://argoproj.github.io/argo-helm +helm repo update +helm upgrade --install argocd argo/argo-cd \ + --namespace argocd --create-namespace \ + --set configs.params.server.insecure=true +``` + +Initial admin password: + +```bash +kubectl -n argocd get secret argocd-initial-admin-secret \ + -o jsonpath='{.data.password}' | base64 -d && echo +``` + +Port-forward UI: + +```bash +kubectl port-forward svc/argocd-server -n argocd 8080:443 +``` + +### 7. Connect Git repository + +In ArgoCD UI → Settings → Repositories → Connect repo: + +- URL: `https://github.com/mck21/devsecops-platform.git` +- Public repo: no credentials required; private repo: PAT or deploy key + +Or CLI: + +```bash +argocd login localhost:8080 --username admin --password --insecure +argocd repo add https://github.com/mck21/devsecops-platform.git +``` + +### 8. Register staging Application (auto-sync) + +```bash +# Validate manifests first +kubectl apply -k k8s/overlays/staging --dry-run=server + +kubectl apply -f k8s/argocd/application-staging.yaml +``` + +Do **not** apply `application-production.yaml` until production CD is enabled (end of PLAN). + +### 9. GitHub Actions CD variables + +Set repository variables in GitHub → Settings → Variables: + +| Variable | Example | +|----------|---------| +| `STAGING_HEALTH_URL` | `https://flags.staging.example.com/health` (or ALB hostname) | +| `EKS_CLUSTER_NAME` | `mck21-devsecops-staging-eks` | + +Ensure branch protection allows GitHub Actions to push GitOps commits to `main`, or configure `CD_BOT_TOKEN` secret. + +## Production (EKS) -## Staging/Production (EKS) -Installed via Helm in Phase 5. \ No newline at end of file +ArgoCD and app registration deferred until multi-env promotion is enabled. See [docs/cd-production-promotion.md](../../docs/cd-production-promotion.md). diff --git a/k8s/istio/istio-notes.md b/k8s/istio/istio-notes.md index 241b89f..f67941e 100644 --- a/k8s/istio/istio-notes.md +++ b/k8s/istio/istio-notes.md @@ -1,22 +1,15 @@ -# Istio Installation +## Staging / Production (EKS) -## Dev (Minikube) -Installed via istioctl with demo profile: -istioctl install --set profile=demo -y +Install Istio with the production profile during Phase 5 bootstrap: -Istio version used: 1.30.0 - -Addons installed: -- Kiali (service mesh topology) -- Jaeger (distributed tracing) -- Prometheus (metrics, Istio-scoped) - -Access dashboards: -istioctl dashboard kiali -istioctl dashboard jaeger +```bash +istioctl install --set profile=production -y +kubectl label namespace staging istio-injection=enabled --overwrite +``` Sidecar injection enabled on namespaces: -- dev -## Staging/Production (EKS) -Installed via istioctl with production profile in Phase 5. \ No newline at end of file +- staging +- production + +Full bootstrap: [k8s/argocd/install-notes.md](../argocd/install-notes.md). diff --git a/k8s/platform/cluster-issuer-letsencrypt.yaml b/k8s/platform/cluster-issuer-letsencrypt.yaml new file mode 100644 index 0000000..6476d1b --- /dev/null +++ b/k8s/platform/cluster-issuer-letsencrypt.yaml @@ -0,0 +1,16 @@ +# Apply after cert-manager Helm install on EKS staging/production. +# Replace email before first apply. +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@example.com + privateKeySecretRef: + name: letsencrypt-prod-account-key + solvers: + - http01: + ingress: + class: nginx From b70d8b2adebbe0574cf1777f506c9c6eac8384ba Mon Sep 17 00:00:00 2001 From: mck21 Date: Mon, 8 Jun 2026 17:50:56 +0200 Subject: [PATCH 6/6] docs: update phase 5 status and troubleshooting Reflect staging-first CD strategy, required GitHub variables, and CD failure runbook entries. Co-authored-by: Cursor --- AGENTS.md | 27 +++++++------ README.md | 5 ++- STATUS.md | 96 +++++++++++++++++----------------------------- TROUBLESHOOTING.md | 47 +++++++++++++++++++++++ k8s/README.md | 46 ++++++++++++++++------ 5 files changed, 134 insertions(+), 87 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4bdc83a..1d521d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ Entry point for AI agents working on this repository. Read this file first, then | Dev cluster | Minikube (local) | | Staging / Production | EKS on AWS | -**Current phase:** Phase 3 complete. **Next:** Phase 4 — CI Pipeline. See [STATUS.md](STATUS.md). +**Current phase:** Phase 5 implemented (staging-first CD). **Next:** Validate CD on EKS staging, then Phase 6 — Security Hardening. See [STATUS.md](STATUS.md). --- @@ -33,7 +33,8 @@ devsecops-platform/ ├── app/frontend/ # Empty — reserved for future UI ├── infrastructure/ # Terraform modules + staging/production envs ├── k8s/ # Kustomize manifests (base + overlays + blue-green + argocd) -├── .github/workflows/ # Empty — Phase 4 +├── .github/workflows/ # ci.yaml, cd.yaml, security.yaml, terraform.yaml +├── scripts/ # blue/green CD automation (Phase 5) ├── docs/ # Placeholder — full docs in Phase 9 ├── images/phase-N/ # Screenshot evidence per phase ├── docker-compose.yml # Local dev stack (postgres + redis + backend) @@ -117,14 +118,12 @@ Summary: | Deferred to | Do not implement yet | |-------------|---------------------| -| Phase 4 | GitHub Actions CI, ECR push, SonarCloud gate | -| Phase 5 | ArgoCD auto-sync, CD image-tag updates, blue/green traffic switch scripts | -| Phase 5 | Live EKS deploy of the application | | Phase 6 | Kyverno policies, NetworkPolicy, External Secrets, manifest `securityContext` | | Phase 7 | Grafana dashboards, Loki, custom Prometheus ServiceMonitors | | Phase 9 | Full `docs/`, ADRs, architecture diagrams | +| End of PLAN | Production CD auto-sync — see [docs/cd-production-promotion.md](docs/cd-production-promotion.md) | -**Scaffolding exists but is not wired:** `k8s/argocd/application-*.yaml`, `k8s/blue-green/`, `k8s/overlays/staging|production/` — do not connect to live CD until Phase 5. +**Phase 5 (staging-first):** CD auto-deploys to **staging EKS only**. Dev is manual. Run EKS bootstrap once: [k8s/argocd/install-notes.md](k8s/argocd/install-notes.md). Set GitHub vars `EKS_CLUSTER_NAME`, `STAGING_HEALTH_URL`. --- @@ -144,6 +143,10 @@ Summary: | Terraform staging/prod | `infrastructure/environments/staging/`, `production/` | | Phase roadmap | `PLAN.md` | | Phase progress | `STATUS.md` | +| CI / CD workflows | `.github/workflows/ci.yaml`, `.github/workflows/cd.yaml` | +| CD scripts | `scripts/gitops-bump.sh`, `scripts/blue-green-switch.sh`, `scripts/rollback.sh` | +| SonarCloud config | `sonar-project.properties` | +| GitHub OIDC (Terraform) | `infrastructure/modules/github-oidc/` | --- @@ -157,12 +160,12 @@ Summary: --- -## Tech Stack Note for CI (Phase 4) +## Tech Stack Note — CI (Phase 4, done) -The application is **NestJS / TypeScript / Jest / Bun**, not Python. When implementing Phase 4 CI: +The application is **NestJS / TypeScript / Jest / Bun**, not Python: -- Lint: ESLint (not ruff) -- Tests: Jest with coverage (not pytest) -- SonarCloud: TypeScript sources under `app/backend/src/` +- Lint: ESLint via `bun run lint:ci` (not ruff) +- Tests: Jest with coverage via `bun run test:ci` (not pytest) +- SonarCloud: TypeScript sources under `app/backend/src/`; quality gate via GitHub App check on free plan -Some older references in PLAN.md Phase 4 may still mention Python — treat NestJS/TypeScript as the source of truth. +Some older references in [PLAN.md](PLAN.md) may still mention Python — treat NestJS/TypeScript as the source of truth. diff --git a/README.md b/README.md index ee599e7..d422b7c 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,9 @@ The application is a **Feature Flag Service** — manage feature toggles per env | 1 | AWS Infrastructure (Terraform) | Done | | 2 | Kubernetes base (Minikube + Istio) | Done | | 3 | Feature Flag Service + K8s manifests | **Done** | -| 4 | CI Pipeline (GitHub Actions, SonarCloud) | Next | -| 5–9 | CD, Security, Monitoring, DR, Docs | Planned | +| 4 | CI Pipeline (GitHub Actions, SonarCloud, ECR) | **Done** | +| 5 | CD Pipeline & GitOps (ArgoCD, staging-first) | **Implemented** — validate on EKS | +| 6–9 | Security, Monitoring, DR, Docs | Planned | Full tracker: [STATUS.md](STATUS.md) · Full roadmap: [PLAN.md](PLAN.md) diff --git a/STATUS.md b/STATUS.md index 37dbc24..bb0273a 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,8 +1,8 @@ # Project Status -Phase tracker for the DevSecOps platform. Updated after Phase 3 completion. +Phase tracker for the DevSecOps platform. Updated after Phase 5 implementation. -**Active work:** Phase 4 — CI Pipeline +**Active work:** Phase 5 validation on EKS staging (screenshots) **Agent entry point:** [AGENTS.md](AGENTS.md) **Deploy runbook:** [TROUBLESHOOTING.md](TROUBLESHOOTING.md) **Full roadmap:** [PLAN.md](PLAN.md) @@ -17,8 +17,8 @@ Phase tracker for the DevSecOps platform. Updated after Phase 3 completion. | 1 | Terraform & AWS (VPC, EKS, ECR, IAM) | Done | `infrastructure/`, `images/phase-1/` | | 2 | Kubernetes base (Minikube: Istio, ArgoCD, Ingress) | Done (Minikube) | `k8s/namespaces/`, `images/phase-2/`, install notes in `k8s/argocd/`, `k8s/istio/` | | 3 | Feature Flag Service + K8s manifests | **Done** | `app/backend/`, `k8s/`, `images/phase-3/` | -| 4 | CI Pipeline (GitHub Actions, SonarCloud, ECR) | Not started | `.github/workflows/` empty | -| 5 | CD & GitOps (ArgoCD sync, blue/green automation) | Not started | Scaffolding in `k8s/argocd/`, `k8s/blue-green/` | +| 4 | CI Pipeline (GitHub Actions, SonarCloud, ECR) | **Done** | `.github/workflows/`, `sonar-project.properties`, `images/phase-4/` | +| 5 | CD & GitOps (ArgoCD sync, blue/green automation) | **Implemented** (staging-first) | `.github/workflows/cd.yaml`, `scripts/`, `images/phase-5/` | | 6 | Security hardening (Kyverno, Cosign, NetworkPolicy) | Not started | — | | 7 | Monitoring & SRE (Prometheus, Grafana, Loki) | Not started | `monitoring/` placeholder | | 8 | DR & resilience (Velero, k6) | Not started | — | @@ -26,78 +26,52 @@ Phase tracker for the DevSecOps platform. Updated after Phase 3 completion. --- -## Phase 3 — Completion Details +## Phase 5 — Implementation Details (staging-first) -All criteria met. See [PLAN.md § Phase 3 Completion Criteria](PLAN.md) for the full checklist (marked `[x]`). +Automatic CD targets **staging EKS only**. Dev (Minikube) remains manual. Production CD deferred — see [docs/cd-production-promotion.md](docs/cd-production-promotion.md). -### Application +### Delivered -- NestJS API with flags, audit, health, metrics modules -- Redis cache on flag reads; PostgreSQL source of truth + audit log -- Structured JSON logging (pino) with request IDs -- Docker Compose local stack; multi-stage Dockerfile with non-root user +- [`.github/workflows/cd.yaml`](.github/workflows/cd.yaml) — post-CI GitOps bump, rollout health check, traffic switch, rollback +- [`scripts/`](scripts/) — `gitops-bump.sh`, `blue-green-switch.sh`, `blue-green-health.sh`, `rollback.sh` +- [`k8s/overlays/staging/`](k8s/overlays/staging/) — blue/green overlay, sync waves, `cd-active-color` ConfigMap +- [`k8s/argocd/install-notes.md`](k8s/argocd/install-notes.md) — EKS staging bootstrap runbook +- Terraform EKS access entry for staging `cicd` role ([`infrastructure/modules/eks/`](infrastructure/modules/eks/)) +- CI: `[skip ci]` guard; production ECR gated by `ENABLE_PROD_ECR` (default off) -### Kubernetes — dev (Minikube) +### GitHub variables required -- `kubectl apply -k k8s/overlays/dev` deploys backend, postgres, redis, HPA -- LimitRange, ResourceQuota, ServiceAccount, Istio TCP DestinationRules applied -- HPA: `minReplicas: 1` (dev patch), `maxReplicas: 10` (base) +| Variable | Example | +|----------|---------| +| `EKS_CLUSTER_NAME` | `mck21-devsecops-staging-eks` | +| `STAGING_HEALTH_URL` | `https:///health` | -### Kubernetes — scaffolding (not live-deployed) +Optional: `ENABLE_PROD_ECR=true`, secret `CD_BOT_TOKEN` if branch protection blocks `GITHUB_TOKEN`. -- `k8s/overlays/staging/` and `k8s/overlays/production/` with ECR placeholders -- `k8s/argocd/application-{dev,staging,production}.yaml` -- `k8s/blue-green/` (deployments + VirtualService + DestinationRule) +### Pending validation -### Screenshots - -Located in `images/phase-3/`: - -| File | Content | -|------|---------| -| `01-docker-compose-running.png` | Docker Compose stack up | -| `02-post-create-flag.png` | POST /api/flags | -| `03-patch-toggle-flag.png` | PATCH toggle | -| `04-get-audit-history.png` | GET /api/audit | -| `05-get-prometheus-metrics.png` | GET /metrics | -| `06-health-checks.png` | GET /health | -| `07-k8s-pods-dev.png` | Pods running in dev namespace | - ---- - -## Active Work — Phase 4 - -**Goal:** Every push triggers lint, test, SonarCloud quality gate, security scans, Docker build, and ECR push. - -**Key deliverables:** - -- `.github/workflows/ci.yaml` — main pipeline -- `.github/workflows/security.yaml` — standalone security scans -- `.github/workflows/terraform.yaml` — terraform validate/plan -- `sonar-project.properties` — SonarCloud config for TypeScript -- AWS OIDC auth for ECR push (no long-lived keys) - -**Stack correction:** Application is NestJS/TypeScript/Jest — not Python. See [AGENTS.md § Tech Stack Note](AGENTS.md). +- [ ] One-time EKS staging bootstrap per install notes +- [ ] `terraform apply` in staging for EKS access entry +- [ ] Merge to `main` → CI → CD green +- [ ] Screenshots in [`images/phase-5/`](images/phase-5/) per README checklist --- ## Blockers / Deferred -| Item | Reason | Target Phase | -|------|--------|--------------| -| Live EKS app deploy | Requires CI image push + CD wiring | 4 + 5 | -| ArgoCD auto-sync | CD pipeline not built | 5 | -| EKS platform components (Ingress, Cert Manager on EKS) | Partially done in Phase 2; full parity deferred | 2 EKS + 5 | -| `infrastructure/environments/dev/` | Empty by design — dev runs on Minikube | N/A | -| Architecture diagram, full deployment guide | Portfolio polish | 9 | -| Grafana dashboards | Monitoring stack not installed | 7 | +| Item | Reason | Target | +|------|--------|--------| +| Production CD | Staging-first cost/risk strategy | [docs/cd-production-promotion.md](docs/cd-production-promotion.md) | +| Dev automated CD | Minikube not reachable from GitHub-hosted runners | Manual dev deploy | +| Architecture diagram, full deployment guide | Portfolio polish | Phase 9 | +| Grafana dashboards | Monitoring stack not installed | Phase 7 | --- ## Environment Strategy -| Environment | Kubernetes | Status | -|-------------|------------|--------| -| `dev` | Minikube (local) | App deployed and validated | -| `staging` | EKS (AWS) | Terraform ready; app not deployed | -| `production` | EKS (AWS) | Terraform ready; app not deployed | +| Environment | Kubernetes | CD | +|-------------|------------|-----| +| `dev` | Minikube (local) | Manual | +| `staging` | EKS (AWS) | **Automatic** (merge to `main`) | +| `production` | EKS (AWS) | Deferred | diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 8f9f783..6d271f6 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -409,6 +409,51 @@ Local stack does not use Istio. Most K8s/Istio issues do not apply. --- +## EKS staging CD (Phase 5) + +### Symptom: CD workflow fails on `git push` + +**Cause:** Branch protection blocks `GITHUB_TOKEN` writes to `main`. + +**Fix:** Allow GitHub Actions in branch rules, or set repository secret `CD_BOT_TOKEN` (PAT with `contents: write`) and re-run CD. + +### Symptom: CD fails at `Configure kubectl` or health check + +**Cause:** Staging `cicd` role lacks EKS access entry, or wrong cluster name. + +**Fix:** + +```bash +cd infrastructure/environments/staging +terraform apply # creates aws_eks_access_entry for cicd role +``` + +Set GitHub variable `EKS_CLUSTER_NAME=mck21-devsecops-staging-eks`. + +### Symptom: ArgoCD OutOfSync on VirtualService weights + +**Cause:** Traffic switch updated Git; ArgoCD self-heal reconciled correctly — verify weights match `k8s/blue-green/virtualservice.yaml`. + +**Fix:** `kubectl get virtualservice backend -n staging -o yaml`. Source of truth is Git. + +### Symptom: Idle color pods ImagePullBackOff + +**Cause:** ECR tag from CD bump not pushed yet, or node lacks ECR pull permissions. + +**Fix:** Confirm CI pushed `sha-` to staging ECR. Check node IAM role includes ECR read. + +### Symptom: Public health check fails after traffic switch + +**Cause:** DNS not pointing at ingress LoadBalancer, or cert-manager pending. + +**Fix:** Use ALB hostname in `STAGING_HEALTH_URL` until DNS is ready. Check `kubectl get ingress -n staging` and cert-manager Certificate status. + +### Bootstrap reference + +Full one-time EKS staging setup: [k8s/argocd/install-notes.md](k8s/argocd/install-notes.md). + +--- + ## Related Files | Topic | File | @@ -424,6 +469,8 @@ Local stack does not use Istio. Most K8s/Istio issues do not apply. | Dev ResourceQuota patch | [`k8s/overlays/dev/patch-resourcequota.yaml`](k8s/overlays/dev/patch-resourcequota.yaml) | | Secret template | [`k8s/base/secret.yaml`](k8s/base/secret.yaml) | | Istio install notes | [`k8s/istio/istio-notes.md`](k8s/istio/istio-notes.md) | +| CD scripts | [`scripts/`](scripts/) | +| Production CD (deferred) | [`docs/cd-production-promotion.md`](docs/cd-production-promotion.md) | --- diff --git a/k8s/README.md b/k8s/README.md index d3338fc..d03d097 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -10,11 +10,12 @@ Kustomize-based manifests for the Feature Flag Service. k8s/ ├── base/ # env-agnostic resources (no namespace) ├── overlays/ -│ ├── dev/ # Minikube — deploy now -│ ├── staging/ # EKS scaffolding -│ └── production/ # EKS scaffolding -├── blue-green/ # Istio blue/green skeleton (Phase 5 automation) -├── argocd/ # Application CRDs (wired in Phase 5) +│ ├── dev/ # Minikube — deploy manually +│ ├── staging/ # EKS — GitOps CD target (Phase 5) +│ └── production/ # EKS — deferred CD +├── blue-green/ # Istio blue/green (included in staging overlay) +├── argocd/ # Application CRDs +├── platform/ # EKS platform manifests (cert-manager issuer) └── namespaces/ # Platform namespaces ``` @@ -22,11 +23,11 @@ k8s/ | Path | Status | Notes | |------|--------|-------| -| `overlays/dev/` | **Live** — deploy on Minikube today | Full stack: backend, postgres, redis, HPA | -| `overlays/staging/` | Scaffolding | ECR image placeholder; live deploy in Phase 5 | -| `overlays/production/` | Scaffolding | Same as staging | -| `blue-green/` | Scaffolding | Traffic switch automation in Phase 5 | -| `argocd/` | Scaffolding | Auto-sync wired in Phase 5 | +| `overlays/dev/` | **Live** — manual Minikube | Full stack: backend, postgres, redis, HPA | +| `overlays/staging/` | **Live** — ArgoCD + CD | Blue/green, ECR images, sync waves | +| `overlays/production/` | Scaffolding | CD deferred — [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) | +| `blue-green/` | Wired in staging | Traffic switch via `scripts/blue-green-switch.sh` | +| `argocd/` | Staging Application active | Bootstrap: [argocd/install-notes.md](argocd/install-notes.md) | ## Deploy to Minikube (dev) @@ -54,9 +55,29 @@ curl http://localhost:3000/health Add to `/etc/hosts`: `127.0.0.1 flags.dev.local` and run `minikube tunnel`. -## Staging / Production +## Staging (EKS) — GitOps CD -Overlays exist as scaffolding. Replace `` in `kustomization.yaml` with your AWS account ID after Phase 4 ECR push. Live deploy deferred to Phase 5. +**One-time bootstrap:** [k8s/argocd/install-notes.md](argocd/install-notes.md) + +After bootstrap, merge to `main` triggers: + +1. CI builds and pushes `sha-` to staging ECR +2. CD workflow bumps idle blue/green color tag in `k8s/overlays/staging/` +3. ArgoCD syncs → health check → Istio traffic switch + +Manual validation: + +```bash +kubectl apply -k k8s/overlays/staging --dry-run=server +kubectl get pods,virtualservice -n staging +kubectl get application feature-flags-staging -n argocd +``` + +GitHub variables: `EKS_CLUSTER_NAME`, `STAGING_HEALTH_URL`. + +## Production + +Overlay and ArgoCD Application exist in Git. Automatic deploy disabled until [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) is activated. ## Troubleshooting @@ -70,3 +91,4 @@ Quick reference — full runbook: [TROUBLESHOOTING.md](../TROUBLESHOOTING.md) | Istio treats Postgres/Redis as HTTP | Name service ports with `tcp-` prefix and set `appProtocol: tcp` — see `base/postgres.yaml`, `base/redis.yaml` | | Stale image after rebuild | Scale to 0, `minikube ssh "docker rmi -f backend:latest"`, `minikube image load backend:latest` | | Prisma config / migrate errors | See [TROUBLESHOOTING.md § Prisma v7](../TROUBLESHOOTING.md#prisma-v7) | +| CD workflow fails on EKS | See [TROUBLESHOOTING.md § EKS staging CD](../TROUBLESHOOTING.md#eks-staging-cd-phase-5) |