diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml new file mode 100644 index 0000000..d93941c --- /dev/null +++ b/.github/workflows/cd.yaml @@ -0,0 +1,150 @@ +name: CD + +on: + workflow_run: + workflows: ['CI'] + types: [completed] + branches: [main] + +concurrency: + group: cd-main + cancel-in-progress: false + +permissions: + contents: write + id-token: write + +env: + NAMESPACE: staging + OVERLAY_PATH: k8s/overlays/staging + +jobs: + gate: + name: Gate on CI success + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'success' + outputs: + sha_tag: sha-${{ github.event.workflow_run.head_sha }} + steps: + - name: Validate CI conclusion + run: echo "Deploying ${{ github.event.workflow_run.head_sha }}" + + gitops-bump: + name: GitOps image bump + needs: gate + runs-on: ubuntu-latest + outputs: + idle_color: ${{ steps.bump.outputs.idle_color }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: main + token: ${{ secrets.CD_BOT_TOKEN || github.token }} + + - name: Bump idle color image tag + id: bump + run: | + OUTPUT="$(./scripts/gitops-bump.sh "${{ needs.gate.outputs.sha_tag }}")" + eval "$OUTPUT" + echo "idle_color=${idle_color}" >> "$GITHUB_OUTPUT" + + - name: Commit and push tag bump + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "${{ env.OVERLAY_PATH }}/kustomization.yaml" + git commit -m "chore(cd): bump staging ${{ steps.bump.outputs.idle_color }} to ${{ needs.gate.outputs.sha_tag }} [skip ci]" + git push + + deploy: + name: Sync, health check, traffic switch + needs: [gate, gitops-bump] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: main + token: ${{ secrets.CD_BOT_TOKEN || github.token }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-staging-cicd-role + aws-region: ${{ vars.AWS_REGION }} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --name "${{ vars.EKS_CLUSTER_NAME }}" \ + --region "${{ vars.AWS_REGION }}" + + - name: Wait for ArgoCD sync and rollout + run: ./scripts/blue-green-health.sh "${{ needs.gitops-bump.outputs.idle_color }}" "${{ env.NAMESPACE }}" + + - name: Switch traffic to new color + run: | + ./scripts/blue-green-switch.sh "${{ needs.gitops-bump.outputs.idle_color }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add k8s/blue-green/virtualservice.yaml k8s/overlays/staging/active-color.yaml + git commit -m "chore(cd): switch staging traffic to ${{ needs.gitops-bump.outputs.idle_color }} [skip ci]" + git push + + - name: Public health check + if: ${{ vars.STAGING_HEALTH_URL != '' }} + run: | + for i in $(seq 1 30); do + if curl -fsS "${{ vars.STAGING_HEALTH_URL }}"; then + echo "Public health check passed" + exit 0 + fi + sleep 10 + done + echo "Public health check failed" >&2 + exit 1 + + rollback: + name: Rollback on failure + needs: [gate, gitops-bump, deploy] + if: failure() && needs.gate.result == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: main + fetch-depth: 0 + token: ${{ secrets.CD_BOT_TOKEN || github.token }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-staging-cicd-role + aws-region: ${{ vars.AWS_REGION }} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --name "${{ vars.EKS_CLUSTER_NAME }}" \ + --region "${{ vars.AWS_REGION }}" + + - name: Revert traffic and active color + run: | + ACTIVE="$(grep 'ACTIVE_COLOR:' k8s/overlays/staging/active-color.yaml | awk '{print $2}')" + if [[ "$ACTIVE" == "${{ needs.gitops-bump.outputs.idle_color }}" ]]; then + ./scripts/rollback.sh + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add k8s/blue-green/virtualservice.yaml k8s/overlays/staging/active-color.yaml + git diff --cached --quiet || git commit -m "chore(cd): rollback staging traffic [skip ci]" + git push || true + fi + + - name: Revert image tag bump commit + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + BUMP_SHA="$(git log -1 --grep 'bump staging' --format=%H || true)" + if [[ -n "$BUMP_SHA" ]]; then + git revert --no-edit "$BUMP_SHA" || true + git push || true + fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..579eedf --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,160 @@ +name: CI + +on: + push: + branches: ['**'] + pull_request: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # Prisma 7 loads prisma.config.ts at postinstall; no .env file in CI + DATABASE_URL: postgresql://ci:ci@localhost:5432/ci + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + + - name: Install yamllint + run: pip install yamllint + + - name: Install backend dependencies + working-directory: app/backend + run: bun install --frozen-lockfile + + - name: ESLint + working-directory: app/backend + run: bun run lint:ci + + - name: yamllint + run: | + yamllint -c .yamllint k8s/ + yamllint -c .yamllint .github/ + + - name: Terraform fmt + run: terraform fmt -check -recursive infrastructure/ + + test: + name: Unit Tests + runs-on: ubuntu-latest + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - name: Cache Bun dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.bun/install/cache + key: bun-${{ hashFiles('app/backend/bun.lock') }} + restore-keys: | + bun- + + - name: Install backend dependencies + working-directory: app/backend + run: bun install --frozen-lockfile + + - name: Jest with coverage + working-directory: app/backend + run: bun run test:ci + + - name: Upload coverage artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: coverage + path: app/backend/coverage/ + retention-days: 7 + + # SonarCloud analysis runs via the SonarCloud GitHub App (check: "SonarCloud Code Analysis"). + # A manual scanner job conflicts when Automatic Analysis is enabled on the free plan. + + build-and-push: + name: Build, Scan & Push + needs: [lint, test] + runs-on: ubuntu-latest + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: app/backend + push: false + load: true + tags: backend:ci + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Trivy image scan + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: backend:ci + format: table + severity: CRITICAL + ignore-unfixed: true + exit-code: '1' + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-staging-cicd-role + aws-region: ${{ vars.AWS_REGION }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2 + id: ecr-login + + - name: Set image tags + id: tags + run: | + REGISTRY="${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com" + STAGING="${REGISTRY}/${{ vars.PROJECT_NAME }}-staging-backend" + echo "staging=${STAGING}" >> "$GITHUB_OUTPUT" + echo "sha_tag=sha-${{ github.sha }}" >> "$GITHUB_OUTPUT" + + - name: Push to staging ECR + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: app/backend + push: true + tags: | + ${{ steps.tags.outputs.staging }}:${{ steps.tags.outputs.sha_tag }} + ${{ steps.tags.outputs.staging }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Push to production ECR + if: github.ref == 'refs/heads/main' && vars.ENABLE_PROD_ECR == 'true' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: app/backend + push: true + tags: | + ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:${{ steps.tags.outputs.sha_tag }} + ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ vars.PROJECT_NAME }}-production-backend:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/AGENTS.md b/AGENTS.md index 4bdc83a..1d521d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ Entry point for AI agents working on this repository. Read this file first, then | Dev cluster | Minikube (local) | | Staging / Production | EKS on AWS | -**Current phase:** Phase 3 complete. **Next:** Phase 4 — CI Pipeline. See [STATUS.md](STATUS.md). +**Current phase:** Phase 5 implemented (staging-first CD). **Next:** Validate CD on EKS staging, then Phase 6 — Security Hardening. See [STATUS.md](STATUS.md). --- @@ -33,7 +33,8 @@ devsecops-platform/ ├── app/frontend/ # Empty — reserved for future UI ├── infrastructure/ # Terraform modules + staging/production envs ├── k8s/ # Kustomize manifests (base + overlays + blue-green + argocd) -├── .github/workflows/ # Empty — Phase 4 +├── .github/workflows/ # ci.yaml, cd.yaml, security.yaml, terraform.yaml +├── scripts/ # blue/green CD automation (Phase 5) ├── docs/ # Placeholder — full docs in Phase 9 ├── images/phase-N/ # Screenshot evidence per phase ├── docker-compose.yml # Local dev stack (postgres + redis + backend) @@ -117,14 +118,12 @@ Summary: | Deferred to | Do not implement yet | |-------------|---------------------| -| Phase 4 | GitHub Actions CI, ECR push, SonarCloud gate | -| Phase 5 | ArgoCD auto-sync, CD image-tag updates, blue/green traffic switch scripts | -| Phase 5 | Live EKS deploy of the application | | Phase 6 | Kyverno policies, NetworkPolicy, External Secrets, manifest `securityContext` | | Phase 7 | Grafana dashboards, Loki, custom Prometheus ServiceMonitors | | Phase 9 | Full `docs/`, ADRs, architecture diagrams | +| End of PLAN | Production CD auto-sync — see [docs/cd-production-promotion.md](docs/cd-production-promotion.md) | -**Scaffolding exists but is not wired:** `k8s/argocd/application-*.yaml`, `k8s/blue-green/`, `k8s/overlays/staging|production/` — do not connect to live CD until Phase 5. +**Phase 5 (staging-first):** CD auto-deploys to **staging EKS only**. Dev is manual. Run EKS bootstrap once: [k8s/argocd/install-notes.md](k8s/argocd/install-notes.md). Set GitHub vars `EKS_CLUSTER_NAME`, `STAGING_HEALTH_URL`. --- @@ -144,6 +143,10 @@ Summary: | Terraform staging/prod | `infrastructure/environments/staging/`, `production/` | | Phase roadmap | `PLAN.md` | | Phase progress | `STATUS.md` | +| CI / CD workflows | `.github/workflows/ci.yaml`, `.github/workflows/cd.yaml` | +| CD scripts | `scripts/gitops-bump.sh`, `scripts/blue-green-switch.sh`, `scripts/rollback.sh` | +| SonarCloud config | `sonar-project.properties` | +| GitHub OIDC (Terraform) | `infrastructure/modules/github-oidc/` | --- @@ -157,12 +160,12 @@ Summary: --- -## Tech Stack Note for CI (Phase 4) +## Tech Stack Note — CI (Phase 4, done) -The application is **NestJS / TypeScript / Jest / Bun**, not Python. When implementing Phase 4 CI: +The application is **NestJS / TypeScript / Jest / Bun**, not Python: -- Lint: ESLint (not ruff) -- Tests: Jest with coverage (not pytest) -- SonarCloud: TypeScript sources under `app/backend/src/` +- Lint: ESLint via `bun run lint:ci` (not ruff) +- Tests: Jest with coverage via `bun run test:ci` (not pytest) +- SonarCloud: TypeScript sources under `app/backend/src/`; quality gate via GitHub App check on free plan -Some older references in PLAN.md Phase 4 may still mention Python — treat NestJS/TypeScript as the source of truth. +Some older references in [PLAN.md](PLAN.md) may still mention Python — treat NestJS/TypeScript as the source of truth. diff --git a/README.md b/README.md index ee599e7..d422b7c 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,9 @@ The application is a **Feature Flag Service** — manage feature toggles per env | 1 | AWS Infrastructure (Terraform) | Done | | 2 | Kubernetes base (Minikube + Istio) | Done | | 3 | Feature Flag Service + K8s manifests | **Done** | -| 4 | CI Pipeline (GitHub Actions, SonarCloud) | Next | -| 5–9 | CD, Security, Monitoring, DR, Docs | Planned | +| 4 | CI Pipeline (GitHub Actions, SonarCloud, ECR) | **Done** | +| 5 | CD Pipeline & GitOps (ArgoCD, staging-first) | **Implemented** — validate on EKS | +| 6–9 | Security, Monitoring, DR, Docs | Planned | Full tracker: [STATUS.md](STATUS.md) · Full roadmap: [PLAN.md](PLAN.md) diff --git a/STATUS.md b/STATUS.md index 37dbc24..bb0273a 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,8 +1,8 @@ # Project Status -Phase tracker for the DevSecOps platform. Updated after Phase 3 completion. +Phase tracker for the DevSecOps platform. Updated after Phase 5 implementation. -**Active work:** Phase 4 — CI Pipeline +**Active work:** Phase 5 validation on EKS staging (screenshots) **Agent entry point:** [AGENTS.md](AGENTS.md) **Deploy runbook:** [TROUBLESHOOTING.md](TROUBLESHOOTING.md) **Full roadmap:** [PLAN.md](PLAN.md) @@ -17,8 +17,8 @@ Phase tracker for the DevSecOps platform. Updated after Phase 3 completion. | 1 | Terraform & AWS (VPC, EKS, ECR, IAM) | Done | `infrastructure/`, `images/phase-1/` | | 2 | Kubernetes base (Minikube: Istio, ArgoCD, Ingress) | Done (Minikube) | `k8s/namespaces/`, `images/phase-2/`, install notes in `k8s/argocd/`, `k8s/istio/` | | 3 | Feature Flag Service + K8s manifests | **Done** | `app/backend/`, `k8s/`, `images/phase-3/` | -| 4 | CI Pipeline (GitHub Actions, SonarCloud, ECR) | Not started | `.github/workflows/` empty | -| 5 | CD & GitOps (ArgoCD sync, blue/green automation) | Not started | Scaffolding in `k8s/argocd/`, `k8s/blue-green/` | +| 4 | CI Pipeline (GitHub Actions, SonarCloud, ECR) | **Done** | `.github/workflows/`, `sonar-project.properties`, `images/phase-4/` | +| 5 | CD & GitOps (ArgoCD sync, blue/green automation) | **Implemented** (staging-first) | `.github/workflows/cd.yaml`, `scripts/`, `images/phase-5/` | | 6 | Security hardening (Kyverno, Cosign, NetworkPolicy) | Not started | — | | 7 | Monitoring & SRE (Prometheus, Grafana, Loki) | Not started | `monitoring/` placeholder | | 8 | DR & resilience (Velero, k6) | Not started | — | @@ -26,78 +26,52 @@ Phase tracker for the DevSecOps platform. Updated after Phase 3 completion. --- -## Phase 3 — Completion Details +## Phase 5 — Implementation Details (staging-first) -All criteria met. See [PLAN.md § Phase 3 Completion Criteria](PLAN.md) for the full checklist (marked `[x]`). +Automatic CD targets **staging EKS only**. Dev (Minikube) remains manual. Production CD deferred — see [docs/cd-production-promotion.md](docs/cd-production-promotion.md). -### Application +### Delivered -- NestJS API with flags, audit, health, metrics modules -- Redis cache on flag reads; PostgreSQL source of truth + audit log -- Structured JSON logging (pino) with request IDs -- Docker Compose local stack; multi-stage Dockerfile with non-root user +- [`.github/workflows/cd.yaml`](.github/workflows/cd.yaml) — post-CI GitOps bump, rollout health check, traffic switch, rollback +- [`scripts/`](scripts/) — `gitops-bump.sh`, `blue-green-switch.sh`, `blue-green-health.sh`, `rollback.sh` +- [`k8s/overlays/staging/`](k8s/overlays/staging/) — blue/green overlay, sync waves, `cd-active-color` ConfigMap +- [`k8s/argocd/install-notes.md`](k8s/argocd/install-notes.md) — EKS staging bootstrap runbook +- Terraform EKS access entry for staging `cicd` role ([`infrastructure/modules/eks/`](infrastructure/modules/eks/)) +- CI: `[skip ci]` guard; production ECR gated by `ENABLE_PROD_ECR` (default off) -### Kubernetes — dev (Minikube) +### GitHub variables required -- `kubectl apply -k k8s/overlays/dev` deploys backend, postgres, redis, HPA -- LimitRange, ResourceQuota, ServiceAccount, Istio TCP DestinationRules applied -- HPA: `minReplicas: 1` (dev patch), `maxReplicas: 10` (base) +| Variable | Example | +|----------|---------| +| `EKS_CLUSTER_NAME` | `mck21-devsecops-staging-eks` | +| `STAGING_HEALTH_URL` | `https:///health` | -### Kubernetes — scaffolding (not live-deployed) +Optional: `ENABLE_PROD_ECR=true`, secret `CD_BOT_TOKEN` if branch protection blocks `GITHUB_TOKEN`. -- `k8s/overlays/staging/` and `k8s/overlays/production/` with ECR placeholders -- `k8s/argocd/application-{dev,staging,production}.yaml` -- `k8s/blue-green/` (deployments + VirtualService + DestinationRule) +### Pending validation -### Screenshots - -Located in `images/phase-3/`: - -| File | Content | -|------|---------| -| `01-docker-compose-running.png` | Docker Compose stack up | -| `02-post-create-flag.png` | POST /api/flags | -| `03-patch-toggle-flag.png` | PATCH toggle | -| `04-get-audit-history.png` | GET /api/audit | -| `05-get-prometheus-metrics.png` | GET /metrics | -| `06-health-checks.png` | GET /health | -| `07-k8s-pods-dev.png` | Pods running in dev namespace | - ---- - -## Active Work — Phase 4 - -**Goal:** Every push triggers lint, test, SonarCloud quality gate, security scans, Docker build, and ECR push. - -**Key deliverables:** - -- `.github/workflows/ci.yaml` — main pipeline -- `.github/workflows/security.yaml` — standalone security scans -- `.github/workflows/terraform.yaml` — terraform validate/plan -- `sonar-project.properties` — SonarCloud config for TypeScript -- AWS OIDC auth for ECR push (no long-lived keys) - -**Stack correction:** Application is NestJS/TypeScript/Jest — not Python. See [AGENTS.md § Tech Stack Note](AGENTS.md). +- [ ] One-time EKS staging bootstrap per install notes +- [ ] `terraform apply` in staging for EKS access entry +- [ ] Merge to `main` → CI → CD green +- [ ] Screenshots in [`images/phase-5/`](images/phase-5/) per README checklist --- ## Blockers / Deferred -| Item | Reason | Target Phase | -|------|--------|--------------| -| Live EKS app deploy | Requires CI image push + CD wiring | 4 + 5 | -| ArgoCD auto-sync | CD pipeline not built | 5 | -| EKS platform components (Ingress, Cert Manager on EKS) | Partially done in Phase 2; full parity deferred | 2 EKS + 5 | -| `infrastructure/environments/dev/` | Empty by design — dev runs on Minikube | N/A | -| Architecture diagram, full deployment guide | Portfolio polish | 9 | -| Grafana dashboards | Monitoring stack not installed | 7 | +| Item | Reason | Target | +|------|--------|--------| +| Production CD | Staging-first cost/risk strategy | [docs/cd-production-promotion.md](docs/cd-production-promotion.md) | +| Dev automated CD | Minikube not reachable from GitHub-hosted runners | Manual dev deploy | +| Architecture diagram, full deployment guide | Portfolio polish | Phase 9 | +| Grafana dashboards | Monitoring stack not installed | Phase 7 | --- ## Environment Strategy -| Environment | Kubernetes | Status | -|-------------|------------|--------| -| `dev` | Minikube (local) | App deployed and validated | -| `staging` | EKS (AWS) | Terraform ready; app not deployed | -| `production` | EKS (AWS) | Terraform ready; app not deployed | +| Environment | Kubernetes | CD | +|-------------|------------|-----| +| `dev` | Minikube (local) | Manual | +| `staging` | EKS (AWS) | **Automatic** (merge to `main`) | +| `production` | EKS (AWS) | Deferred | diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 8f9f783..6d271f6 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -409,6 +409,51 @@ Local stack does not use Istio. Most K8s/Istio issues do not apply. --- +## EKS staging CD (Phase 5) + +### Symptom: CD workflow fails on `git push` + +**Cause:** Branch protection blocks `GITHUB_TOKEN` writes to `main`. + +**Fix:** Allow GitHub Actions in branch rules, or set repository secret `CD_BOT_TOKEN` (PAT with `contents: write`) and re-run CD. + +### Symptom: CD fails at `Configure kubectl` or health check + +**Cause:** Staging `cicd` role lacks EKS access entry, or wrong cluster name. + +**Fix:** + +```bash +cd infrastructure/environments/staging +terraform apply # creates aws_eks_access_entry for cicd role +``` + +Set GitHub variable `EKS_CLUSTER_NAME=mck21-devsecops-staging-eks`. + +### Symptom: ArgoCD OutOfSync on VirtualService weights + +**Cause:** Traffic switch updated Git; ArgoCD self-heal reconciled correctly — verify weights match `k8s/blue-green/virtualservice.yaml`. + +**Fix:** `kubectl get virtualservice backend -n staging -o yaml`. Source of truth is Git. + +### Symptom: Idle color pods ImagePullBackOff + +**Cause:** ECR tag from CD bump not pushed yet, or node lacks ECR pull permissions. + +**Fix:** Confirm CI pushed `sha-` to staging ECR. Check node IAM role includes ECR read. + +### Symptom: Public health check fails after traffic switch + +**Cause:** DNS not pointing at ingress LoadBalancer, or cert-manager pending. + +**Fix:** Use ALB hostname in `STAGING_HEALTH_URL` until DNS is ready. Check `kubectl get ingress -n staging` and cert-manager Certificate status. + +### Bootstrap reference + +Full one-time EKS staging setup: [k8s/argocd/install-notes.md](k8s/argocd/install-notes.md). + +--- + ## Related Files | Topic | File | @@ -424,6 +469,8 @@ Local stack does not use Istio. Most K8s/Istio issues do not apply. | Dev ResourceQuota patch | [`k8s/overlays/dev/patch-resourcequota.yaml`](k8s/overlays/dev/patch-resourcequota.yaml) | | Secret template | [`k8s/base/secret.yaml`](k8s/base/secret.yaml) | | Istio install notes | [`k8s/istio/istio-notes.md`](k8s/istio/istio-notes.md) | +| CD scripts | [`scripts/`](scripts/) | +| Production CD (deferred) | [`docs/cd-production-promotion.md`](docs/cd-production-promotion.md) | --- diff --git a/docs/cd-production-promotion.md b/docs/cd-production-promotion.md new file mode 100644 index 0000000..b0d65fe --- /dev/null +++ b/docs/cd-production-promotion.md @@ -0,0 +1,61 @@ +# Production CD promotion (deferred) + +Production deploy is **intentionally disabled** during Phase 5 build to keep the pipeline green and AWS costs low. Staging is the sole automated CD target until the rest of the PLAN is complete. + +## When to enable + +After Phase 5 staging CD is stable and remaining PLAN phases are done (or when you explicitly choose to turn on multi-env). + +## Steps + +### 1. GitHub repository variables + +| Variable | Value | +|----------|-------| +| `ENABLE_PROD_ECR` | `true` — restores production ECR push on `main` in CI | +| `ENABLE_PROD_CD` | `true` — enable when the promote job is wired (future) | +| `PRODUCTION_HEALTH_URL` | `https://flags.example.com/health` | + +### 2. Bootstrap production EKS platform + +Repeat the staging bootstrap from [k8s/argocd/install-notes.md](../k8s/argocd/install-notes.md) on the production cluster: + +- Istio production profile +- Ingress NGINX + cert-manager + Let's Encrypt issuer +- ArgoCD Helm install +- `backend-secrets` in namespace `production` + +### 3. Register ArgoCD Application + +```bash +kubectl apply -f k8s/argocd/application-production.yaml +``` + +### 4. Add promote job to CD workflow + +Add a job (manual or automatic) after staging public health check succeeds: + +1. Read the validated `sha-` from the staging bump commit +2. Update `k8s/overlays/production/kustomization.yaml` `newTag` +3. Commit `chore(cd): promote production to sha-xxx [skip ci]` +4. Wait for ArgoCD sync on production +5. `curl -f $PRODUCTION_HEALTH_URL` + +Example trigger: + +```yaml +promote-production: + needs: deploy + if: vars.ENABLE_PROD_CD == 'true' + # workflow_dispatch for manual approval is recommended first +``` + +### 5. Blue/green on production (optional) + +Reuse `scripts/blue-green-switch.sh` and production overlay changes mirroring staging if zero-downtime promotion is required on production. + +## Rollback + +- Revert the production GitOps commit +- ArgoCD history rollback: `argocd app rollback feature-flags-production` +- Keep staging as the pre-production gate — never promote a SHA that failed staging diff --git a/images/phase-5/README.md b/images/phase-5/README.md new file mode 100644 index 0000000..7fd1c69 --- /dev/null +++ b/images/phase-5/README.md @@ -0,0 +1,32 @@ +# Phase 5 — CD & GitOps evidence + +Capture screenshots after the staging CD pipeline runs end-to-end on EKS. + +## Checklist + +| # | File | Content | +|---|------|---------| +| 1 | `01-cd-workflow-green.png` | GitHub Actions CD workflow success with GitOps commits | +| 2 | `02-argocd-staging-synced.png` | ArgoCD UI — `feature-flags-staging` Synced + Healthy | +| 3 | `03-blue-green-switch.png` | ArgoCD or terminal during traffic switch | +| 4 | `04-virtualservice-weights.png` | `kubectl get virtualservice backend -n staging -o yaml` showing 100/0 split | +| 5 | `05-rollback-complete.png` | Rollback job after simulated health failure (optional) | + +## Manual validation commands + +```bash +# After merge to main +gh run list --workflow=CD --limit 3 + +# Cluster +kubectl get pods,virtualservice -n staging +kubectl get application feature-flags-staging -n argocd + +# Health +curl -f "$STAGING_HEALTH_URL" +``` + +## Notes + +- Dev (Minikube) CD is manual during Phase 5 — no screenshot required for phase closure. +- Production ArgoCD screenshots deferred until [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) is activated. diff --git a/infrastructure/environments/staging/main.tf b/infrastructure/environments/staging/main.tf index b1d62e9..ea5f739 100644 --- a/infrastructure/environments/staging/main.tf +++ b/infrastructure/environments/staging/main.tf @@ -60,6 +60,7 @@ module "eks" { node_desired_size = 2 node_min_size = 1 node_max_size = 4 + cicd_role_arn = module.iam.cicd_role_arn } module "ecr" { diff --git a/infrastructure/modules/eks/main.tf b/infrastructure/modules/eks/main.tf index 6a22c2b..05dec2e 100644 --- a/infrastructure/modules/eks/main.tf +++ b/infrastructure/modules/eks/main.tf @@ -79,4 +79,25 @@ resource "aws_eks_node_group" "main" { depends_on = [ aws_eks_cluster.main ] +} + +# --- CI/CD cluster access (GitHub Actions kubectl) --- +resource "aws_eks_access_entry" "cicd" { + count = var.cicd_role_arn != "" ? 1 : 0 + + cluster_name = aws_eks_cluster.main.name + principal_arn = var.cicd_role_arn + type = "STANDARD" +} + +resource "aws_eks_access_policy_association" "cicd_admin" { + count = var.cicd_role_arn != "" ? 1 : 0 + + cluster_name = aws_eks_cluster.main.name + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + principal_arn = aws_eks_access_entry.cicd[0].principal_arn + + access_scope { + type = "cluster" + } } \ No newline at end of file diff --git a/infrastructure/modules/eks/variables.tf b/infrastructure/modules/eks/variables.tf index 51a2472..1e7ded2 100644 --- a/infrastructure/modules/eks/variables.tf +++ b/infrastructure/modules/eks/variables.tf @@ -66,4 +66,10 @@ variable "node_max_size" { description = "Maximum number of worker nodes" type = number default = 4 +} + +variable "cicd_role_arn" { + description = "IAM role ARN for GitHub Actions CD (EKS access entry). Empty disables access entry." + type = string + default = "" } \ No newline at end of file diff --git a/k8s/README.md b/k8s/README.md index d3338fc..d03d097 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -10,11 +10,12 @@ Kustomize-based manifests for the Feature Flag Service. k8s/ ├── base/ # env-agnostic resources (no namespace) ├── overlays/ -│ ├── dev/ # Minikube — deploy now -│ ├── staging/ # EKS scaffolding -│ └── production/ # EKS scaffolding -├── blue-green/ # Istio blue/green skeleton (Phase 5 automation) -├── argocd/ # Application CRDs (wired in Phase 5) +│ ├── dev/ # Minikube — deploy manually +│ ├── staging/ # EKS — GitOps CD target (Phase 5) +│ └── production/ # EKS — deferred CD +├── blue-green/ # Istio blue/green (included in staging overlay) +├── argocd/ # Application CRDs +├── platform/ # EKS platform manifests (cert-manager issuer) └── namespaces/ # Platform namespaces ``` @@ -22,11 +23,11 @@ k8s/ | Path | Status | Notes | |------|--------|-------| -| `overlays/dev/` | **Live** — deploy on Minikube today | Full stack: backend, postgres, redis, HPA | -| `overlays/staging/` | Scaffolding | ECR image placeholder; live deploy in Phase 5 | -| `overlays/production/` | Scaffolding | Same as staging | -| `blue-green/` | Scaffolding | Traffic switch automation in Phase 5 | -| `argocd/` | Scaffolding | Auto-sync wired in Phase 5 | +| `overlays/dev/` | **Live** — manual Minikube | Full stack: backend, postgres, redis, HPA | +| `overlays/staging/` | **Live** — ArgoCD + CD | Blue/green, ECR images, sync waves | +| `overlays/production/` | Scaffolding | CD deferred — [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) | +| `blue-green/` | Wired in staging | Traffic switch via `scripts/blue-green-switch.sh` | +| `argocd/` | Staging Application active | Bootstrap: [argocd/install-notes.md](argocd/install-notes.md) | ## Deploy to Minikube (dev) @@ -54,9 +55,29 @@ curl http://localhost:3000/health Add to `/etc/hosts`: `127.0.0.1 flags.dev.local` and run `minikube tunnel`. -## Staging / Production +## Staging (EKS) — GitOps CD -Overlays exist as scaffolding. Replace `` in `kustomization.yaml` with your AWS account ID after Phase 4 ECR push. Live deploy deferred to Phase 5. +**One-time bootstrap:** [k8s/argocd/install-notes.md](argocd/install-notes.md) + +After bootstrap, merge to `main` triggers: + +1. CI builds and pushes `sha-` to staging ECR +2. CD workflow bumps idle blue/green color tag in `k8s/overlays/staging/` +3. ArgoCD syncs → health check → Istio traffic switch + +Manual validation: + +```bash +kubectl apply -k k8s/overlays/staging --dry-run=server +kubectl get pods,virtualservice -n staging +kubectl get application feature-flags-staging -n argocd +``` + +GitHub variables: `EKS_CLUSTER_NAME`, `STAGING_HEALTH_URL`. + +## Production + +Overlay and ArgoCD Application exist in Git. Automatic deploy disabled until [docs/cd-production-promotion.md](../docs/cd-production-promotion.md) is activated. ## Troubleshooting @@ -70,3 +91,4 @@ Quick reference — full runbook: [TROUBLESHOOTING.md](../TROUBLESHOOTING.md) | Istio treats Postgres/Redis as HTTP | Name service ports with `tcp-` prefix and set `appProtocol: tcp` — see `base/postgres.yaml`, `base/redis.yaml` | | Stale image after rebuild | Scale to 0, `minikube ssh "docker rmi -f backend:latest"`, `minikube image load backend:latest` | | Prisma config / migrate errors | See [TROUBLESHOOTING.md § Prisma v7](../TROUBLESHOOTING.md#prisma-v7) | +| CD workflow fails on EKS | See [TROUBLESHOOTING.md § EKS staging CD](../TROUBLESHOOTING.md#eks-staging-cd-phase-5) | diff --git a/k8s/argocd/install-notes.md b/k8s/argocd/install-notes.md index c9f589a..411879a 100644 --- a/k8s/argocd/install-notes.md +++ b/k8s/argocd/install-notes.md @@ -1,13 +1,138 @@ # ArgoCD Installation ## Dev (Minikube) + Installed via official stable manifest: + +```bash kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml +``` Access: + +```bash kubectl port-forward svc/argocd-server -n argocd 8080:443 -URL: https://localhost:8080 -User: admin +``` + +- URL: https://localhost:8080 +- User: `admin` +- Password: `kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d` + +Register the dev app (optional, local GitOps): + +```bash +kubectl apply -f k8s/argocd/application-dev.yaml +``` + +## Staging (EKS) — Phase 5 bootstrap + +Prerequisites: `kubectl` context pointing at the staging EKS cluster (`mck21-devsecops-staging-eks`). + +### 1. Namespaces + +```bash +kubectl apply -f k8s/namespaces/namespaces.yaml +``` + +### 2. Istio (production profile) + +```bash +istioctl install --set profile=production -y +kubectl label namespace staging istio-injection=enabled --overwrite +``` + +See [k8s/istio/istio-notes.md](../istio/istio-notes.md). + +### 3. Ingress NGINX + +```bash +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx --create-namespace \ + --set controller.service.type=LoadBalancer +``` + +### 4. cert-manager + Let's Encrypt issuer + +```bash +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --create-namespace \ + --set crds.enabled=true + +kubectl apply -f k8s/platform/cluster-issuer-letsencrypt.yaml +``` + +### 5. Application secrets (never commit real values) + +```bash +kubectl apply -f k8s/base/secret.yaml -n staging +# Or create from template with env-specific DATABASE_URL: +# kubectl create secret generic backend-secrets -n staging \ +# --from-literal=DATABASE_URL='postgresql://...' +``` + +### 6. ArgoCD (Helm) + +```bash +helm repo add argo https://argoproj.github.io/argo-helm +helm repo update +helm upgrade --install argocd argo/argo-cd \ + --namespace argocd --create-namespace \ + --set configs.params.server.insecure=true +``` + +Initial admin password: + +```bash +kubectl -n argocd get secret argocd-initial-admin-secret \ + -o jsonpath='{.data.password}' | base64 -d && echo +``` + +Port-forward UI: + +```bash +kubectl port-forward svc/argocd-server -n argocd 8080:443 +``` + +### 7. Connect Git repository + +In ArgoCD UI → Settings → Repositories → Connect repo: + +- URL: `https://github.com/mck21/devsecops-platform.git` +- Public repo: no credentials required; private repo: PAT or deploy key + +Or CLI: + +```bash +argocd login localhost:8080 --username admin --password --insecure +argocd repo add https://github.com/mck21/devsecops-platform.git +``` + +### 8. Register staging Application (auto-sync) + +```bash +# Validate manifests first +kubectl apply -k k8s/overlays/staging --dry-run=server + +kubectl apply -f k8s/argocd/application-staging.yaml +``` + +Do **not** apply `application-production.yaml` until production CD is enabled (end of PLAN). + +### 9. GitHub Actions CD variables + +Set repository variables in GitHub → Settings → Variables: + +| Variable | Example | +|----------|---------| +| `STAGING_HEALTH_URL` | `https://flags.staging.example.com/health` (or ALB hostname) | +| `EKS_CLUSTER_NAME` | `mck21-devsecops-staging-eks` | + +Ensure branch protection allows GitHub Actions to push GitOps commits to `main`, or configure `CD_BOT_TOKEN` secret. + +## Production (EKS) -## Staging/Production (EKS) -Installed via Helm in Phase 5. \ No newline at end of file +ArgoCD and app registration deferred until multi-env promotion is enabled. See [docs/cd-production-promotion.md](../../docs/cd-production-promotion.md). diff --git a/k8s/blue-green/deployment-blue.yaml b/k8s/blue-green/deployment-blue.yaml index e967720..c57980e 100644 --- a/k8s/blue-green/deployment-blue.yaml +++ b/k8s/blue-green/deployment-blue.yaml @@ -1,5 +1,4 @@ -# Blue/green skeleton — not included in dev overlay by default. -# Switch traffic via VirtualService in Phase 5 automation. +# Blue/green — wired in staging overlay; traffic switch via scripts/cd pipeline. apiVersion: apps/v1 kind: Deployment metadata: @@ -7,6 +6,8 @@ metadata: labels: app: backend version: blue + annotations: + argocd.argoproj.io/sync-wave: "1" spec: replicas: 1 selector: @@ -22,7 +23,7 @@ spec: serviceAccountName: backend-sa containers: - name: backend - image: backend:latest + image: backend-blue:latest imagePullPolicy: IfNotPresent ports: - containerPort: 3000 diff --git a/k8s/blue-green/deployment-green.yaml b/k8s/blue-green/deployment-green.yaml index 47d2da4..3642804 100644 --- a/k8s/blue-green/deployment-green.yaml +++ b/k8s/blue-green/deployment-green.yaml @@ -5,6 +5,8 @@ metadata: labels: app: backend version: green + annotations: + argocd.argoproj.io/sync-wave: "1" spec: replicas: 1 selector: @@ -20,7 +22,7 @@ spec: serviceAccountName: backend-sa containers: - name: backend - image: backend:latest + image: backend-green:latest imagePullPolicy: IfNotPresent ports: - containerPort: 3000 diff --git a/k8s/blue-green/destinationrule.yaml b/k8s/blue-green/destinationrule.yaml index d6d8617..d48cbdb 100644 --- a/k8s/blue-green/destinationrule.yaml +++ b/k8s/blue-green/destinationrule.yaml @@ -2,6 +2,8 @@ apiVersion: networking.istio.io/v1 kind: DestinationRule metadata: name: backend + annotations: + argocd.argoproj.io/sync-wave: "2" spec: host: backend subsets: diff --git a/k8s/blue-green/virtualservice.yaml b/k8s/blue-green/virtualservice.yaml index daac189..3f86252 100644 --- a/k8s/blue-green/virtualservice.yaml +++ b/k8s/blue-green/virtualservice.yaml @@ -2,6 +2,8 @@ apiVersion: networking.istio.io/v1 kind: VirtualService metadata: name: backend + annotations: + argocd.argoproj.io/sync-wave: "2" spec: hosts: - backend diff --git a/k8s/istio/istio-notes.md b/k8s/istio/istio-notes.md index 241b89f..f67941e 100644 --- a/k8s/istio/istio-notes.md +++ b/k8s/istio/istio-notes.md @@ -1,22 +1,15 @@ -# Istio Installation +## Staging / Production (EKS) -## Dev (Minikube) -Installed via istioctl with demo profile: -istioctl install --set profile=demo -y +Install Istio with the production profile during Phase 5 bootstrap: -Istio version used: 1.30.0 - -Addons installed: -- Kiali (service mesh topology) -- Jaeger (distributed tracing) -- Prometheus (metrics, Istio-scoped) - -Access dashboards: -istioctl dashboard kiali -istioctl dashboard jaeger +```bash +istioctl install --set profile=production -y +kubectl label namespace staging istio-injection=enabled --overwrite +``` Sidecar injection enabled on namespaces: -- dev -## Staging/Production (EKS) -Installed via istioctl with production profile in Phase 5. \ No newline at end of file +- staging +- production + +Full bootstrap: [k8s/argocd/install-notes.md](../argocd/install-notes.md). diff --git a/k8s/overlays/staging/active-color.yaml b/k8s/overlays/staging/active-color.yaml new file mode 100644 index 0000000..30b154c --- /dev/null +++ b/k8s/overlays/staging/active-color.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cd-active-color + annotations: + argocd.argoproj.io/sync-wave: "0" +data: + ACTIVE_COLOR: blue diff --git a/k8s/overlays/staging/kustomization.yaml b/k8s/overlays/staging/kustomization.yaml index df2bd55..2b0b172 100644 --- a/k8s/overlays/staging/kustomization.yaml +++ b/k8s/overlays/staging/kustomization.yaml @@ -3,14 +3,25 @@ kind: Kustomization namespace: staging +commonAnnotations: + argocd.argoproj.io/sync-wave: "0" + resources: - ../../base + - ../../blue-green + - active-color.yaml patches: - - path: patch-deployment.yaml + - path: patch-delete-backend.yaml + - path: patch-delete-hpa.yaml - path: patch-ingress.yaml + - path: patch-blue-deployment.yaml + - path: patch-green-deployment.yaml images: - - name: backend - newName: .dkr.ecr.us-east-1.amazonaws.com/devsecops-staging-backend + - name: backend-blue + newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend + newTag: latest + - name: backend-green + newName: 125156866917.dkr.ecr.us-east-1.amazonaws.com/mck21-devsecops-staging-backend newTag: latest diff --git a/k8s/overlays/staging/patch-deployment.yaml b/k8s/overlays/staging/patch-blue-deployment.yaml similarity index 55% rename from k8s/overlays/staging/patch-deployment.yaml rename to k8s/overlays/staging/patch-blue-deployment.yaml index 69d4041..13d1dd4 100644 --- a/k8s/overlays/staging/patch-deployment.yaml +++ b/k8s/overlays/staging/patch-blue-deployment.yaml @@ -1,10 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: backend + name: backend-blue spec: replicas: 2 template: + metadata: + annotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true spec: containers: - name: backend diff --git a/k8s/overlays/staging/patch-delete-backend.yaml b/k8s/overlays/staging/patch-delete-backend.yaml new file mode 100644 index 0000000..dedb15b --- /dev/null +++ b/k8s/overlays/staging/patch-delete-backend.yaml @@ -0,0 +1,5 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +$patch: delete diff --git a/k8s/overlays/staging/patch-delete-hpa.yaml b/k8s/overlays/staging/patch-delete-hpa.yaml new file mode 100644 index 0000000..4a677fb --- /dev/null +++ b/k8s/overlays/staging/patch-delete-hpa.yaml @@ -0,0 +1,5 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: backend-hpa +$patch: delete diff --git a/k8s/overlays/staging/patch-green-deployment.yaml b/k8s/overlays/staging/patch-green-deployment.yaml new file mode 100644 index 0000000..c1618bb --- /dev/null +++ b/k8s/overlays/staging/patch-green-deployment.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend-green +spec: + replicas: 2 + template: + metadata: + annotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true + spec: + containers: + - name: backend + imagePullPolicy: Always diff --git a/k8s/platform/cluster-issuer-letsencrypt.yaml b/k8s/platform/cluster-issuer-letsencrypt.yaml new file mode 100644 index 0000000..6476d1b --- /dev/null +++ b/k8s/platform/cluster-issuer-letsencrypt.yaml @@ -0,0 +1,16 @@ +# Apply after cert-manager Helm install on EKS staging/production. +# Replace email before first apply. +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@example.com + privateKeySecretRef: + name: letsencrypt-prod-account-key + solvers: + - http01: + ingress: + class: nginx diff --git a/scripts/blue-green-health.sh b/scripts/blue-green-health.sh new file mode 100755 index 0000000..d59d081 --- /dev/null +++ b/scripts/blue-green-health.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +COLOR="${1:?Usage: blue-green-health.sh [namespace]}" +NAMESPACE="${2:-staging}" +DEPLOYMENT="backend-${COLOR}" +TIMEOUT="${TIMEOUT:-300s}" + +echo "Waiting for deployment/${DEPLOYMENT} rollout in namespace ${NAMESPACE}..." +kubectl rollout status "deployment/${DEPLOYMENT}" -n "$NAMESPACE" --timeout="$TIMEOUT" + +POD="$(kubectl get pods -n "$NAMESPACE" -l "app=backend,version=${COLOR}" \ + -o jsonpath='{.items[0].metadata.name}')" + +if [[ -z "$POD" ]]; then + echo "No pod found for color ${COLOR}" >&2 + exit 1 +fi + +echo "Checking /health on pod ${POD}..." +for _ in $(seq 1 30); do + if kubectl exec -n "$NAMESPACE" "$POD" -c backend -- \ + wget -qO- http://127.0.0.1:3000/health >/dev/null 2>&1; then + echo "Health check passed for ${COLOR}" + exit 0 + fi + sleep 5 +done + +echo "Health check failed for ${COLOR}" >&2 +exit 1 diff --git a/scripts/blue-green-switch.sh b/scripts/blue-green-switch.sh new file mode 100755 index 0000000..44852a0 --- /dev/null +++ b/scripts/blue-green-switch.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TARGET_COLOR="${1:?Usage: blue-green-switch.sh }" + +python3 "$ROOT/scripts/traffic_switch.py" "$TARGET_COLOR" diff --git a/scripts/gitops-bump.sh b/scripts/gitops-bump.sh new file mode 100755 index 0000000..9a47086 --- /dev/null +++ b/scripts/gitops-bump.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SHA_TAG="${1:?Usage: gitops-bump.sh }" + +python3 "$ROOT/scripts/gitops_bump.py" "$SHA_TAG" diff --git a/scripts/gitops_bump.py b/scripts/gitops_bump.py new file mode 100644 index 0000000..446411f --- /dev/null +++ b/scripts/gitops_bump.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Update the idle color image tag in staging kustomization.yaml.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ACTIVE_COLOR_FILE = ROOT / "k8s/overlays/staging/active-color.yaml" +KUSTOMIZATION_FILE = ROOT / "k8s/overlays/staging/kustomization.yaml" + + +def read_active_color() -> str: + for line in ACTIVE_COLOR_FILE.read_text().splitlines(): + if line.strip().startswith("ACTIVE_COLOR:"): + return line.split(":", 1)[1].strip() + raise SystemExit("ACTIVE_COLOR not found in active-color.yaml") + + +def idle_color(active: str) -> str: + if active == "blue": + return "green" + if active == "green": + return "blue" + raise SystemExit(f"Invalid ACTIVE_COLOR: {active}") + + +def bump_tag(sha_tag: str) -> tuple[str, str]: + active = read_active_color() + idle = idle_color(active) + image_name = f"backend-{idle}" + + content = KUSTOMIZATION_FILE.read_text() + pattern = rf"(- name: {re.escape(image_name)}\n newName: [^\n]+\n newTag: )[^\n]+" + updated, count = re.subn(pattern, rf"\g<1>{sha_tag}", content, count=1) + if count != 1: + raise SystemExit(f"Failed to update newTag for {image_name}") + + KUSTOMIZATION_FILE.write_text(updated) + return idle, image_name + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("Usage: gitops_bump.py ") + + idle, image_name = bump_tag(sys.argv[1]) + print(f"idle_color={idle}") + print(f"image_name={image_name}") + + +if __name__ == "__main__": + main() diff --git a/scripts/rollback.sh b/scripts/rollback.sh new file mode 100755 index 0000000..7fac518 --- /dev/null +++ b/scripts/rollback.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +NAMESPACE="${NAMESPACE:-staging}" + +CURRENT="$(grep 'ACTIVE_COLOR:' "$ROOT/k8s/overlays/staging/active-color.yaml" | awk '{print $2}')" +if [[ "$CURRENT" == "blue" ]]; then + REVERT_TO="green" +else + REVERT_TO="blue" +fi + +python3 "$ROOT/scripts/traffic_switch.py" "$REVERT_TO" + +echo "Rollback: reverted VirtualService and active-color to ${REVERT_TO}" +echo "Revert the GitOps image bump commit if the idle color image must be rolled back." + +if command -v argocd >/dev/null 2>&1; then + argocd app rollback feature-flags-staging || true +fi + +kubectl rollout status "deployment/backend-${REVERT_TO}" -n "$NAMESPACE" --timeout=120s || true diff --git a/scripts/traffic_switch.py b/scripts/traffic_switch.py new file mode 100644 index 0000000..0ec8112 --- /dev/null +++ b/scripts/traffic_switch.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Switch Istio VirtualService weights and update active color in Git.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ACTIVE_COLOR_FILE = ROOT / "k8s/overlays/staging/active-color.yaml" +VIRTUALSERVICE_FILE = ROOT / "k8s/blue-green/virtualservice.yaml" + + +def switch_to(target: str) -> None: + if target not in {"blue", "green"}: + raise SystemExit("Target color must be blue or green") + + other = "green" if target == "blue" else "blue" + target_weight = 100 + other_weight = 0 + + vs = VIRTUALSERVICE_FILE.read_text() + + vs = re.sub( + rf"(subset: {target}\n weight: )\d+", + rf"\g<1>{target_weight}", + vs, + count=1, + ) + vs = re.sub( + rf"(subset: {other}\n weight: )\d+", + rf"\g<1>{other_weight}", + vs, + count=1, + ) + VIRTUALSERVICE_FILE.write_text(vs) + + active = ACTIVE_COLOR_FILE.read_text() + active = re.sub( + r"(ACTIVE_COLOR: )\w+", + rf"\g<1>{target}", + active, + count=1, + ) + ACTIVE_COLOR_FILE.write_text(active) + + print(f"active_color={target}") + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("Usage: traffic_switch.py ") + switch_to(sys.argv[1]) + + +if __name__ == "__main__": + main()