diff --git a/.env.example b/.env.example index fc9d375..9689358 100644 --- a/.env.example +++ b/.env.example @@ -11,8 +11,12 @@ DB_CONNECT_RETRIES=20 DB_CONNECT_DELAY=2 DB_POOL_MAX_SIZE=10 +NGINX_HTTP_PORT=80 +NGINX_EXPORTER_PORT=9113 + LOAD_DURATION=120 LOAD_CONCURRENCY=20 LOAD_INTERVAL=0.05 LOAD_WRITE_RATIO=0.35 -TARGET_URL=http://app:8000 +LOAD_TIMEOUT=5 +LOADTEST_TARGET_URL=http://nginx diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..97757df --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,113 @@ +name: CI / CD + +on: + pull_request: + push: + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + name: Validate Source + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Prepare executable scripts + run: chmod +x scripts/*.sh + + - name: Validate Docker Compose + run: docker compose config + + - name: Validate Python syntax + run: python3 -m py_compile app/main.py loadtest/load_test.py + + - name: Validate shell scripts + run: | + bash -n scripts/backup_postgres.sh + bash -n scripts/smoke_test.sh + + smoke-deploy: + name: Smoke Deploy + runs-on: ubuntu-latest + needs: validate + env: + NGINX_HTTP_PORT: 8088 + BASE_URL: http://localhost:8088 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Prepare executable scripts + run: chmod +x scripts/*.sh + + - name: Start full stack + run: docker compose up -d --build --wait --wait-timeout 120 + + - name: Run smoke test + run: ./scripts/smoke_test.sh + + - name: Exercise bundled load tester + run: | + docker compose --profile loadtest run --rm --no-deps \ + -e LOAD_DURATION=10 \ + -e LOAD_CONCURRENCY=8 \ + -e LOAD_INTERVAL=0.05 \ + -e LOAD_TIMEOUT=5 \ + loadtester + + - name: Show service status on failure + if: failure() + run: | + docker compose ps + docker compose logs --no-color + + - name: Tear down stack + if: always() + run: docker compose down -v --remove-orphans + + publish-app-image: + name: Publish App Image + runs-on: ubuntu-latest + needs: smoke-deploy + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/containerized-observability-app + tags: | + type=raw,value=latest + type=sha + + - name: Build and publish app image + uses: docker/build-push-action@v6 + with: + context: ./app + file: ./app/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/README.md b/README.md index 3845b8c..63307d6 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,164 @@ -# Member C Deliverables +# Containerized Observability App -This repository contains the application and data-layer artifacts for the team project: +## Project Topic -- A Flask demo app with PostgreSQL integration. -- A production-ready application Dockerfile. -- PostgreSQL persistence and initialization assets. -- A simple backup script based on `pg_dump`. -- A containerized load generator for dashboards and log aggregation. +**Highly Available Containerized Web App with Centralized Monitoring and Logging** -## Project layout +This project is a simple Flask web app with PostgreSQL. +It runs multiple replicas with Docker Compose, uses NGINX as a reverse proxy, and includes monitoring, centralized logging, and CI/CD. -- `app/`: Flask application source and Dockerfile. -- `loadtest/`: Async HTTP load generator and Dockerfile. -- `database/init/`: SQL executed by the PostgreSQL container on first boot. -- `scripts/backup_postgres.sh`: Creates compressed SQL backups into `./backups`. -- `compose.member-c.yml`: Compose fragment for Member A to merge into the final stack. +## Stack -## Application endpoints +- Python Flask +- PostgreSQL +- Docker Compose +- NGINX +- Prometheus +- Grafana +- Loki +- Promtail +- Node Exporter +- cAdvisor +- GitHub Actions -- `GET /`: service info, hostname, instance id, and exposed routes. -- `GET /health`: liveness endpoint. -- `GET /ready`: readiness endpoint with live PostgreSQL probe. -- `GET /api/visits`: persists a visit event and returns the running total. -- `GET /api/messages`: returns the most recent messages. -- `POST /api/messages`: inserts a new message row. -- `GET /metrics`: Prometheus scrape endpoint for request and app metrics. +## Features -The app writes JSON logs to stdout, which makes it easy for Promtail/Loki or ELK to collect and index container logs. +- Flask API +- PostgreSQL database +- 3 app replicas +- NGINX load balancing and rate limiting +- Prometheus and Grafana for metrics +- Loki and Promtail for logs +- Node Exporter and cAdvisor for host/container metrics +- GitHub Actions CI/CD +- PostgreSQL backup script -## Local run +## Run the project -1. Copy `.env.example` values into your local shell or a `.env` file. -2. Start the app and database: +```bash +cp .env.example .env +docker compose up -d --build +``` + +## Useful links + +- App: `http://127.0.0.1/` +- Grafana: `http://127.0.0.1:3000` +- Prometheus: `http://127.0.0.1:9090` +- Loki health check: `http://127.0.0.1:3100/ready` +- NGINX exporter: `http://127.0.0.1:9113/metrics` + +Grafana login: + +- user: `admin` +- password: `admin` + +## Quick check + +```bash +curl http://127.0.0.1/health +curl http://127.0.0.1/ready +curl http://127.0.0.1/api/visits +curl http://127.0.0.1/api/messages +``` + +You can also run: + +```bash +bash scripts/smoke_test.sh +``` + +## Load balancing + +Default mode is `round robin`. + +Check: + +```bash +for i in $(seq 1 6); do + curl -s -D - http://127.0.0.1/ -o /dev/null | grep X-App-Instance +done +``` + +PowerShell alternative: + +```powershell +1..6 | ForEach-Object { (Invoke-WebRequest http://127.0.0.1/ -UseBasicParsing).Headers["X-App-Instance"] } +``` + +If you want `least connections`: + +```bash +docker compose -f docker-compose.yml -f docker-compose.leastconn.yml up -d --build +``` + +After that, run the load balancing check again, because Docker Compose recreates containers when switching the mode. + +To return to the default `round robin` mode: ```bash -docker compose -f compose.member-c.yml up --build +docker compose up -d --build ``` -3. Generate traffic for dashboards: +## Monitoring and logs + +Prometheus collects metrics from: + +- `app1` +- `app2` +- `app3` +- `nginx-exporter` +- `node-exporter` +- `cadvisor` + +Grafana dashboards: + +- `Flask App Dashboard` +- `Infrastructure Overview` + +Logs are viewed in Grafana Explore through the Loki datasource. + +In `Explore`, switch the datasource from `Prometheus` to `Loki` first. + +Example Loki query in Grafana Explore: + +```logql +{compose_service=~".+"} +``` + +## Load test + +```bash +docker compose --profile loadtest run --rm --no-deps loadtester +``` + +## Database backup ```bash -docker compose -f compose.member-c.yml --profile loadtest up --build loadtester +bash scripts/backup_postgres.sh ``` -4. Create a PostgreSQL backup: +## CI/CD + +Workflow file: + +`.github/workflows/ci-cd.yml` + +The pipeline: + +- validates the project +- starts the stack +- runs the smoke test +- publishes the app image on push to `main` + +## Stop the project ```bash -./scripts/backup_postgres.sh +docker compose down ``` -## Handoff notes +Full cleanup: -- Member A can merge `compose.member-c.yml` into the final `docker-compose.yml`. -- The PostgreSQL persistence volume is `postgres_data`. -- The PostgreSQL initialization scripts live in `./database/init`. -- Member B can scrape the Flask app at `http://app:8000/metrics`. -- The load generator is meant to target NGINX in the final topology, so `TARGET_URL` defaults should point to the reverse proxy in the team compose file. +```bash +docker compose down -v +``` diff --git a/compose.member-c.yml b/compose.member-c.yml deleted file mode 100644 index 2af4093..0000000 --- a/compose.member-c.yml +++ /dev/null @@ -1,56 +0,0 @@ -services: - app: - build: - context: ./app - image: sna-flask-app:latest - environment: - APP_NAME: ${APP_NAME:-sna-demo-app} - PORT: ${PORT:-8000} - DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} - DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} - DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} - DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} - depends_on: - postgres: - condition: service_healthy - ports: - - "${PORT:-8000}:8000" - restart: unless-stopped - - postgres: - image: postgres:16-alpine - environment: - POSTGRES_DB: ${POSTGRES_DB:-app_db} - POSTGRES_USER: ${POSTGRES_USER:-app_user} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-app_password} - volumes: - - postgres_data:/var/lib/postgresql/data - - ./database/init:/docker-entrypoint-initdb.d:ro - - ./backups:/backups - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-app_user} -d ${POSTGRES_DB:-app_db}"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - restart: unless-stopped - - loadtester: - build: - context: ./loadtest - image: sna-loadtester:latest - environment: - TARGET_URL: ${TARGET_URL:-http://app:8000} - LOAD_DURATION: ${LOAD_DURATION:-120} - LOAD_CONCURRENCY: ${LOAD_CONCURRENCY:-20} - LOAD_INTERVAL: ${LOAD_INTERVAL:-0.05} - LOAD_WRITE_RATIO: ${LOAD_WRITE_RATIO:-0.35} - depends_on: - app: - condition: service_started - profiles: - - loadtest - -volumes: - postgres_data: - diff --git a/context.txt b/context.txt deleted file mode 100644 index 9f75f06..0000000 --- a/context.txt +++ /dev/null @@ -1,23 +0,0 @@ -Project Title: -Highly Available Containerized Web App with Centralized Monitoring and Logging - - -"Develop a simple multi-tier web application with a PostgreSQL backend. -Write Docker Compose file. -Deploy NGINX as a reverse proxy with load balancing and rate limiting capabilities. -Deploy Prometheus to collect metrics from containers and the host system. -Deploy Grafana to visualize system health and request rates. -Deploy Loki and Promtail to aggregate all container logs in one interface. -Create a CI/CD pipeline with GitHub Actions to automate testing and deployment." - -Technology stack: -"Python Flask -PostgreSQL -Docker Compose -NGINX -Prometheus -Grafana -Grafana Loki -Promtail -GitHub Actions" - diff --git a/docker-compose.leastconn.yml b/docker-compose.leastconn.yml new file mode 100644 index 0000000..5f910a4 --- /dev/null +++ b/docker-compose.leastconn.yml @@ -0,0 +1,4 @@ +services: + nginx: + volumes: + - ./nginx/upstreams/least-connections.conf:/etc/nginx/conf.d/upstream.conf:ro diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5f55ec3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,199 @@ +services: + # ── Data layer ────────────────────────────────────── + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-app_db} + POSTGRES_USER: ${POSTGRES_USER:-app_user} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-app_password} + volumes: + - postgres_data:/var/lib/postgresql/data + - ./database/init:/docker-entrypoint-initdb.d:ro + - ./backups:/backups + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-app_user} -d ${POSTGRES_DB:-app_db}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + restart: unless-stopped + + # ── Application replicas ──────────────────────────── + app1: &app-template + build: + context: ./app + image: sna-flask-app:latest + environment: + APP_NAME: ${APP_NAME:-sna-demo-app} + APP_INSTANCE_ID: app-1 + PORT: "8000" + DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} + DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} + DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} + depends_on: + postgres: + condition: service_healthy + expose: + - "8000" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + restart: unless-stopped + + app2: + <<: *app-template + environment: + APP_NAME: ${APP_NAME:-sna-demo-app} + APP_INSTANCE_ID: app-2 + PORT: "8000" + DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} + DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} + DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} + + app3: + <<: *app-template + environment: + APP_NAME: ${APP_NAME:-sna-demo-app} + APP_INSTANCE_ID: app-3 + PORT: "8000" + DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} + DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} + DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} + + # ── Reverse proxy & exporter ──────────────────────── + nginx: + image: nginx:1.27-alpine + ports: + - "${NGINX_HTTP_PORT:-80}:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + # Mount the upstream and public vhost over the default image config so localhost + # always hits the reverse proxy instead of NGINX's bundled welcome page. + - ./nginx/upstreams/round-robin.conf:/etc/nginx/conf.d/upstream.conf:ro + - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/default.conf:ro + - ./nginx/conf.d/metrics.conf:/etc/nginx/conf.d/metrics.conf:ro + depends_on: + app1: + condition: service_healthy + app2: + condition: service_healthy + app3: + condition: service_healthy + restart: unless-stopped + + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.5.1 + command: + - --nginx.scrape-uri=http://nginx:8080/stub_status + expose: + - "9113" + ports: + - "${NGINX_EXPORTER_PORT:-9113}:9113" + depends_on: + nginx: + condition: service_started + restart: unless-stopped + + node-exporter: + image: prom/node-exporter:v1.8.1 + command: + - '--path.rootfs=/host' + volumes: + - /:/host:ro + expose: + - "9100" + restart: unless-stopped + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + privileged: true + volumes: + - /:/rootfs:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + expose: + - "8080" + restart: unless-stopped + + # ── Monitoring (Member B) ─────────────────────────── + prometheus: + image: prom/prometheus:v2.52.0 + container_name: prometheus + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + ports: + - "9090:9090" + restart: unless-stopped + + grafana: + image: grafana/grafana:11.0.0 + container_name: grafana + volumes: + - ./monitoring/grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml + - ./monitoring/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + ports: + - "3000:3000" + restart: unless-stopped + + loki: + image: grafana/loki:3.0.0 + container_name: loki + volumes: + - ./monitoring/loki/local-config.yaml:/etc/loki/local-config.yaml + - loki_data:/loki + command: -config.file=/etc/loki/local-config.yaml + ports: + - "3100:3100" + restart: unless-stopped + + promtail: + image: grafana/promtail:3.0.0 + container_name: promtail + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./monitoring/promtail/config.yml:/etc/promtail/config.yml + command: -config.file=/etc/promtail/config.yml + restart: unless-stopped + + # ── Load testing (optional) ───────────────────────── + loadtester: + build: + context: ./loadtest + image: sna-loadtester:latest + profiles: + - loadtest # Only starts with --profile loadtest + environment: + TARGET_URL: ${LOADTEST_TARGET_URL:-http://nginx} + LOAD_DURATION: ${LOAD_DURATION:-120} + LOAD_CONCURRENCY: ${LOAD_CONCURRENCY:-20} + LOAD_INTERVAL: ${LOAD_INTERVAL:-0.05} + LOAD_WRITE_RATIO: ${LOAD_WRITE_RATIO:-0.35} + LOAD_TIMEOUT: ${LOAD_TIMEOUT:-5} + depends_on: + - nginx + restart: "no" + +volumes: + postgres_data: + prometheus_data: + grafana_data: + loki_data: diff --git a/monitoring/grafana/dashboards.yml b/monitoring/grafana/dashboards.yml new file mode 100644 index 0000000..a26d4a3 --- /dev/null +++ b/monitoring/grafana/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/monitoring/grafana/dashboards/flask-app.json b/monitoring/grafana/dashboards/flask-app.json new file mode 100644 index 0000000..aa56f13 --- /dev/null +++ b/monitoring/grafana/dashboards/flask-app.json @@ -0,0 +1,97 @@ +{ + "uid": "flask-main-dashboard", + "title": "Flask App Dashboard", + "tags": ["flask", "prometheus"], + "timezone": "browser", + "schemaVersion": 38, + "refresh": "10s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "Request Rate (by method)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "sum by (method) (rate(flask_http_request_total[1m]))", + "legendFormat": "{{method}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + } + } + }, + { + "id": 2, + "title": "Request Duration (p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(flask_http_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + } + }, + { + "id": 3, + "title": "Total Visits", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "sum(app_visit_events_total)" + } + ] + }, + { + "id": 4, + "title": "Total Messages", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 8 }, + "targets": [ + { + "expr": "sum(app_messages_created_total)" + } + ] + }, + { + "id": 5, + "title": "Memory (RSS)", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "sum(process_resident_memory_bytes)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + } + }, + { + "id": 6, + "title": "NGINX Active Connections", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 8 }, + "targets": [ + { + "expr": "nginx_connections_active" + } + ] + } + ] +} diff --git a/monitoring/grafana/dashboards/infrastructure-overview.json b/monitoring/grafana/dashboards/infrastructure-overview.json new file mode 100644 index 0000000..d4157ae --- /dev/null +++ b/monitoring/grafana/dashboards/infrastructure-overview.json @@ -0,0 +1,102 @@ +{ + "uid": "infra-overview-dashboard", + "title": "Infrastructure Overview", + "tags": ["infrastructure", "prometheus"], + "timezone": "browser", + "schemaVersion": 38, + "refresh": "10s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "Host CPU Usage", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + } + } + }, + { + "id": 2, + "title": "Host Memory Usage", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + } + } + }, + { + "id": 3, + "title": "Prometheus Targets Up", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "sum(up)" + } + ] + }, + { + "id": 4, + "title": "Observed Docker Containers", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "targets": [ + { + "expr": "count(container_last_seen{id=~\".*[a-f0-9]{12,}.*\"})" + } + ] + }, + { + "id": 5, + "title": "Container CPU Usage (by container)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "targets": [ + { + "expr": "sum by (id) (rate(container_cpu_usage_seconds_total{id=~\".*[a-f0-9]{12,}.*\",cpu=\"total\"}[5m]))", + "legendFormat": "{{id}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + } + } + }, + { + "id": 6, + "title": "Container Memory (by container)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "targets": [ + { + "expr": "sum by (id) (container_memory_working_set_bytes{id=~\".*[a-f0-9]{12,}.*\"})", + "legendFormat": "{{id}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + } + } + ] +} diff --git a/monitoring/grafana/datasources.yml b/monitoring/grafana/datasources.yml new file mode 100644 index 0000000..42466bc --- /dev/null +++ b/monitoring/grafana/datasources.yml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: true \ No newline at end of file diff --git a/monitoring/loki/local-config.yaml b/monitoring/loki/local-config.yaml new file mode 100644 index 0000000..749f6a4 --- /dev/null +++ b/monitoring/loki/local-config.yaml @@ -0,0 +1,51 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + +limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 168h + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false \ No newline at end of file diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..d245161 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,31 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'flask-app' + static_configs: + - targets: + - 'app1:8000' + - 'app2:8000' + - 'app3:8000' + labels: + service: 'flask' + + - job_name: 'nginx-exporter' + static_configs: + - targets: ['nginx-exporter:9113'] # NGINX metrics exporter (Member A must expose) + labels: + service: 'nginx' + + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + labels: + service: 'host' + + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + labels: + service: 'containers' diff --git a/monitoring/promtail/config.yml b/monitoring/promtail/config.yml new file mode 100644 index 0000000..0eb39b9 --- /dev/null +++ b/monitoring/promtail/config.yml @@ -0,0 +1,56 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker-containers + + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + replacement: '$1' + target_label: job + + - source_labels: ['__meta_docker_container_log_stream'] + target_label: stream + + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: compose_service + + pipeline_stages: + - docker: {} + + - json: + expressions: + level: level + logger: logger + message: message + timestamp: time + + - labels: + level: + logger: + compose_service: + + - timestamp: + source: timestamp + format: RFC3339 + fallback_formats: + - RFC3339Nano + + - output: + source: message \ No newline at end of file diff --git a/nginx/conf.d/default.conf b/nginx/conf.d/default.conf new file mode 100644 index 0000000..21caa51 --- /dev/null +++ b/nginx/conf.d/default.conf @@ -0,0 +1,40 @@ +# Public virtual host — port 80. Proxies to upstream flask_backend (see conf.d/upstream.conf). +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + limit_req_status 429; + proxy_http_version 1.1; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 3; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Default site: moderate rate limit (see general_limit in nginx.conf). + location / { + limit_req zone=general_limit burst=100 nodelay; + proxy_pass http://flask_backend; + } + + # API routes: stricter limit to protect the database and application logic. + location /api/ { + limit_req zone=api_limit burst=20 nodelay; + proxy_pass http://flask_backend; + } + + # No limit_req here — avoids throttling Docker/liveness checks that hit /health on the apps directly; + # through NGINX, /health is still cheap (access_log off only). + location /health { + proxy_pass http://flask_backend; + access_log off; + } + + location /ready { + proxy_pass http://flask_backend; + access_log off; + } +} diff --git a/nginx/conf.d/metrics.conf b/nginx/conf.d/metrics.conf new file mode 100644 index 0000000..ceca846 --- /dev/null +++ b/nginx/conf.d/metrics.conf @@ -0,0 +1,10 @@ + +server { + listen 8080; + server_name _; + + location /stub_status { + stub_status; + access_log off; + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..f203914 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,26 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s; + limit_req_zone $binary_remote_addr zone=general_limit:10m rate=50r/s; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/nginx/upstreams/least-connections.conf b/nginx/upstreams/least-connections.conf new file mode 100644 index 0000000..b1287f1 --- /dev/null +++ b/nginx/upstreams/least-connections.conf @@ -0,0 +1,7 @@ + +upstream flask_backend { + least_conn; + server app1:8000 max_fails=3 fail_timeout=30s; + server app2:8000 max_fails=3 fail_timeout=30s; + server app3:8000 max_fails=3 fail_timeout=30s; +} diff --git a/nginx/upstreams/round-robin.conf b/nginx/upstreams/round-robin.conf new file mode 100644 index 0000000..cc5b81d --- /dev/null +++ b/nginx/upstreams/round-robin.conf @@ -0,0 +1,6 @@ + +upstream flask_backend { + server app1:8000 max_fails=3 fail_timeout=30s; + server app2:8000 max_fails=3 fail_timeout=30s; + server app3:8000 max_fails=3 fail_timeout=30s; +} diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh new file mode 100755 index 0000000..c90b618 --- /dev/null +++ b/scripts/smoke_test.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:${NGINX_HTTP_PORT:-80}}" +PROMETHEUS_URL="${PROMETHEUS_URL:-http://localhost:9090}" +GRAFANA_URL="${GRAFANA_URL:-http://localhost:3000}" +LOKI_URL="${LOKI_URL:-http://localhost:3100}" +EXPORTER_URL="${EXPORTER_URL:-http://localhost:${NGINX_EXPORTER_PORT:-9113}}" + +retry() { + local attempts="$1" + local delay_seconds="$2" + shift 2 + + local attempt + for attempt in $(seq 1 "$attempts"); do + if "$@" >/dev/null 2>&1; then + return 0 + fi + sleep "$delay_seconds" + done + + return 1 +} + +assert_json_field() { + local json_payload="$1" + local field_path="$2" + local expected="$3" + + JSON_PAYLOAD="$json_payload" FIELD_PATH="$field_path" EXPECTED_VALUE="$expected" python3 - <<'PY' +import json +import os +import sys + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +expected = os.environ["EXPECTED_VALUE"] +value = payload +for part in os.environ["FIELD_PATH"].split("."): + value = value[part] + +if str(value) != expected: + raise SystemExit(f"Expected {os.environ['FIELD_PATH']}={expected!r}, got {value!r}") +PY +} + +echo "Waiting for public application endpoints..." +retry 30 2 curl -fsS "${BASE_URL}/health" >/dev/null +retry 30 2 curl -fsS "${BASE_URL}/ready" >/dev/null + +root_response="$(curl -fsS "${BASE_URL}/")" +health_response="$(curl -fsS "${BASE_URL}/health")" +ready_response="$(curl -fsS "${BASE_URL}/ready")" + +assert_json_field "$root_response" "status" "ok" +assert_json_field "$health_response" "status" "ok" +assert_json_field "$ready_response" "status" "ready" + +echo "Checking write/read API flow..." +review_message="smoke-test-$(date +%s)" +create_response="$(curl -fsS -X POST "${BASE_URL}/api/messages" -H 'Content-Type: application/json' -d "{\"message\":\"${review_message}\"}")" +assert_json_field "$create_response" "message.message" "$review_message" + +messages_response="$(curl -fsS "${BASE_URL}/api/messages?limit=20")" +JSON_PAYLOAD="$messages_response" EXPECTED_SUBSTRING="$review_message" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +needle = os.environ["EXPECTED_SUBSTRING"] +messages = [item["message"] for item in payload["messages"]] +if needle not in messages: + raise SystemExit(f"Message {needle!r} not found in recent messages") +PY + +echo "Checking load balancing across replicas..." +instance_sample_file="$(mktemp)" +for _ in $(seq 1 9); do + curl -fsSI "${BASE_URL}/" | tr -d '\r' | awk '/^X-App-Instance:/ {print $2}' >> "$instance_sample_file" +done + +unique_instances="$(sort -u "$instance_sample_file" | sed '/^$/d' | wc -l | tr -d ' ')" +if [[ "$unique_instances" -lt 3 ]]; then + echo "Expected traffic to hit 3 replicas, got ${unique_instances}" >&2 + sort -u "$instance_sample_file" >&2 + exit 1 +fi +rm -f "$instance_sample_file" + +echo "Checking NGINX rate limiting..." +rate_limit_summary="$(BASE_URL="$BASE_URL" python3 - <<'PY' +import collections +import concurrent.futures +import json +import os +import urllib.error +import urllib.parse +import urllib.request + +base_url = os.environ["BASE_URL"].rstrip("/") + +def issue_request() -> int: + try: + with urllib.request.urlopen(f"{base_url}/api/visits", timeout=10) as response: + return response.getcode() + except urllib.error.HTTPError as exc: + return exc.code + +counter = collections.Counter() +with concurrent.futures.ThreadPoolExecutor(max_workers=60) as executor: + for status_code in executor.map(lambda _: issue_request(), range(60)): + counter[status_code] += 1 + +print(json.dumps(counter)) +PY +)" + +JSON_PAYLOAD="$rate_limit_summary" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +if int(payload.get("429", 0)) <= 0: + raise SystemExit(f"Expected at least one 429 response, got: {payload}") +if int(payload.get("200", 0)) <= 0: + raise SystemExit(f"Expected successful API responses alongside 429s, got: {payload}") +PY + +echo "Checking Prometheus targets and app metrics..." +retry 30 2 curl -fsS "${PROMETHEUS_URL}/-/healthy" >/dev/null +PROMETHEUS_URL="$PROMETHEUS_URL" python3 - <<'PY' +import json +import os +import time +import urllib.request + +prometheus_url = os.environ["PROMETHEUS_URL"].rstrip("/") +required_jobs = {"flask-app", "nginx-exporter", "node-exporter", "cadvisor"} + +def query(expr: str): + with urllib.request.urlopen( + f"{prometheus_url}/api/v1/query?query={urllib.parse.quote(expr, safe='')}", + timeout=10, + ) as response: + return json.load(response) + +deadline = time.time() + 60 +while time.time() < deadline: + up_payload = query("up") + targets = { + (item["metric"]["job"], item["metric"]["instance"]): item["value"][1] + for item in up_payload["data"]["result"] + } + seen_jobs = {job for job, _ in targets} + node_payload = query("node_cpu_seconds_total") + cadvisor_payload = query("container_cpu_usage_seconds_total") + + if ( + required_jobs.issubset(seen_jobs) + and all(value == "1" for value in targets.values()) + and node_payload["data"]["result"] + and cadvisor_payload["data"]["result"] + ): + break + time.sleep(2) +else: + raise SystemExit( + "Prometheus did not expose all required jobs and metrics within 60 seconds" + ) +PY + +curl -fsS "${EXPORTER_URL}/metrics" >/dev/null + +echo "Checking Grafana provisioning..." +retry 30 2 curl -fsS "${GRAFANA_URL}/api/health" >/dev/null +datasources_response="$(curl -fsS -u admin:admin "${GRAFANA_URL}/api/datasources")" +dashboards_response="$(curl -fsS -u admin:admin "${GRAFANA_URL}/api/search")" + +JSON_PAYLOAD="$datasources_response" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +names = {item["name"] for item in payload} +required = {"Prometheus", "Loki"} +missing = required - names +if missing: + raise SystemExit(f"Missing Grafana datasources: {sorted(missing)}") +PY + +JSON_PAYLOAD="$dashboards_response" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +titles = {item["title"] for item in payload} +required = {"Flask App Dashboard", "Infrastructure Overview"} +missing = required - titles +if missing: + raise SystemExit(f"Missing Grafana dashboards: {sorted(missing)}") +PY + +echo "Checking Loki ingestion..." +retry 30 2 curl -fsS "${LOKI_URL}/ready" >/dev/null +loki_labels="$(curl -fsS "${LOKI_URL}/loki/api/v1/label/compose_service/values")" + +JSON_PAYLOAD="$loki_labels" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +services = set(payload["data"]) +required_services = {"app1", "app2", "app3", "nginx"} +missing = required_services - services +if missing: + raise SystemExit(f"Missing Loki compose_service labels: {sorted(missing)}") +PY + +echo "Checking PostgreSQL backup flow..." +./scripts/backup_postgres.sh >/dev/null +latest_backup="$(ls -1t backups/*.sql.gz | head -n 1)" +if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then + echo "Backup file was not created" >&2 + exit 1 +fi + +echo "Smoke test passed."