From e23102d10952e6020791a5200547cdeb1f1c3bb0 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Tue, 12 May 2026 03:12:13 +0800 Subject: [PATCH 01/23] feat: added prometheus --- monitoring/prometheus/prometheus.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 monitoring/prometheus/prometheus.yml diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..fb8ef07 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,21 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'flask-app' + static_configs: + - targets: ['app:8000'] # Flask /metrics endpoint from Member C + labels: + service: 'flask' + + - job_name: 'nginx-exporter' + static_configs: + - targets: ['nginx-exporter:9113'] # NGINX metrics exporter (Member A must expose) + labels: + service: 'nginx' + + # optional: add node_exporter if you want host/container sys metrics + - job_name: 'node' + static_configs: + - targets: ['node-exporter:9100'] \ No newline at end of file From 408160c017bed69c1207104e12441c6976f87695 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Tue, 12 May 2026 03:14:15 +0800 Subject: [PATCH 02/23] feat: added grafana --- monitoring/grafana/dashboards.yml | 11 +++ monitoring/grafana/dashboards/flask-app.json | 71 ++++++++++++++++++++ monitoring/grafana/datasources.yml | 15 +++++ 3 files changed, 97 insertions(+) create mode 100644 monitoring/grafana/dashboards.yml create mode 100644 monitoring/grafana/dashboards/flask-app.json create mode 100644 monitoring/grafana/datasources.yml diff --git a/monitoring/grafana/dashboards.yml b/monitoring/grafana/dashboards.yml new file mode 100644 index 0000000..a26d4a3 --- /dev/null +++ b/monitoring/grafana/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/monitoring/grafana/dashboards/flask-app.json b/monitoring/grafana/dashboards/flask-app.json new file mode 100644 index 0000000..a97d4cf --- /dev/null +++ b/monitoring/grafana/dashboards/flask-app.json @@ -0,0 +1,71 @@ +{ + "dashboard": { + "title": "Flask Application Overview", + "templating": { + "list": [] + }, + "panels": [ + { + "title": "Requests per second", + "type": "graph", + "targets": [ + { + "expr": "rate(flask_http_request_total[1m])", + "legendFormat": "{{method}} {{endpoint}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0} + }, + { + "title": "Request duration (p95)", + "type": "graph", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(flask_http_request_duration_seconds_bucket[1m])) by (le, endpoint))", + "legendFormat": "{{endpoint}}" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0} + }, + { + "title": "Total Visits", + "type": "stat", + "targets": [ + { + "expr": "visits_total", + "legendFormat": "Total visits" + } + ], + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 8} + }, + { + "title": "Error rate (5xx)", + "type": "graph", + "targets": [ + { + "expr": "sum(rate(flask_http_request_total{status=~'5..'}[1m])) / sum(rate(flask_http_request_total[1m]))", + "legendFormat": "5xx ratio" + } + ], + "gridPos": {"h": 4, "w": 6, "x": 6, "y": 8} + }, + { + "title": "Active requests", + "type": "graph", + "targets": [ + { + "expr": "flask_http_request_in_progress", + "legendFormat": "in-flight" + } + ], + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 8} + } + ], + "refresh": "10s", + "schemaVersion": 16, + "time": { + "from": "now-15m", + "to": "now" + } + } +} \ No newline at end of file diff --git a/monitoring/grafana/datasources.yml b/monitoring/grafana/datasources.yml new file mode 100644 index 0000000..42466bc --- /dev/null +++ b/monitoring/grafana/datasources.yml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: true \ No newline at end of file From 04a8e57bd825e48221b02dd0b785c4670277cd71 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Tue, 12 May 2026 03:15:52 +0800 Subject: [PATCH 03/23] feat: added loki --- monitoring/loki/local-config.yml | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 monitoring/loki/local-config.yml diff --git a/monitoring/loki/local-config.yml b/monitoring/loki/local-config.yml new file mode 100644 index 0000000..482f877 --- /dev/null +++ b/monitoring/loki/local-config.yml @@ -0,0 +1,35 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + ring: + kvstore: + store: inmemory + replication_factor: 1 + chunk_idle_period: 5m + max_chunk_age: 1h + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: loki_index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/boltdb-cache + filesystem: + directory: /loki/chunks + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h \ No newline at end of file From 93753162d7f878adc469708b516fd9bd83900181 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Tue, 12 May 2026 03:16:32 +0800 Subject: [PATCH 04/23] feat: added promtail --- monitoring/promtail/config.yml | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 monitoring/promtail/config.yml diff --git a/monitoring/promtail/config.yml b/monitoring/promtail/config.yml new file mode 100644 index 0000000..ebaecb5 --- /dev/null +++ b/monitoring/promtail/config.yml @@ -0,0 +1,41 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker_containers + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + # Drop Prometheus node exporter and other metrics-only containers to reduce noise + - source_labels: ['__meta_docker_container_name'] + regex: '.*(prometheus|node-exporter|grafana|loki|promtail).*' + action: drop + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + replacement: '$1' + target_label: 'job' + pipeline_stages: + - json: + expressions: + level: level + logger: logger + message: message + - labels: + level: + logger: + - timestamp: + source: time + format: RFC3339 + - output: + source: message \ No newline at end of file From 665075393f646b5a575edde6e2ec1a303670bdfb Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Tue, 12 May 2026 03:18:46 +0800 Subject: [PATCH 05/23] feat: created docker compose --- monitoring/compose.member-b.yml | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 monitoring/compose.member-b.yml diff --git a/monitoring/compose.member-b.yml b/monitoring/compose.member-b.yml new file mode 100644 index 0000000..1860f9d --- /dev/null +++ b/monitoring/compose.member-b.yml @@ -0,0 +1,57 @@ +version: "3.8" + +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + ports: + - "9090:9090" + restart: unless-stopped + + grafana: + image: grafana/grafana:latest + container_name: grafana + volumes: + - ./monitoring/grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml + - ./monitoring/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml + - ./monitoring/grafana/dashboards/flask-app.json:/var/lib/grafana/dashboards/flask-app.json + - grafana_data:/var/lib/grafana + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin # Change for real projects + ports: + - "3000:3000" + restart: unless-stopped + + loki: + image: grafana/loki:latest + container_name: loki + volumes: + - ./monitoring/loki/local-config.yaml:/etc/loki/local-config.yaml + - loki_data:/loki + command: -config.file=/etc/loki/local-config.yaml + ports: + - "3100:3100" + restart: unless-stopped + + promtail: + image: grafana/promtail:latest + container_name: promtail + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./monitoring/promtail/config.yml:/etc/promtail/config.yml + command: -config.file=/etc/promtail/config.yml + restart: unless-stopped + +volumes: + prometheus_data: + grafana_data: + loki_data: \ No newline at end of file From d5bcb61428b5a948a154af8838068138aa30d571 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Wed, 13 May 2026 03:35:54 +0800 Subject: [PATCH 06/23] fix: fixed issue with wrapper in grafana --- monitoring/grafana/dashboards/flask-app.json | 147 ++++++++++--------- 1 file changed, 81 insertions(+), 66 deletions(-) diff --git a/monitoring/grafana/dashboards/flask-app.json b/monitoring/grafana/dashboards/flask-app.json index a97d4cf..e293dd1 100644 --- a/monitoring/grafana/dashboards/flask-app.json +++ b/monitoring/grafana/dashboards/flask-app.json @@ -1,71 +1,86 @@ { - "dashboard": { - "title": "Flask Application Overview", - "templating": { - "list": [] + "uid": "flask-main-dashboard", + "title": "Flask App Dashboard", + "tags": ["flask", "prometheus"], + "timezone": "browser", + "schemaVersion": 38, + "refresh": "10s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "Request Rate (by method)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "rate(flask_http_request_total[1m])", + "legendFormat": "{{method}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + } + } + }, + { + "id": 2, + "title": "Request Duration (p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(flask_http_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + } + }, + { + "id": 3, + "title": "Total Visits", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "app_visit_events_total" + } + ] + }, + { + "id": 4, + "title": "Total Messages", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 8 }, + "targets": [ + { + "expr": "app_messages_created_total" + } + ] }, - "panels": [ - { - "title": "Requests per second", - "type": "graph", - "targets": [ - { - "expr": "rate(flask_http_request_total[1m])", - "legendFormat": "{{method}} {{endpoint}}" - } - ], - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0} - }, - { - "title": "Request duration (p95)", - "type": "graph", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(flask_http_request_duration_seconds_bucket[1m])) by (le, endpoint))", - "legendFormat": "{{endpoint}}" - } - ], - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0} - }, - { - "title": "Total Visits", - "type": "stat", - "targets": [ - { - "expr": "visits_total", - "legendFormat": "Total visits" - } - ], - "gridPos": {"h": 4, "w": 6, "x": 0, "y": 8} - }, - { - "title": "Error rate (5xx)", - "type": "graph", - "targets": [ - { - "expr": "sum(rate(flask_http_request_total{status=~'5..'}[1m])) / sum(rate(flask_http_request_total[1m]))", - "legendFormat": "5xx ratio" - } - ], - "gridPos": {"h": 4, "w": 6, "x": 6, "y": 8} - }, - { - "title": "Active requests", - "type": "graph", - "targets": [ - { - "expr": "flask_http_request_in_progress", - "legendFormat": "in-flight" - } - ], - "gridPos": {"h": 4, "w": 6, "x": 12, "y": 8} + { + "id": 5, + "title": "Memory (RSS)", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "process_resident_memory_bytes" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + } } - ], - "refresh": "10s", - "schemaVersion": 16, - "time": { - "from": "now-15m", - "to": "now" } - } + ] } \ No newline at end of file From e7e1b2ba0e3eef60226b5ee8bbc5abfb63cfac13 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Wed, 13 May 2026 03:36:10 +0800 Subject: [PATCH 07/23] chore: deleted version number --- monitoring/compose.member-b.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/monitoring/compose.member-b.yml b/monitoring/compose.member-b.yml index 1860f9d..c8fa925 100644 --- a/monitoring/compose.member-b.yml +++ b/monitoring/compose.member-b.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: prometheus: image: prom/prometheus:latest @@ -35,9 +33,9 @@ services: image: grafana/loki:latest container_name: loki volumes: - - ./monitoring/loki/local-config.yaml:/etc/loki/local-config.yaml + - ./monitoring/loki/local-config.yml:/etc/loki/local-config.yml - loki_data:/loki - command: -config.file=/etc/loki/local-config.yaml + command: -config.file=/etc/loki/local-config.yml ports: - "3100:3100" restart: unless-stopped From 29fba72190d010d48de20c7eb674aea80fe8919d Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Wed, 13 May 2026 03:43:08 +0800 Subject: [PATCH 08/23] feat: added readme of member b --- monitoring/README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 monitoring/README.md diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..93468de --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,46 @@ +# Member B Deliverables + +This directory contains the observability stack for the team project: + +- Prometheus configured to scrape the Flask app and NGINX exporter. +- Grafana with provisioned Prometheus and Loki datasources. +- A pre-built Grafana dashboard for the Flask application. +- Loki + Promtail for container log aggregation. +- A Compose fragment ready to merge into the final `docker-compose.yml`. + +## Project layout + +- `compose.member-b.yml`: Compose fragment with all observability services (Member A merges this). +- `prometheus/prometheus.yml`: Scrape configs for Flask app and NGINX exporter. +- `grafana/datasources.yml`: Pre-configures Prometheus and Loki as data sources in Grafana. +- `grafana/dashboards.yml`: Dashboard provider config pointing to the `dashboards/` folder. +- `grafana/dashboards/flask-app.json`: A ready-to-use dashboard showing request rate, latency, error rate, and visit count. +- `loki/local-config.yaml`: Loki server settings (in-memory ring, filesystem storage). +- `promtail/config.yml`: Promtail configuration that discovers all running containers via Docker socket and ships logs to Loki. +- `README.md`: This file. + +## Observability endpoints + +Once the final stack is running, the following will be available: + +| Service | Port | URL | Description | +|------------|-------|---------------------------|------------------------------------| +| Prometheus | 9090 | `http://localhost:9090` | Metrics query & alerting UI | +| Grafana | 3000 | `http://localhost:3000` | Dashboards (login: `admin`/`admin`)| +| Loki | 3100 | (internal only) | Log aggregation backend | +| Promtail | — | (internal only) | Log collector (must access Docker socket) | + +The Flask app `/metrics` endpoint is scraped by Prometheus at `http://app:8000/metrics`. +The NGINX exporter is expected at `http://nginx-exporter:9113/metrics` (Member A must include it). + +Container logs are automatically collected by Promtail and indexed into Loki. +In Grafana, explore logs with a query like `{container="flask-app"} |= ""`. + +## Local / standalone test + +The observability services cannot run meaningfully without the application and NGINX, +but you can start them together with Member C’s app to verify Prometheus targets: + +```bash +# Start the whole team stack (Member A's merged compose) or test with Member C's fragment: +docker compose -f ../compose.member-c.yml -f compose.member-b.yml up -d \ No newline at end of file From 7332d16590537124ca0bbba98e4a2e31f505dbc5 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Wed, 13 May 2026 04:06:54 +0800 Subject: [PATCH 09/23] fix: specified versions instead of latest --- monitoring/compose.member-b.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitoring/compose.member-b.yml b/monitoring/compose.member-b.yml index c8fa925..cfcf195 100644 --- a/monitoring/compose.member-b.yml +++ b/monitoring/compose.member-b.yml @@ -1,6 +1,6 @@ services: prometheus: - image: prom/prometheus:latest + image: prom/prometheus:v2.52.0 container_name: prometheus volumes: - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml @@ -15,7 +15,7 @@ services: restart: unless-stopped grafana: - image: grafana/grafana:latest + image: grafana/grafana:11.0.0 container_name: grafana volumes: - ./monitoring/grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml @@ -30,7 +30,7 @@ services: restart: unless-stopped loki: - image: grafana/loki:latest + image: grafana/loki:3.0.0 container_name: loki volumes: - ./monitoring/loki/local-config.yml:/etc/loki/local-config.yml @@ -41,7 +41,7 @@ services: restart: unless-stopped promtail: - image: grafana/promtail:latest + image: grafana/promtail:3.0.0 container_name: promtail volumes: - /var/run/docker.sock:/var/run/docker.sock From bd1d1bb8725c9bf54ba137ad9e02c1ea442aae91 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Wed, 13 May 2026 04:07:44 +0800 Subject: [PATCH 10/23] fix: made loki work with the new version --- monitoring/loki/local-config.yml | 52 +++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/monitoring/loki/local-config.yml b/monitoring/loki/local-config.yml index 482f877..749f6a4 100644 --- a/monitoring/loki/local-config.yml +++ b/monitoring/loki/local-config.yml @@ -2,34 +2,50 @@ auth_enabled: false server: http_listen_port: 3100 + grpc_listen_port: 9096 -ingester: - lifecycler: - ring: - kvstore: - store: inmemory - replication_factor: 1 - chunk_idle_period: 5m - max_chunk_age: 1h +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory schema_config: configs: - - from: 2020-10-24 - store: boltdb-shipper + - from: 2024-01-01 + store: tsdb object_store: filesystem - schema: v11 + schema: v13 index: - prefix: loki_index_ + prefix: index_ period: 24h storage_config: - boltdb_shipper: - active_index_directory: /loki/index - cache_location: /loki/boltdb-cache filesystem: directory: /loki/chunks limits_config: - enforce_metric_name: false - reject_old_samples: true - reject_old_samples_max_age: 168h \ No newline at end of file + allow_structured_metadata: true + volume_enabled: true + retention_period: 168h + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false \ No newline at end of file From 7a31662693474a02b436cf78f2a268d22b45f620 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Wed, 13 May 2026 04:08:58 +0800 Subject: [PATCH 11/23] fix: made promtail work with the new version --- monitoring/promtail/config.yml | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/monitoring/promtail/config.yml b/monitoring/promtail/config.yml index ebaecb5..0eb39b9 100644 --- a/monitoring/promtail/config.yml +++ b/monitoring/promtail/config.yml @@ -9,33 +9,48 @@ clients: - url: http://loki:3100/loki/api/v1/push scrape_configs: - - job_name: docker_containers + - job_name: docker-containers + docker_sd_configs: - host: unix:///var/run/docker.sock refresh_interval: 5s + relabel_configs: - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' - target_label: 'container' - # Drop Prometheus node exporter and other metrics-only containers to reduce noise - - source_labels: ['__meta_docker_container_name'] - regex: '.*(prometheus|node-exporter|grafana|loki|promtail).*' - action: drop + target_label: container + - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' replacement: '$1' - target_label: 'job' + target_label: job + + - source_labels: ['__meta_docker_container_log_stream'] + target_label: stream + + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: compose_service + pipeline_stages: + - docker: {} + - json: expressions: level: level logger: logger message: message + timestamp: time + - labels: level: logger: + compose_service: + - timestamp: - source: time + source: timestamp format: RFC3339 + fallback_formats: + - RFC3339Nano + - output: source: message \ No newline at end of file From 5497962e98cafe2be93ea5688b0b22da22c5465b Mon Sep 17 00:00:00 2001 From: tayaorshulskaya-oss Date: Wed, 13 May 2026 21:33:07 +0300 Subject: [PATCH 12/23] Add Docker Compose stack with NGINX load balancing and rate limits --- docker-compose.leastconn.yml | 15 +++ docker-compose.yml | 146 +++++++++++++++++++++++++ nginx/conf.d/default.conf | 31 ++++++ nginx/conf.d/metrics.conf | 10 ++ nginx/nginx.conf | 26 +++++ nginx/upstreams/least-connections.conf | 7 ++ nginx/upstreams/round-robin.conf | 6 + 7 files changed, 241 insertions(+) create mode 100644 docker-compose.leastconn.yml create mode 100644 docker-compose.yml create mode 100644 nginx/conf.d/default.conf create mode 100644 nginx/conf.d/metrics.conf create mode 100644 nginx/nginx.conf create mode 100644 nginx/upstreams/least-connections.conf create mode 100644 nginx/upstreams/round-robin.conf diff --git a/docker-compose.leastconn.yml b/docker-compose.leastconn.yml new file mode 100644 index 0000000..56e24f9 --- /dev/null +++ b/docker-compose.leastconn.yml @@ -0,0 +1,15 @@ + +services: + nginx: + depends_on: + app1: + condition: service_healthy + app2: + condition: service_healthy + app3: + condition: service_healthy + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/upstreams/least-connections.conf:/etc/nginx/conf.d/01-upstream.conf:ro + - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/02-default.conf:ro + - ./nginx/conf.d/metrics.conf:/etc/nginx/conf.d/03-metrics.conf:ro diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..80eb28a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,146 @@ + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-app_db} + POSTGRES_USER: ${POSTGRES_USER:-app_user} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-app_password} + volumes: + - postgres_data:/var/lib/postgresql/data + - ./database/init:/docker-entrypoint-initdb.d:ro + - ./backups:/backups + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-app_user} -d ${POSTGRES_DB:-app_db}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + restart: unless-stopped + + app1: + build: + context: ./app + image: sna-flask-app:latest + environment: + APP_NAME: ${APP_NAME:-sna-demo-app} + APP_INSTANCE_ID: app-1 + PORT: "8000" + DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} + DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} + DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} + depends_on: + postgres: + condition: service_healthy + expose: + - "8000" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()", + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + restart: unless-stopped + + app2: + build: + context: ./app + image: sna-flask-app:latest + environment: + APP_NAME: ${APP_NAME:-sna-demo-app} + APP_INSTANCE_ID: app-2 + PORT: "8000" + DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} + DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} + DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} + depends_on: + postgres: + condition: service_healthy + expose: + - "8000" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()", + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + restart: unless-stopped + + app3: + build: + context: ./app + image: sna-flask-app:latest + environment: + APP_NAME: ${APP_NAME:-sna-demo-app} + APP_INSTANCE_ID: app-3 + PORT: "8000" + DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} + DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} + DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} + depends_on: + postgres: + condition: service_healthy + expose: + - "8000" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()", + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + restart: unless-stopped + + nginx: + image: nginx:1.27-alpine + ports: + - "${NGINX_HTTP_PORT:-80}:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/upstreams/round-robin.conf:/etc/nginx/conf.d/01-upstream.conf:ro + - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/02-default.conf:ro + - ./nginx/conf.d/metrics.conf:/etc/nginx/conf.d/03-metrics.conf:ro + depends_on: + app1: + condition: service_healthy + app2: + condition: service_healthy + app3: + condition: service_healthy + restart: unless-stopped + + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.5.1 + command: + - --nginx.scrape-uri=http://nginx:8080/stub_status + expose: + - "9113" + ports: + - "${NGINX_EXPORTER_PORT:-9113}:9113" + depends_on: + nginx: + condition: service_started + restart: unless-stopped + +volumes: + postgres_data: diff --git a/nginx/conf.d/default.conf b/nginx/conf.d/default.conf new file mode 100644 index 0000000..399503e --- /dev/null +++ b/nginx/conf.d/default.conf @@ -0,0 +1,31 @@ +# Public virtual host — port 80. Proxies to upstream flask_backend (see conf.d/01-upstream.conf). +server { + listen 80; + server_name _; + + limit_req_status 429; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Default site: moderate rate limit (see general_limit in nginx.conf). + location / { + limit_req zone=general_limit burst=100 nodelay; + proxy_pass http://flask_backend; + } + + # API routes: stricter limit to protect the database and application logic. + location /api/ { + limit_req zone=api_limit burst=20 nodelay; + proxy_pass http://flask_backend; + } + + # No limit_req here — avoids throttling Docker/liveness checks that hit /health on the apps directly; + # through NGINX, /health is still cheap (access_log off only). + location /health { + proxy_pass http://flask_backend; + access_log off; + } +} diff --git a/nginx/conf.d/metrics.conf b/nginx/conf.d/metrics.conf new file mode 100644 index 0000000..ceca846 --- /dev/null +++ b/nginx/conf.d/metrics.conf @@ -0,0 +1,10 @@ + +server { + listen 8080; + server_name _; + + location /stub_status { + stub_status; + access_log off; + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..f203914 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,26 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s; + limit_req_zone $binary_remote_addr zone=general_limit:10m rate=50r/s; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/nginx/upstreams/least-connections.conf b/nginx/upstreams/least-connections.conf new file mode 100644 index 0000000..b1287f1 --- /dev/null +++ b/nginx/upstreams/least-connections.conf @@ -0,0 +1,7 @@ + +upstream flask_backend { + least_conn; + server app1:8000 max_fails=3 fail_timeout=30s; + server app2:8000 max_fails=3 fail_timeout=30s; + server app3:8000 max_fails=3 fail_timeout=30s; +} diff --git a/nginx/upstreams/round-robin.conf b/nginx/upstreams/round-robin.conf new file mode 100644 index 0000000..cc5b81d --- /dev/null +++ b/nginx/upstreams/round-robin.conf @@ -0,0 +1,6 @@ + +upstream flask_backend { + server app1:8000 max_fails=3 fail_timeout=30s; + server app2:8000 max_fails=3 fail_timeout=30s; + server app3:8000 max_fails=3 fail_timeout=30s; +} From a6be17d5d80a1be7b0ae5e54982f370f865ac1f3 Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Fri, 15 May 2026 00:51:00 +0800 Subject: [PATCH 13/23] BREAKING CHANGE: merge with deletion of some files --- README.md | 63 ++------- compose.member-c.yml | 56 -------- context.txt | 23 ---- docker-compose.leastconn.yml | 13 +- docker-compose.yml | 127 +++++++++++------- monitoring/compose.member-b.yml | 55 -------- .../{local-config.yml => local-config.yaml} | 0 7 files changed, 85 insertions(+), 252 deletions(-) delete mode 100644 compose.member-c.yml delete mode 100644 context.txt delete mode 100644 monitoring/compose.member-b.yml rename monitoring/loki/{local-config.yml => local-config.yaml} (100%) diff --git a/README.md b/README.md index 3845b8c..e0c13df 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,11 @@ -# Member C Deliverables +# Containerized Observability App -This repository contains the application and data-layer artifacts for the team project: +Group project for [Course Name / S25]. +Team: Member A (Infrastructure & Proxy), Member B (Observability), Member C (Application & Data). -- A Flask demo app with PostgreSQL integration. -- A production-ready application Dockerfile. -- PostgreSQL persistence and initialization assets. -- A simple backup script based on `pg_dump`. -- A containerized load generator for dashboards and log aggregation. +## Quick start -## Project layout - -- `app/`: Flask application source and Dockerfile. -- `loadtest/`: Async HTTP load generator and Dockerfile. -- `database/init/`: SQL executed by the PostgreSQL container on first boot. -- `scripts/backup_postgres.sh`: Creates compressed SQL backups into `./backups`. -- `compose.member-c.yml`: Compose fragment for Member A to merge into the final stack. - -## Application endpoints - -- `GET /`: service info, hostname, instance id, and exposed routes. -- `GET /health`: liveness endpoint. -- `GET /ready`: readiness endpoint with live PostgreSQL probe. -- `GET /api/visits`: persists a visit event and returns the running total. -- `GET /api/messages`: returns the most recent messages. -- `POST /api/messages`: inserts a new message row. -- `GET /metrics`: Prometheus scrape endpoint for request and app metrics. - -The app writes JSON logs to stdout, which makes it easy for Promtail/Loki or ELK to collect and index container logs. - -## Local run - -1. Copy `.env.example` values into your local shell or a `.env` file. -2. Start the app and database: - -```bash -docker compose -f compose.member-c.yml up --build -``` - -3. Generate traffic for dashboards: - -```bash -docker compose -f compose.member-c.yml --profile loadtest up --build loadtester -``` - -4. Create a PostgreSQL backup: - -```bash -./scripts/backup_postgres.sh -``` - -## Handoff notes - -- Member A can merge `compose.member-c.yml` into the final `docker-compose.yml`. -- The PostgreSQL persistence volume is `postgres_data`. -- The PostgreSQL initialization scripts live in `./database/init`. -- Member B can scrape the Flask app at `http://app:8000/metrics`. -- The load generator is meant to target NGINX in the final topology, so `TARGET_URL` defaults should point to the reverse proxy in the team compose file. +1. Copy `.env.example` to `.env` and adjust values if needed. +2. Start the full stack (round‑robin load balancing): + ```bash + docker compose up -d \ No newline at end of file diff --git a/compose.member-c.yml b/compose.member-c.yml deleted file mode 100644 index 2af4093..0000000 --- a/compose.member-c.yml +++ /dev/null @@ -1,56 +0,0 @@ -services: - app: - build: - context: ./app - image: sna-flask-app:latest - environment: - APP_NAME: ${APP_NAME:-sna-demo-app} - PORT: ${PORT:-8000} - DATABASE_URL: postgresql://${POSTGRES_USER:-app_user}:${POSTGRES_PASSWORD:-app_password}@postgres:5432/${POSTGRES_DB:-app_db} - DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} - DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} - DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} - depends_on: - postgres: - condition: service_healthy - ports: - - "${PORT:-8000}:8000" - restart: unless-stopped - - postgres: - image: postgres:16-alpine - environment: - POSTGRES_DB: ${POSTGRES_DB:-app_db} - POSTGRES_USER: ${POSTGRES_USER:-app_user} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-app_password} - volumes: - - postgres_data:/var/lib/postgresql/data - - ./database/init:/docker-entrypoint-initdb.d:ro - - ./backups:/backups - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-app_user} -d ${POSTGRES_DB:-app_db}"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - restart: unless-stopped - - loadtester: - build: - context: ./loadtest - image: sna-loadtester:latest - environment: - TARGET_URL: ${TARGET_URL:-http://app:8000} - LOAD_DURATION: ${LOAD_DURATION:-120} - LOAD_CONCURRENCY: ${LOAD_CONCURRENCY:-20} - LOAD_INTERVAL: ${LOAD_INTERVAL:-0.05} - LOAD_WRITE_RATIO: ${LOAD_WRITE_RATIO:-0.35} - depends_on: - app: - condition: service_started - profiles: - - loadtest - -volumes: - postgres_data: - diff --git a/context.txt b/context.txt deleted file mode 100644 index 9f75f06..0000000 --- a/context.txt +++ /dev/null @@ -1,23 +0,0 @@ -Project Title: -Highly Available Containerized Web App with Centralized Monitoring and Logging - - -"Develop a simple multi-tier web application with a PostgreSQL backend. -Write Docker Compose file. -Deploy NGINX as a reverse proxy with load balancing and rate limiting capabilities. -Deploy Prometheus to collect metrics from containers and the host system. -Deploy Grafana to visualize system health and request rates. -Deploy Loki and Promtail to aggregate all container logs in one interface. -Create a CI/CD pipeline with GitHub Actions to automate testing and deployment." - -Technology stack: -"Python Flask -PostgreSQL -Docker Compose -NGINX -Prometheus -Grafana -Grafana Loki -Promtail -GitHub Actions" - diff --git a/docker-compose.leastconn.yml b/docker-compose.leastconn.yml index 56e24f9..8c4c699 100644 --- a/docker-compose.leastconn.yml +++ b/docker-compose.leastconn.yml @@ -1,15 +1,4 @@ - services: nginx: - depends_on: - app1: - condition: service_healthy - app2: - condition: service_healthy - app3: - condition: service_healthy volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./nginx/upstreams/least-connections.conf:/etc/nginx/conf.d/01-upstream.conf:ro - - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/02-default.conf:ro - - ./nginx/conf.d/metrics.conf:/etc/nginx/conf.d/03-metrics.conf:ro + - ./nginx/upstreams/least-connections.conf:/etc/nginx/conf.d/01-upstream.conf:ro \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 80eb28a..0076cdc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ - services: + # ── Data layer ────────────────────────────────────── postgres: image: postgres:16-alpine environment: @@ -18,7 +18,8 @@ services: start_period: 10s restart: unless-stopped - app1: + # ── Application replicas ──────────────────────────── + app1: &app-template build: context: ./app image: sna-flask-app:latest @@ -36,13 +37,7 @@ services: expose: - "8000" healthcheck: - test: - [ - "CMD", - "python", - "-c", - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()", - ] + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()"] interval: 10s timeout: 5s retries: 3 @@ -50,9 +45,7 @@ services: restart: unless-stopped app2: - build: - context: ./app - image: sna-flask-app:latest + <<: *app-template environment: APP_NAME: ${APP_NAME:-sna-demo-app} APP_INSTANCE_ID: app-2 @@ -61,29 +54,9 @@ services: DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} - depends_on: - postgres: - condition: service_healthy - expose: - - "8000" - healthcheck: - test: - [ - "CMD", - "python", - "-c", - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()", - ] - interval: 10s - timeout: 5s - retries: 3 - start_period: 15s - restart: unless-stopped app3: - build: - context: ./app - image: sna-flask-app:latest + <<: *app-template environment: APP_NAME: ${APP_NAME:-sna-demo-app} APP_INSTANCE_ID: app-3 @@ -92,31 +65,15 @@ services: DB_CONNECT_RETRIES: ${DB_CONNECT_RETRIES:-20} DB_CONNECT_DELAY: ${DB_CONNECT_DELAY:-2} DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10} - depends_on: - postgres: - condition: service_healthy - expose: - - "8000" - healthcheck: - test: - [ - "CMD", - "python", - "-c", - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()", - ] - interval: 10s - timeout: 5s - retries: 3 - start_period: 15s - restart: unless-stopped + # ── Reverse proxy & exporter ──────────────────────── nginx: image: nginx:1.27-alpine ports: - "${NGINX_HTTP_PORT:-80}:80" volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + # Default: round-robin. Override file changes this to least-connections. - ./nginx/upstreams/round-robin.conf:/etc/nginx/conf.d/01-upstream.conf:ro - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/02-default.conf:ro - ./nginx/conf.d/metrics.conf:/etc/nginx/conf.d/03-metrics.conf:ro @@ -142,5 +99,73 @@ services: condition: service_started restart: unless-stopped + # ── Monitoring (Member B) ─────────────────────────── + prometheus: + image: prom/prometheus:v2.52.0 + container_name: prometheus + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + ports: + - "9090:9090" + restart: unless-stopped + + grafana: + image: grafana/grafana:11.0.0 + container_name: grafana + volumes: + - ./monitoring/grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml + - ./monitoring/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml + - ./monitoring/grafana/dashboards/flask-app.json:/var/lib/grafana/dashboards/flask-app.json + - grafana_data:/var/lib/grafana + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + ports: + - "3000:3000" + restart: unless-stopped + + loki: + image: grafana/loki:3.0.0 + container_name: loki + volumes: + - ./monitoring/loki/local-config.yaml:/etc/loki/local-config.yaml + - loki_data:/loki + command: -config.file=/etc/loki/local-config.yaml + ports: + - "3100:3100" + restart: unless-stopped + + promtail: + image: grafana/promtail:3.0.0 + container_name: promtail + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./monitoring/promtail/config.yml:/etc/promtail/config.yml + command: -config.file=/etc/promtail/config.yml + restart: unless-stopped + + # ── Load testing (optional) ───────────────────────── + loadtester: + build: + context: ./loadtest + image: sna-loadtester:latest + profiles: + - loadtest # Only starts with --profile loadtest + environment: + TARGET_URL: ${LOADTEST_TARGET_URL:-http://nginx} + REQUESTS_PER_SECOND: ${LOADTEST_RPS:-50} + depends_on: + - nginx + restart: "no" + volumes: postgres_data: + prometheus_data: + grafana_data: + loki_data: \ No newline at end of file diff --git a/monitoring/compose.member-b.yml b/monitoring/compose.member-b.yml deleted file mode 100644 index cfcf195..0000000 --- a/monitoring/compose.member-b.yml +++ /dev/null @@ -1,55 +0,0 @@ -services: - prometheus: - image: prom/prometheus:v2.52.0 - container_name: prometheus - volumes: - - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - - prometheus_data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/usr/share/prometheus/console_libraries' - - '--web.console.templates=/usr/share/prometheus/consoles' - ports: - - "9090:9090" - restart: unless-stopped - - grafana: - image: grafana/grafana:11.0.0 - container_name: grafana - volumes: - - ./monitoring/grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml - - ./monitoring/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml - - ./monitoring/grafana/dashboards/flask-app.json:/var/lib/grafana/dashboards/flask-app.json - - grafana_data:/var/lib/grafana - environment: - GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: admin # Change for real projects - ports: - - "3000:3000" - restart: unless-stopped - - loki: - image: grafana/loki:3.0.0 - container_name: loki - volumes: - - ./monitoring/loki/local-config.yml:/etc/loki/local-config.yml - - loki_data:/loki - command: -config.file=/etc/loki/local-config.yml - ports: - - "3100:3100" - restart: unless-stopped - - promtail: - image: grafana/promtail:3.0.0 - container_name: promtail - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./monitoring/promtail/config.yml:/etc/promtail/config.yml - command: -config.file=/etc/promtail/config.yml - restart: unless-stopped - -volumes: - prometheus_data: - grafana_data: - loki_data: \ No newline at end of file diff --git a/monitoring/loki/local-config.yml b/monitoring/loki/local-config.yaml similarity index 100% rename from monitoring/loki/local-config.yml rename to monitoring/loki/local-config.yaml From 16e15434fd8598827ccbedcc28f08f65588db8ff Mon Sep 17 00:00:00 2001 From: kostya2505 Date: Fri, 15 May 2026 02:03:43 +0800 Subject: [PATCH 14/23] fix: prometheus targets all apps --- monitoring/prometheus/prometheus.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index fb8ef07..b635dc0 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -5,17 +5,15 @@ global: scrape_configs: - job_name: 'flask-app' static_configs: - - targets: ['app:8000'] # Flask /metrics endpoint from Member C - labels: - service: 'flask' + - targets: + - 'app1:8000' + - 'app2:8000' + - 'app3:8000' + labels: + service: 'flask' - job_name: 'nginx-exporter' static_configs: - targets: ['nginx-exporter:9113'] # NGINX metrics exporter (Member A must expose) labels: - service: 'nginx' - - # optional: add node_exporter if you want host/container sys metrics - - job_name: 'node' - static_configs: - - targets: ['node-exporter:9100'] \ No newline at end of file + service: 'nginx' \ No newline at end of file From d886a91a63cc20428318faf45c00b8221bac4daf Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 00:23:56 +0300 Subject: [PATCH 15/23] README.md changed --- README.md | 545 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 538 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e0c13df..c51d781 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,542 @@ # Containerized Observability App -Group project for [Course Name / S25]. -Team: Member A (Infrastructure & Proxy), Member B (Observability), Member C (Application & Data). +This project is a small production-style demo platform built to show how an application can be deployed behind a reverse proxy, scaled horizontally, observed with metrics and logs, and exercised with synthetic traffic. -## Quick start +The stack includes: -1. Copy `.env.example` to `.env` and adjust values if needed. -2. Start the full stack (round‑robin load balancing): - ```bash - docker compose up -d \ No newline at end of file +- A Flask API with Prometheus instrumentation and structured JSON logging +- Three application replicas behind NGINX +- PostgreSQL for persistent storage +- Prometheus for metrics collection +- Grafana for dashboards +- Loki + Promtail for centralized container logs +- An optional load generator for demos and testing +- A PostgreSQL backup script + +The goal is not just to run a web app, but to demonstrate the full operational flow around it: routing, persistence, health checks, observability, rate limiting, and repeatable local deployment with Docker Compose. + +## What the Project Does + +The application exposes a small HTTP API that: + +- returns service metadata from the root endpoint +- reports health and readiness +- records visit events in PostgreSQL +- stores and lists short messages +- exports Prometheus metrics + +Every request goes through NGINX, which forwards traffic to one of three Flask replicas. Each replica: + +- writes structured logs to stdout +- exports HTTP and custom business metrics on `/metrics` +- adds `X-Request-ID` and `X-App-Instance` headers to responses +- persists data into PostgreSQL through a connection pool + +The observability stack collects both metrics and logs: + +- Prometheus scrapes all three Flask instances and the NGINX exporter +- Grafana is pre-provisioned with Prometheus and Loki data sources +- Promtail discovers Docker containers through the Docker socket +- Loki stores container logs for log exploration in Grafana + +## Architecture + +```text +Client + | + v +NGINX (:80) + | + +--> app1 (:8000) + +--> app2 (:8000) + +--> app3 (:8000) + | + v + PostgreSQL (:5432, internal) + +Metrics flow: +Flask /metrics ----------> Prometheus (:9090) ----------> Grafana (:3000) +NGINX stub_status -> exporter (:9113) --/ + +Logs flow: +Docker container logs -> Promtail -> Loki (:3100) -> Grafana +``` + +## Main Components + +### Application + +The Flask app lives in [`app/`](./app) and is started with Gunicorn inside a Python 3.12 container. + +It includes: + +- automatic database connection retries on startup +- on-start schema safety through `CREATE TABLE IF NOT EXISTS` +- Prometheus HTTP metrics via `prometheus-flask-exporter` +- custom counters: + - `app_visit_events_total` + - `app_messages_created_total` +- structured JSON logs via `python-json-logger` + +### Database + +PostgreSQL stores two tables: + +- `visit_events` +- `messages` + +The schema is initialized in two ways: + +- `database/init/01-init.sql` creates tables and inserts a seed message +- the Flask app also runs `ensure_schema()` on startup for extra safety + +### Reverse Proxy + +NGINX provides: + +- a single public entrypoint on port `80` +- load balancing across `app1`, `app2`, and `app3` +- forwarded client headers +- request rate limiting +- a `stub_status` endpoint on port `8080` for exporter scraping + +Two balancing modes are supported: + +- round robin by default +- least connections through an override Compose file + +### Monitoring and Logging + +The `monitoring/` directory provisions the full observability layer: + +- Prometheus scrape configuration +- Grafana data sources +- Grafana dashboard provisioning +- Loki configuration +- Promtail configuration + +The prebuilt Grafana dashboard shows: + +- request rate +- p95 request duration +- total visit events +- total created messages +- process RSS memory + +### Load Testing + +The `loadtest/` service generates mixed traffic against the app: + +- `GET /` +- `GET /api/visits` +- `POST /api/messages` + +It prints a JSON summary with: + +- total requests +- errors +- effective requests per second +- latency statistics +- status code distribution +- endpoint distribution + +### Backups + +The `scripts/backup_postgres.sh` script creates a compressed PostgreSQL dump in the local `backups/` directory using `pg_dump` executed inside the running database container. + +## Project Structure + +```text +. +|-- app/ +| |-- Dockerfile +| |-- main.py +| `-- requirements.txt +|-- database/ +| `-- init/ +| `-- 01-init.sql +|-- loadtest/ +| |-- Dockerfile +| |-- load_test.py +| `-- requirements.txt +|-- monitoring/ +| |-- grafana/ +| | |-- dashboards/ +| | | `-- flask-app.json +| | |-- dashboards.yml +| | `-- datasources.yml +| |-- loki/ +| | `-- local-config.yaml +| |-- prometheus/ +| | `-- prometheus.yml +| |-- promtail/ +| | `-- config.yml +| `-- README.md +|-- nginx/ +| |-- conf.d/ +| | |-- default.conf +| | `-- metrics.conf +| |-- upstreams/ +| | |-- least-connections.conf +| | `-- round-robin.conf +| `-- nginx.conf +|-- scripts/ +| `-- backup_postgres.sh +|-- backups/ +|-- docker-compose.yml +|-- docker-compose.leastconn.yml +|-- .env.example +`-- LICENSE +``` + +## Requirements + +Before starting, make sure you have: + +- Docker +- Docker Compose v2 +- free local ports for `80`, `3000`, `3100`, `9090`, and `9113` + +## Environment Variables + +Copy the example environment file first: + +```bash +cp .env.example .env +``` + +The main variables are: + +| Variable | Default | Purpose | +|---|---|---| +| `APP_NAME` | `sna-demo-app` | Logical application name returned by the API | +| `PORT` | `8000` | Internal Flask/Gunicorn port | +| `POSTGRES_DB` | `app_db` | PostgreSQL database name | +| `POSTGRES_USER` | `app_user` | PostgreSQL user | +| `POSTGRES_PASSWORD` | `app_password` | PostgreSQL password | +| `POSTGRES_HOST` | `postgres` | Database hostname inside Docker network | +| `POSTGRES_PORT` | `5432` | Database port | +| `DB_CONNECT_RETRIES` | `20` | Number of DB connection retry attempts | +| `DB_CONNECT_DELAY` | `2` | Delay between DB retries in seconds | +| `DB_POOL_MAX_SIZE` | `10` | Max size of the app DB pool | + +Additional variables in `.env.example` are useful for manual load-test runs, but the load generator itself reads `TARGET_URL` and the `LOAD_*` variables when the container starts. + +## How to Run the Project + +### 1. Start the default stack + +This starts PostgreSQL, three app replicas, NGINX, Prometheus, Grafana, Loki, Promtail, and the NGINX exporter. + +```bash +docker compose up -d --build +``` + +### 2. Open the services + +| Service | URL | Notes | +|---|---|---| +| Application entrypoint | `http://localhost/` | Goes through NGINX | +| Grafana | `http://localhost:3000` | Login: `admin` / `admin` | +| Prometheus | `http://localhost:9090` | Scrape targets and PromQL | +| Loki | `http://localhost:3100` | Usually consumed through Grafana | +| NGINX exporter | `http://localhost:9113/metrics` | Exported NGINX metrics | + +### 3. Check that the stack is healthy + +```bash +curl http://localhost/health +curl http://localhost/ready +curl http://localhost/api/visits +curl http://localhost/api/messages +``` + +## Using the Least-Connections Balancer + +Round robin is the default. To switch to least connections, start Compose with the override file: + +```bash +docker compose -f docker-compose.yml -f docker-compose.leastconn.yml up -d --build +``` + +That override only changes the mounted upstream file used by NGINX. + +## API Reference + +### `GET /` + +Returns general service information, including: + +- application name +- replica instance ID +- hostname +- current UTC timestamp +- available endpoints + +### `GET /health` + +Simple liveness endpoint. It confirms that the application process is running. + +### `GET /ready` + +Readiness endpoint. It checks whether the app can successfully connect to PostgreSQL. If the database is unavailable, it returns HTTP `503`. + +### `GET /metrics` + +Prometheus metrics endpoint exposed by each Flask replica. + +### `GET /api/visits` + +Creates a new row in `visit_events` and returns: + +- the created visit record +- the total number of recorded visits + +This endpoint is intentionally state-changing so that dashboards have write activity to observe. + +### `GET /api/messages?limit=N` + +Returns the most recent messages ordered by `created_at DESC`. + +Rules: + +- default limit is `10` +- maximum limit is `100` +- non-integer values return HTTP `400` + +### `POST /api/messages` + +Stores a message in PostgreSQL. + +Example: + +```bash +curl -X POST http://localhost/api/messages \ + -H 'Content-Type: application/json' \ + -d '{"message":"hello from reviewer"}' +``` + +Behavior: + +- empty or missing input becomes an auto-generated message +- messages are trimmed to 500 characters +- successful creation returns HTTP `201` + +## How Load Balancing Works + +The public endpoint is NGINX. It proxies requests to the upstream group named `flask_backend`, which contains: + +- `app1:8000` +- `app2:8000` +- `app3:8000` + +To help demonstrate which replica handled a request, the Flask app adds: + +- `X-Request-ID` +- `X-App-Instance` + +You can verify balancing with: + +```bash +for i in $(seq 1 6); do + curl -s -D - http://localhost/ -o /dev/null | grep X-App-Instance +done +``` + +## Rate Limiting + +NGINX applies two request-rate policies: + +- general traffic: `50r/s` with `burst=100` +- `/api/` traffic: `10r/s` with `burst=20` + +When a limit is exceeded, NGINX returns HTTP `429`. + +The `/health` location is intentionally lightweight and does not apply `limit_req`. + +## Observability Guide + +### Metrics + +Prometheus scrapes: + +- `app1:8000/metrics` +- `app2:8000/metrics` +- `app3:8000/metrics` +- `nginx-exporter:9113/metrics` + +Useful metric families include: + +- `flask_http_request_total` +- `flask_http_request_duration_seconds_*` +- `app_visit_events_total` +- `app_messages_created_total` +- `process_resident_memory_bytes` + +### Dashboards + +Grafana is preconfigured automatically. After login: + +1. Open the default dashboard list. +2. Select `Flask App Dashboard`. +3. Generate a little traffic if the panels are empty at first. + +### Logs + +Promtail reads Docker container logs from `/var/run/docker.sock` and sends them to Loki. + +The Flask app writes JSON logs with fields such as: + +- log level +- logger name +- message +- request path +- request method +- response status +- request duration +- forwarded client IP + +In Grafana Explore, you can inspect logs with queries such as: + +```logql +{compose_service="app1"} +{compose_service="nginx"} +{level="INFO"} +``` + +## Load Testing + +The load generator is optional and attached to the `loadtest` profile. + +### Run with defaults + +```bash +docker compose --profile loadtest run --rm loadtester +``` + +### Run with custom parameters + +```bash +docker compose --profile loadtest run --rm \ + -e TARGET_URL=http://nginx \ + -e LOAD_DURATION=120 \ + -e LOAD_CONCURRENCY=20 \ + -e LOAD_INTERVAL=0.05 \ + -e LOAD_WRITE_RATIO=0.35 \ + loadtester +``` + +Parameter meaning: + +| Variable | Meaning | +|---|---| +| `TARGET_URL` | Base URL to attack, usually `http://nginx` inside Compose | +| `LOAD_DURATION` | Test duration in seconds | +| `LOAD_CONCURRENCY` | Number of concurrent async workers | +| `LOAD_INTERVAL` | Sleep time between worker requests | +| `LOAD_WRITE_RATIO` | Probability of sending `POST /api/messages` | + +This is useful for: + +- populating Grafana charts +- testing rate limits +- generating logs for Loki +- showing behavior under concurrent traffic + +## Database Backups + +Create a compressed SQL backup with: + +```bash +bash scripts/backup_postgres.sh +``` + +The script: + +- detects the Compose file automatically +- runs `pg_dump` inside the `postgres` service +- compresses the dump with `gzip` +- stores the result in `backups/` + +Common override variables: + +| Variable | Purpose | +|---|---| +| `BACKUP_DIR` | Where the backup file will be written | +| `POSTGRES_SERVICE_NAME` | Compose service name of PostgreSQL | +| `POSTGRES_DB` | Database name | +| `POSTGRES_USER` | Database user | +| `POSTGRES_PASSWORD` | Database password | +| `COMPOSE_FILE_PATH` | Explicit Compose file path | + +## Data Persistence + +Docker named volumes are used for: + +- PostgreSQL data +- Prometheus TSDB data +- Grafana state +- Loki data + +Local files in `backups/` are stored on the host so they remain available outside the containers. + +## Stop and Clean Up + +Stop the stack: + +```bash +docker compose down +``` + +Stop the least-connections variant: + +```bash +docker compose -f docker-compose.yml -f docker-compose.leastconn.yml down +``` + +Remove containers and volumes: + +```bash +docker compose down -v +``` + +## Troubleshooting + +### `docker compose up` fails because a port is already in use + +Free the conflicting local port or override the published ports before starting the stack. + +### `/ready` returns `503` + +The app is reachable, but PostgreSQL is not ready yet or the DB settings are incorrect. Wait a few seconds and try again. + +### Grafana opens but panels are empty + +This usually means there has not been enough traffic yet. Call the API a few times or run the load generator. + +### You receive HTTP `429` + +That means NGINX rate limiting is working. Slow down the request rate or reduce concurrency. + +### No logs appear in Loki + +Make sure Promtail is running and still has access to `/var/run/docker.sock`. + +## Why a Reviewer Should Have Confidence in This Project + +This repository demonstrates more than a basic Flask app: + +- it is containerized end to end +- it runs multiple replicas behind a real reverse proxy +- it persists data in PostgreSQL +- it exposes health, readiness, and metrics endpoints +- it includes built-in observability for both metrics and logs +- it supports two balancing strategies +- it includes a repeatable load generator +- it includes an operational backup script + +In other words, the project shows both application functionality and the operational concerns required to run and observe it like a small real-world service. + +## License + +This project is licensed under the MIT License. See [`LICENSE`](./LICENSE). From a894de477e2b0eede01e5249db1963ec710c5270 Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 01:28:30 +0300 Subject: [PATCH 16/23] Polish observability project: fix docs, monitoring, routing, and CI --- .env.example | 6 +- .github/workflows/ci-cd.yml | 113 ++++ README.md | 552 +++--------------- docker-compose.leastconn.yml | 2 +- docker-compose.yml | 41 +- monitoring/README.md | 46 -- monitoring/grafana/dashboards/flask-app.json | 21 +- .../dashboards/infrastructure-overview.json | 102 ++++ monitoring/prometheus/prometheus.yml | 14 +- nginx/conf.d/default.conf | 13 +- scripts/smoke_test.sh | 230 ++++++++ 11 files changed, 603 insertions(+), 537 deletions(-) create mode 100644 .github/workflows/ci-cd.yml delete mode 100644 monitoring/README.md create mode 100644 monitoring/grafana/dashboards/infrastructure-overview.json create mode 100755 scripts/smoke_test.sh diff --git a/.env.example b/.env.example index fc9d375..9689358 100644 --- a/.env.example +++ b/.env.example @@ -11,8 +11,12 @@ DB_CONNECT_RETRIES=20 DB_CONNECT_DELAY=2 DB_POOL_MAX_SIZE=10 +NGINX_HTTP_PORT=80 +NGINX_EXPORTER_PORT=9113 + LOAD_DURATION=120 LOAD_CONCURRENCY=20 LOAD_INTERVAL=0.05 LOAD_WRITE_RATIO=0.35 -TARGET_URL=http://app:8000 +LOAD_TIMEOUT=5 +LOADTEST_TARGET_URL=http://nginx diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..a251e90 --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,113 @@ +name: CI / CD + +on: + pull_request: + push: + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + name: Validate Source + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Prepare executable scripts + run: chmod +x scripts/*.sh + + - name: Validate Docker Compose + run: docker compose config + + - name: Validate Python syntax + run: python3 -m py_compile app/main.py loadtest/load_test.py + + - name: Validate shell scripts + run: | + bash -n scripts/backup_postgres.sh + bash -n scripts/smoke_test.sh + + smoke-deploy: + name: Smoke Deploy + runs-on: ubuntu-latest + needs: validate + env: + NGINX_HTTP_PORT: 8088 + BASE_URL: http://localhost:8088 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Prepare executable scripts + run: chmod +x scripts/*.sh + + - name: Start full stack + run: docker compose up -d --build + + - name: Run smoke test + run: ./scripts/smoke_test.sh + + - name: Exercise bundled load tester + run: | + docker compose --profile loadtest run --rm --no-deps \ + -e LOAD_DURATION=10 \ + -e LOAD_CONCURRENCY=8 \ + -e LOAD_INTERVAL=0.05 \ + -e LOAD_TIMEOUT=5 \ + loadtester + + - name: Show service status on failure + if: failure() + run: | + docker compose ps + docker compose logs --no-color + + - name: Tear down stack + if: always() + run: docker compose down -v --remove-orphans + + publish-app-image: + name: Publish App Image + runs-on: ubuntu-latest + needs: smoke-deploy + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/containerized-observability-app + tags: | + type=raw,value=latest + type=sha + + - name: Build and publish app image + uses: docker/build-push-action@v6 + with: + context: ./app + file: ./app/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/README.md b/README.md index c51d781..53e1748 100644 --- a/README.md +++ b/README.md @@ -1,542 +1,146 @@ -# Containerized Observability App +# Student Project: Containerized Observability App -This project is a small production-style demo platform built to show how an application can be deployed behind a reverse proxy, scaled horizontally, observed with metrics and logs, and exercised with synthetic traffic. +## Project Topic -The stack includes: +**Highly Available Containerized Web App with Centralized Monitoring and Logging** -- A Flask API with Prometheus instrumentation and structured JSON logging -- Three application replicas behind NGINX -- PostgreSQL for persistent storage -- Prometheus for metrics collection -- Grafana for dashboards -- Loki + Promtail for centralized container logs -- An optional load generator for demos and testing -- A PostgreSQL backup script +This is a student project for Docker and observability practice. +We built a simple Flask web app with PostgreSQL, ran multiple replicas with Docker Compose, used NGINX as a reverse proxy, and added monitoring, logging, and CI/CD. -The goal is not just to run a web app, but to demonstrate the full operational flow around it: routing, persistence, health checks, observability, rate limiting, and repeatable local deployment with Docker Compose. +## Stack -## What the Project Does +- Python Flask +- PostgreSQL +- Docker Compose +- NGINX +- Prometheus +- Grafana +- Loki +- Promtail +- Node Exporter +- cAdvisor +- GitHub Actions -The application exposes a small HTTP API that: +## What is included -- returns service metadata from the root endpoint -- reports health and readiness -- records visit events in PostgreSQL -- stores and lists short messages -- exports Prometheus metrics +- Flask API +- PostgreSQL database +- 3 app replicas +- NGINX load balancing and rate limiting +- Prometheus and Grafana for metrics +- Loki and Promtail for logs +- Node Exporter and cAdvisor for host/container metrics +- GitHub Actions CI/CD +- PostgreSQL backup script -Every request goes through NGINX, which forwards traffic to one of three Flask replicas. Each replica: - -- writes structured logs to stdout -- exports HTTP and custom business metrics on `/metrics` -- adds `X-Request-ID` and `X-App-Instance` headers to responses -- persists data into PostgreSQL through a connection pool - -The observability stack collects both metrics and logs: - -- Prometheus scrapes all three Flask instances and the NGINX exporter -- Grafana is pre-provisioned with Prometheus and Loki data sources -- Promtail discovers Docker containers through the Docker socket -- Loki stores container logs for log exploration in Grafana - -## Architecture - -```text -Client - | - v -NGINX (:80) - | - +--> app1 (:8000) - +--> app2 (:8000) - +--> app3 (:8000) - | - v - PostgreSQL (:5432, internal) - -Metrics flow: -Flask /metrics ----------> Prometheus (:9090) ----------> Grafana (:3000) -NGINX stub_status -> exporter (:9113) --/ - -Logs flow: -Docker container logs -> Promtail -> Loki (:3100) -> Grafana -``` - -## Main Components - -### Application - -The Flask app lives in [`app/`](./app) and is started with Gunicorn inside a Python 3.12 container. - -It includes: - -- automatic database connection retries on startup -- on-start schema safety through `CREATE TABLE IF NOT EXISTS` -- Prometheus HTTP metrics via `prometheus-flask-exporter` -- custom counters: - - `app_visit_events_total` - - `app_messages_created_total` -- structured JSON logs via `python-json-logger` - -### Database - -PostgreSQL stores two tables: - -- `visit_events` -- `messages` - -The schema is initialized in two ways: - -- `database/init/01-init.sql` creates tables and inserts a seed message -- the Flask app also runs `ensure_schema()` on startup for extra safety - -### Reverse Proxy - -NGINX provides: - -- a single public entrypoint on port `80` -- load balancing across `app1`, `app2`, and `app3` -- forwarded client headers -- request rate limiting -- a `stub_status` endpoint on port `8080` for exporter scraping - -Two balancing modes are supported: - -- round robin by default -- least connections through an override Compose file - -### Monitoring and Logging - -The `monitoring/` directory provisions the full observability layer: - -- Prometheus scrape configuration -- Grafana data sources -- Grafana dashboard provisioning -- Loki configuration -- Promtail configuration - -The prebuilt Grafana dashboard shows: - -- request rate -- p95 request duration -- total visit events -- total created messages -- process RSS memory - -### Load Testing - -The `loadtest/` service generates mixed traffic against the app: - -- `GET /` -- `GET /api/visits` -- `POST /api/messages` - -It prints a JSON summary with: - -- total requests -- errors -- effective requests per second -- latency statistics -- status code distribution -- endpoint distribution - -### Backups - -The `scripts/backup_postgres.sh` script creates a compressed PostgreSQL dump in the local `backups/` directory using `pg_dump` executed inside the running database container. - -## Project Structure - -```text -. -|-- app/ -| |-- Dockerfile -| |-- main.py -| `-- requirements.txt -|-- database/ -| `-- init/ -| `-- 01-init.sql -|-- loadtest/ -| |-- Dockerfile -| |-- load_test.py -| `-- requirements.txt -|-- monitoring/ -| |-- grafana/ -| | |-- dashboards/ -| | | `-- flask-app.json -| | |-- dashboards.yml -| | `-- datasources.yml -| |-- loki/ -| | `-- local-config.yaml -| |-- prometheus/ -| | `-- prometheus.yml -| |-- promtail/ -| | `-- config.yml -| `-- README.md -|-- nginx/ -| |-- conf.d/ -| | |-- default.conf -| | `-- metrics.conf -| |-- upstreams/ -| | |-- least-connections.conf -| | `-- round-robin.conf -| `-- nginx.conf -|-- scripts/ -| `-- backup_postgres.sh -|-- backups/ -|-- docker-compose.yml -|-- docker-compose.leastconn.yml -|-- .env.example -`-- LICENSE -``` - -## Requirements - -Before starting, make sure you have: - -- Docker -- Docker Compose v2 -- free local ports for `80`, `3000`, `3100`, `9090`, and `9113` - -## Environment Variables - -Copy the example environment file first: +## Run the project ```bash cp .env.example .env -``` - -The main variables are: - -| Variable | Default | Purpose | -|---|---|---| -| `APP_NAME` | `sna-demo-app` | Logical application name returned by the API | -| `PORT` | `8000` | Internal Flask/Gunicorn port | -| `POSTGRES_DB` | `app_db` | PostgreSQL database name | -| `POSTGRES_USER` | `app_user` | PostgreSQL user | -| `POSTGRES_PASSWORD` | `app_password` | PostgreSQL password | -| `POSTGRES_HOST` | `postgres` | Database hostname inside Docker network | -| `POSTGRES_PORT` | `5432` | Database port | -| `DB_CONNECT_RETRIES` | `20` | Number of DB connection retry attempts | -| `DB_CONNECT_DELAY` | `2` | Delay between DB retries in seconds | -| `DB_POOL_MAX_SIZE` | `10` | Max size of the app DB pool | - -Additional variables in `.env.example` are useful for manual load-test runs, but the load generator itself reads `TARGET_URL` and the `LOAD_*` variables when the container starts. - -## How to Run the Project - -### 1. Start the default stack - -This starts PostgreSQL, three app replicas, NGINX, Prometheus, Grafana, Loki, Promtail, and the NGINX exporter. - -```bash docker compose up -d --build ``` -### 2. Open the services +## Useful links -| Service | URL | Notes | -|---|---|---| -| Application entrypoint | `http://localhost/` | Goes through NGINX | -| Grafana | `http://localhost:3000` | Login: `admin` / `admin` | -| Prometheus | `http://localhost:9090` | Scrape targets and PromQL | -| Loki | `http://localhost:3100` | Usually consumed through Grafana | -| NGINX exporter | `http://localhost:9113/metrics` | Exported NGINX metrics | +- App: `http://127.0.0.1/` +- Grafana: `http://127.0.0.1:3000` +- Prometheus: `http://127.0.0.1:9090` +- Loki: `http://127.0.0.1:3100` +- NGINX exporter: `http://127.0.0.1:9113/metrics` -### 3. Check that the stack is healthy +Grafana login: -```bash -curl http://localhost/health -curl http://localhost/ready -curl http://localhost/api/visits -curl http://localhost/api/messages -``` +- user: `admin` +- password: `admin` -## Using the Least-Connections Balancer - -Round robin is the default. To switch to least connections, start Compose with the override file: +## Quick check ```bash -docker compose -f docker-compose.yml -f docker-compose.leastconn.yml up -d --build +curl http://127.0.0.1/health +curl http://127.0.0.1/ready +curl http://127.0.0.1/api/visits +curl http://127.0.0.1/api/messages ``` -That override only changes the mounted upstream file used by NGINX. - -## API Reference - -### `GET /` - -Returns general service information, including: - -- application name -- replica instance ID -- hostname -- current UTC timestamp -- available endpoints - -### `GET /health` - -Simple liveness endpoint. It confirms that the application process is running. - -### `GET /ready` - -Readiness endpoint. It checks whether the app can successfully connect to PostgreSQL. If the database is unavailable, it returns HTTP `503`. - -### `GET /metrics` - -Prometheus metrics endpoint exposed by each Flask replica. - -### `GET /api/visits` - -Creates a new row in `visit_events` and returns: - -- the created visit record -- the total number of recorded visits - -This endpoint is intentionally state-changing so that dashboards have write activity to observe. - -### `GET /api/messages?limit=N` - -Returns the most recent messages ordered by `created_at DESC`. - -Rules: - -- default limit is `10` -- maximum limit is `100` -- non-integer values return HTTP `400` - -### `POST /api/messages` - -Stores a message in PostgreSQL. - -Example: +You can also run: ```bash -curl -X POST http://localhost/api/messages \ - -H 'Content-Type: application/json' \ - -d '{"message":"hello from reviewer"}' +bash scripts/smoke_test.sh ``` -Behavior: +## Load balancing -- empty or missing input becomes an auto-generated message -- messages are trimmed to 500 characters -- successful creation returns HTTP `201` +Default mode is `round robin`. -## How Load Balancing Works - -The public endpoint is NGINX. It proxies requests to the upstream group named `flask_backend`, which contains: - -- `app1:8000` -- `app2:8000` -- `app3:8000` - -To help demonstrate which replica handled a request, the Flask app adds: - -- `X-Request-ID` -- `X-App-Instance` - -You can verify balancing with: +Check: ```bash for i in $(seq 1 6); do - curl -s -D - http://localhost/ -o /dev/null | grep X-App-Instance + curl -s -D - http://127.0.0.1/ -o /dev/null | grep X-App-Instance done ``` -## Rate Limiting - -NGINX applies two request-rate policies: - -- general traffic: `50r/s` with `burst=100` -- `/api/` traffic: `10r/s` with `burst=20` - -When a limit is exceeded, NGINX returns HTTP `429`. - -The `/health` location is intentionally lightweight and does not apply `limit_req`. - -## Observability Guide - -### Metrics - -Prometheus scrapes: - -- `app1:8000/metrics` -- `app2:8000/metrics` -- `app3:8000/metrics` -- `nginx-exporter:9113/metrics` - -Useful metric families include: +If you want `least connections`: -- `flask_http_request_total` -- `flask_http_request_duration_seconds_*` -- `app_visit_events_total` -- `app_messages_created_total` -- `process_resident_memory_bytes` - -### Dashboards - -Grafana is preconfigured automatically. After login: +```bash +docker compose -f docker-compose.yml -f docker-compose.leastconn.yml up -d --build +``` -1. Open the default dashboard list. -2. Select `Flask App Dashboard`. -3. Generate a little traffic if the panels are empty at first. +## Monitoring and logs -### Logs +Prometheus collects metrics from: -Promtail reads Docker container logs from `/var/run/docker.sock` and sends them to Loki. +- `app1` +- `app2` +- `app3` +- `nginx-exporter` +- `node-exporter` +- `cadvisor` -The Flask app writes JSON logs with fields such as: +Grafana dashboards: -- log level -- logger name -- message -- request path -- request method -- response status -- request duration -- forwarded client IP +- `Flask App Dashboard` +- `Infrastructure Overview` -In Grafana Explore, you can inspect logs with queries such as: +Example Loki query in Grafana Explore: ```logql -{compose_service="app1"} -{compose_service="nginx"} -{level="INFO"} +{container=~"containerized-observability-app.*"} ``` -## Load Testing - -The load generator is optional and attached to the `loadtest` profile. - -### Run with defaults +## Load test ```bash -docker compose --profile loadtest run --rm loadtester +docker compose --profile loadtest run --rm --no-deps loadtester ``` -### Run with custom parameters - -```bash -docker compose --profile loadtest run --rm \ - -e TARGET_URL=http://nginx \ - -e LOAD_DURATION=120 \ - -e LOAD_CONCURRENCY=20 \ - -e LOAD_INTERVAL=0.05 \ - -e LOAD_WRITE_RATIO=0.35 \ - loadtester -``` - -Parameter meaning: - -| Variable | Meaning | -|---|---| -| `TARGET_URL` | Base URL to attack, usually `http://nginx` inside Compose | -| `LOAD_DURATION` | Test duration in seconds | -| `LOAD_CONCURRENCY` | Number of concurrent async workers | -| `LOAD_INTERVAL` | Sleep time between worker requests | -| `LOAD_WRITE_RATIO` | Probability of sending `POST /api/messages` | - -This is useful for: - -- populating Grafana charts -- testing rate limits -- generating logs for Loki -- showing behavior under concurrent traffic - -## Database Backups - -Create a compressed SQL backup with: +## Database backup ```bash bash scripts/backup_postgres.sh ``` -The script: - -- detects the Compose file automatically -- runs `pg_dump` inside the `postgres` service -- compresses the dump with `gzip` -- stores the result in `backups/` - -Common override variables: +## CI/CD -| Variable | Purpose | -|---|---| -| `BACKUP_DIR` | Where the backup file will be written | -| `POSTGRES_SERVICE_NAME` | Compose service name of PostgreSQL | -| `POSTGRES_DB` | Database name | -| `POSTGRES_USER` | Database user | -| `POSTGRES_PASSWORD` | Database password | -| `COMPOSE_FILE_PATH` | Explicit Compose file path | +Workflow file: -## Data Persistence +`.github/workflows/ci-cd.yml` -Docker named volumes are used for: +The pipeline: -- PostgreSQL data -- Prometheus TSDB data -- Grafana state -- Loki data +- validates the project +- starts the stack +- runs the smoke test +- publishes the app image on push to `main` -Local files in `backups/` are stored on the host so they remain available outside the containers. - -## Stop and Clean Up - -Stop the stack: +## Stop the project ```bash docker compose down ``` -Stop the least-connections variant: - -```bash -docker compose -f docker-compose.yml -f docker-compose.leastconn.yml down -``` - -Remove containers and volumes: +Full cleanup: ```bash docker compose down -v ``` - -## Troubleshooting - -### `docker compose up` fails because a port is already in use - -Free the conflicting local port or override the published ports before starting the stack. - -### `/ready` returns `503` - -The app is reachable, but PostgreSQL is not ready yet or the DB settings are incorrect. Wait a few seconds and try again. - -### Grafana opens but panels are empty - -This usually means there has not been enough traffic yet. Call the API a few times or run the load generator. - -### You receive HTTP `429` - -That means NGINX rate limiting is working. Slow down the request rate or reduce concurrency. - -### No logs appear in Loki - -Make sure Promtail is running and still has access to `/var/run/docker.sock`. - -## Why a Reviewer Should Have Confidence in This Project - -This repository demonstrates more than a basic Flask app: - -- it is containerized end to end -- it runs multiple replicas behind a real reverse proxy -- it persists data in PostgreSQL -- it exposes health, readiness, and metrics endpoints -- it includes built-in observability for both metrics and logs -- it supports two balancing strategies -- it includes a repeatable load generator -- it includes an operational backup script - -In other words, the project shows both application functionality and the operational concerns required to run and observe it like a small real-world service. - -## License - -This project is licensed under the MIT License. See [`LICENSE`](./LICENSE). diff --git a/docker-compose.leastconn.yml b/docker-compose.leastconn.yml index 8c4c699..5f910a4 100644 --- a/docker-compose.leastconn.yml +++ b/docker-compose.leastconn.yml @@ -1,4 +1,4 @@ services: nginx: volumes: - - ./nginx/upstreams/least-connections.conf:/etc/nginx/conf.d/01-upstream.conf:ro \ No newline at end of file + - ./nginx/upstreams/least-connections.conf:/etc/nginx/conf.d/upstream.conf:ro diff --git a/docker-compose.yml b/docker-compose.yml index 0076cdc..dbe3c32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,10 +73,11 @@ services: - "${NGINX_HTTP_PORT:-80}:80" volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - # Default: round-robin. Override file changes this to least-connections. - - ./nginx/upstreams/round-robin.conf:/etc/nginx/conf.d/01-upstream.conf:ro - - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/02-default.conf:ro - - ./nginx/conf.d/metrics.conf:/etc/nginx/conf.d/03-metrics.conf:ro + # Mount the upstream and public vhost over the default image config so localhost + # always hits the reverse proxy instead of NGINX's bundled welcome page. + - ./nginx/upstreams/round-robin.conf:/etc/nginx/conf.d/upstream.conf:ro + - ./nginx/conf.d/default.conf:/etc/nginx/conf.d/default.conf:ro + - ./nginx/conf.d/metrics.conf:/etc/nginx/conf.d/metrics.conf:ro depends_on: app1: condition: service_healthy @@ -99,6 +100,28 @@ services: condition: service_started restart: unless-stopped + node-exporter: + image: prom/node-exporter:v1.8.1 + command: + - '--path.rootfs=/host' + volumes: + - /:/host:ro + expose: + - "9100" + restart: unless-stopped + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + privileged: true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + expose: + - "8080" + restart: unless-stopped + # ── Monitoring (Member B) ─────────────────────────── prometheus: image: prom/prometheus:v2.52.0 @@ -121,7 +144,7 @@ services: volumes: - ./monitoring/grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml - ./monitoring/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml - - ./monitoring/grafana/dashboards/flask-app.json:/var/lib/grafana/dashboards/flask-app.json + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro - grafana_data:/var/lib/grafana environment: GF_SECURITY_ADMIN_USER: admin @@ -159,7 +182,11 @@ services: - loadtest # Only starts with --profile loadtest environment: TARGET_URL: ${LOADTEST_TARGET_URL:-http://nginx} - REQUESTS_PER_SECOND: ${LOADTEST_RPS:-50} + LOAD_DURATION: ${LOAD_DURATION:-120} + LOAD_CONCURRENCY: ${LOAD_CONCURRENCY:-20} + LOAD_INTERVAL: ${LOAD_INTERVAL:-0.05} + LOAD_WRITE_RATIO: ${LOAD_WRITE_RATIO:-0.35} + LOAD_TIMEOUT: ${LOAD_TIMEOUT:-5} depends_on: - nginx restart: "no" @@ -168,4 +195,4 @@ volumes: postgres_data: prometheus_data: grafana_data: - loki_data: \ No newline at end of file + loki_data: diff --git a/monitoring/README.md b/monitoring/README.md deleted file mode 100644 index 93468de..0000000 --- a/monitoring/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Member B Deliverables - -This directory contains the observability stack for the team project: - -- Prometheus configured to scrape the Flask app and NGINX exporter. -- Grafana with provisioned Prometheus and Loki datasources. -- A pre-built Grafana dashboard for the Flask application. -- Loki + Promtail for container log aggregation. -- A Compose fragment ready to merge into the final `docker-compose.yml`. - -## Project layout - -- `compose.member-b.yml`: Compose fragment with all observability services (Member A merges this). -- `prometheus/prometheus.yml`: Scrape configs for Flask app and NGINX exporter. -- `grafana/datasources.yml`: Pre-configures Prometheus and Loki as data sources in Grafana. -- `grafana/dashboards.yml`: Dashboard provider config pointing to the `dashboards/` folder. -- `grafana/dashboards/flask-app.json`: A ready-to-use dashboard showing request rate, latency, error rate, and visit count. -- `loki/local-config.yaml`: Loki server settings (in-memory ring, filesystem storage). -- `promtail/config.yml`: Promtail configuration that discovers all running containers via Docker socket and ships logs to Loki. -- `README.md`: This file. - -## Observability endpoints - -Once the final stack is running, the following will be available: - -| Service | Port | URL | Description | -|------------|-------|---------------------------|------------------------------------| -| Prometheus | 9090 | `http://localhost:9090` | Metrics query & alerting UI | -| Grafana | 3000 | `http://localhost:3000` | Dashboards (login: `admin`/`admin`)| -| Loki | 3100 | (internal only) | Log aggregation backend | -| Promtail | — | (internal only) | Log collector (must access Docker socket) | - -The Flask app `/metrics` endpoint is scraped by Prometheus at `http://app:8000/metrics`. -The NGINX exporter is expected at `http://nginx-exporter:9113/metrics` (Member A must include it). - -Container logs are automatically collected by Promtail and indexed into Loki. -In Grafana, explore logs with a query like `{container="flask-app"} |= ""`. - -## Local / standalone test - -The observability services cannot run meaningfully without the application and NGINX, -but you can start them together with Member C’s app to verify Prometheus targets: - -```bash -# Start the whole team stack (Member A's merged compose) or test with Member C's fragment: -docker compose -f ../compose.member-c.yml -f compose.member-b.yml up -d \ No newline at end of file diff --git a/monitoring/grafana/dashboards/flask-app.json b/monitoring/grafana/dashboards/flask-app.json index e293dd1..aa56f13 100644 --- a/monitoring/grafana/dashboards/flask-app.json +++ b/monitoring/grafana/dashboards/flask-app.json @@ -17,7 +17,7 @@ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "targets": [ { - "expr": "rate(flask_http_request_total[1m])", + "expr": "sum by (method) (rate(flask_http_request_total[1m]))", "legendFormat": "{{method}}" } ], @@ -51,7 +51,7 @@ "gridPos": { "h": 4, "w": 6, "x": 0, "y": 8 }, "targets": [ { - "expr": "app_visit_events_total" + "expr": "sum(app_visit_events_total)" } ] }, @@ -62,7 +62,7 @@ "gridPos": { "h": 4, "w": 6, "x": 6, "y": 8 }, "targets": [ { - "expr": "app_messages_created_total" + "expr": "sum(app_messages_created_total)" } ] }, @@ -73,7 +73,7 @@ "gridPos": { "h": 4, "w": 6, "x": 12, "y": 8 }, "targets": [ { - "expr": "process_resident_memory_bytes" + "expr": "sum(process_resident_memory_bytes)" } ], "fieldConfig": { @@ -81,6 +81,17 @@ "unit": "bytes" } } + }, + { + "id": 6, + "title": "NGINX Active Connections", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 8 }, + "targets": [ + { + "expr": "nginx_connections_active" + } + ] } ] -} \ No newline at end of file +} diff --git a/monitoring/grafana/dashboards/infrastructure-overview.json b/monitoring/grafana/dashboards/infrastructure-overview.json new file mode 100644 index 0000000..69dffed --- /dev/null +++ b/monitoring/grafana/dashboards/infrastructure-overview.json @@ -0,0 +1,102 @@ +{ + "uid": "infra-overview-dashboard", + "title": "Infrastructure Overview", + "tags": ["infrastructure", "prometheus"], + "timezone": "browser", + "schemaVersion": 38, + "refresh": "10s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "Host CPU Usage", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + } + } + }, + { + "id": 2, + "title": "Host Memory Usage", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + } + } + }, + { + "id": 3, + "title": "Prometheus Targets Up", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "sum(up)" + } + ] + }, + { + "id": 4, + "title": "Observed Compose Containers", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "targets": [ + { + "expr": "count(container_last_seen{container_label_com_docker_compose_service!=\"\"})" + } + ] + }, + { + "id": 5, + "title": "Container CPU Usage (by service)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "targets": [ + { + "expr": "sum by (container_label_com_docker_compose_service) (rate(container_cpu_usage_seconds_total{container_label_com_docker_compose_service!=\"\"}[5m]))", + "legendFormat": "{{container_label_com_docker_compose_service}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + } + } + }, + { + "id": 6, + "title": "Container Memory (by service)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "targets": [ + { + "expr": "sum by (container_label_com_docker_compose_service) (container_memory_working_set_bytes{container_label_com_docker_compose_service!=\"\"})", + "legendFormat": "{{container_label_com_docker_compose_service}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + } + } + ] +} diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index b635dc0..d245161 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -16,4 +16,16 @@ scrape_configs: static_configs: - targets: ['nginx-exporter:9113'] # NGINX metrics exporter (Member A must expose) labels: - service: 'nginx' \ No newline at end of file + service: 'nginx' + + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + labels: + service: 'host' + + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + labels: + service: 'containers' diff --git a/nginx/conf.d/default.conf b/nginx/conf.d/default.conf index 399503e..21caa51 100644 --- a/nginx/conf.d/default.conf +++ b/nginx/conf.d/default.conf @@ -1,9 +1,13 @@ -# Public virtual host — port 80. Proxies to upstream flask_backend (see conf.d/01-upstream.conf). +# Public virtual host — port 80. Proxies to upstream flask_backend (see conf.d/upstream.conf). server { - listen 80; + listen 80 default_server; + listen [::]:80 default_server; server_name _; limit_req_status 429; + proxy_http_version 1.1; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 3; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -28,4 +32,9 @@ server { proxy_pass http://flask_backend; access_log off; } + + location /ready { + proxy_pass http://flask_backend; + access_log off; + } } diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh new file mode 100755 index 0000000..100a96d --- /dev/null +++ b/scripts/smoke_test.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:${NGINX_HTTP_PORT:-80}}" +PROMETHEUS_URL="${PROMETHEUS_URL:-http://localhost:9090}" +GRAFANA_URL="${GRAFANA_URL:-http://localhost:3000}" +LOKI_URL="${LOKI_URL:-http://localhost:3100}" +EXPORTER_URL="${EXPORTER_URL:-http://localhost:${NGINX_EXPORTER_PORT:-9113}}" + +retry() { + local attempts="$1" + local delay_seconds="$2" + shift 2 + + local attempt + for attempt in $(seq 1 "$attempts"); do + if "$@" >/dev/null 2>&1; then + return 0 + fi + sleep "$delay_seconds" + done + + return 1 +} + +assert_json_field() { + local json_payload="$1" + local field_path="$2" + local expected="$3" + + JSON_PAYLOAD="$json_payload" FIELD_PATH="$field_path" EXPECTED_VALUE="$expected" python3 - <<'PY' +import json +import os +import sys + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +expected = os.environ["EXPECTED_VALUE"] +value = payload +for part in os.environ["FIELD_PATH"].split("."): + value = value[part] + +if str(value) != expected: + raise SystemExit(f"Expected {os.environ['FIELD_PATH']}={expected!r}, got {value!r}") +PY +} + +echo "Waiting for public application endpoints..." +retry 30 2 curl -fsS "${BASE_URL}/health" >/dev/null +retry 30 2 curl -fsS "${BASE_URL}/ready" >/dev/null + +root_response="$(curl -fsS "${BASE_URL}/")" +health_response="$(curl -fsS "${BASE_URL}/health")" +ready_response="$(curl -fsS "${BASE_URL}/ready")" + +assert_json_field "$root_response" "status" "ok" +assert_json_field "$health_response" "status" "ok" +assert_json_field "$ready_response" "status" "ready" + +echo "Checking write/read API flow..." +review_message="smoke-test-$(date +%s)" +create_response="$(curl -fsS -X POST "${BASE_URL}/api/messages" -H 'Content-Type: application/json' -d "{\"message\":\"${review_message}\"}")" +assert_json_field "$create_response" "message.message" "$review_message" + +messages_response="$(curl -fsS "${BASE_URL}/api/messages?limit=20")" +JSON_PAYLOAD="$messages_response" EXPECTED_SUBSTRING="$review_message" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +needle = os.environ["EXPECTED_SUBSTRING"] +messages = [item["message"] for item in payload["messages"]] +if needle not in messages: + raise SystemExit(f"Message {needle!r} not found in recent messages") +PY + +echo "Checking load balancing across replicas..." +instance_sample_file="$(mktemp)" +for _ in $(seq 1 9); do + curl -fsSI "${BASE_URL}/" | tr -d '\r' | awk '/^X-App-Instance:/ {print $2}' >> "$instance_sample_file" +done + +unique_instances="$(sort -u "$instance_sample_file" | sed '/^$/d' | wc -l | tr -d ' ')" +if [[ "$unique_instances" -lt 3 ]]; then + echo "Expected traffic to hit 3 replicas, got ${unique_instances}" >&2 + sort -u "$instance_sample_file" >&2 + exit 1 +fi +rm -f "$instance_sample_file" + +echo "Checking NGINX rate limiting..." +rate_limit_summary="$(BASE_URL="$BASE_URL" python3 - <<'PY' +import collections +import concurrent.futures +import json +import os +import urllib.error +import urllib.parse +import urllib.request + +base_url = os.environ["BASE_URL"].rstrip("/") + +def issue_request() -> int: + try: + with urllib.request.urlopen(f"{base_url}/api/visits", timeout=10) as response: + return response.getcode() + except urllib.error.HTTPError as exc: + return exc.code + +counter = collections.Counter() +with concurrent.futures.ThreadPoolExecutor(max_workers=60) as executor: + for status_code in executor.map(lambda _: issue_request(), range(60)): + counter[status_code] += 1 + +print(json.dumps(counter)) +PY +)" + +JSON_PAYLOAD="$rate_limit_summary" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +if int(payload.get("429", 0)) <= 0: + raise SystemExit(f"Expected at least one 429 response, got: {payload}") +if int(payload.get("200", 0)) <= 0: + raise SystemExit(f"Expected successful API responses alongside 429s, got: {payload}") +PY + +echo "Checking Prometheus targets and app metrics..." +retry 30 2 curl -fsS "${PROMETHEUS_URL}/-/healthy" >/dev/null +PROMETHEUS_URL="$PROMETHEUS_URL" python3 - <<'PY' +import json +import os +import time +import urllib.request + +prometheus_url = os.environ["PROMETHEUS_URL"].rstrip("/") +required_jobs = {"flask-app", "nginx-exporter", "node-exporter", "cadvisor"} + +def query(expr: str): + with urllib.request.urlopen( + f"{prometheus_url}/api/v1/query?query={urllib.parse.quote(expr, safe='')}", + timeout=10, + ) as response: + return json.load(response) + +deadline = time.time() + 60 +while time.time() < deadline: + up_payload = query("up") + targets = { + (item["metric"]["job"], item["metric"]["instance"]): item["value"][1] + for item in up_payload["data"]["result"] + } + seen_jobs = {job for job, _ in targets} + node_payload = query("node_cpu_seconds_total") + cadvisor_payload = query("container_cpu_usage_seconds_total") + + if ( + required_jobs.issubset(seen_jobs) + and all(value == "1" for value in targets.values()) + and node_payload["data"]["result"] + and cadvisor_payload["data"]["result"] + ): + break + time.sleep(2) +else: + raise SystemExit( + "Prometheus did not expose all required jobs and metrics within 60 seconds" + ) +PY + +curl -fsS "${EXPORTER_URL}/metrics" >/dev/null + +echo "Checking Grafana provisioning..." +retry 30 2 curl -fsS "${GRAFANA_URL}/api/health" >/dev/null +datasources_response="$(curl -fsS -u admin:admin "${GRAFANA_URL}/api/datasources")" +dashboards_response="$(curl -fsS -u admin:admin "${GRAFANA_URL}/api/search")" + +JSON_PAYLOAD="$datasources_response" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +names = {item["name"] for item in payload} +required = {"Prometheus", "Loki"} +missing = required - names +if missing: + raise SystemExit(f"Missing Grafana datasources: {sorted(missing)}") +PY + +JSON_PAYLOAD="$dashboards_response" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +titles = {item["title"] for item in payload} +required = {"Flask App Dashboard", "Infrastructure Overview"} +missing = required - titles +if missing: + raise SystemExit(f"Missing Grafana dashboards: {sorted(missing)}") +PY + +echo "Checking Loki ingestion..." +retry 30 2 curl -fsS "${LOKI_URL}/ready" >/dev/null +loki_labels="$(curl -fsS "${LOKI_URL}/loki/api/v1/label/container/values")" + +JSON_PAYLOAD="$loki_labels" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["JSON_PAYLOAD"]) +containers = set(payload["data"]) +required_suffixes = {"-app1-1", "-app2-1", "-app3-1"} +missing = { + suffix for suffix in required_suffixes + if not any(container.endswith(suffix) for container in containers) +} +if missing: + raise SystemExit(f"Missing Loki container labels with suffixes: {sorted(missing)}") +PY + +echo "Checking PostgreSQL backup flow..." +./scripts/backup_postgres.sh >/dev/null +latest_backup="$(ls -1t backups/*.sql.gz | head -n 1)" +if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then + echo "Backup file was not created" >&2 + exit 1 +fi + +echo "Smoke test passed." From 317a461ddc46e20d23c5a6c33c56c4fcbd93618d Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 09:19:06 +0300 Subject: [PATCH 17/23] fix(README.md): update correct links --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 53e1748..10cb463 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ docker compose up -d --build - App: `http://127.0.0.1/` - Grafana: `http://127.0.0.1:3000` - Prometheus: `http://127.0.0.1:9090` -- Loki: `http://127.0.0.1:3100` +- Loki health check: `http://127.0.0.1:3100/ready` - NGINX exporter: `http://127.0.0.1:9113/metrics` Grafana login: @@ -102,6 +102,8 @@ Grafana dashboards: - `Flask App Dashboard` - `Infrastructure Overview` +Logs are viewed in Grafana Explore through the Loki datasource. + Example Loki query in Grafana Explore: ```logql From 3b44980077b614b63160c620fed47af7b3a302c1 Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 09:28:32 +0300 Subject: [PATCH 18/23] Updated README.md: added powershell alternative command --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 10cb463..f98f2b3 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,12 @@ for i in $(seq 1 6); do done ``` +PowerShell alternative: + +```powershell +1..6 | ForEach-Object { (Invoke-WebRequest http://127.0.0.1/ -UseBasicParsing).Headers["X-App-Instance"] } +``` + If you want `least connections`: ```bash From 74f3268a79fc7038f719f4865bbfdf231c8e4f24 Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 09:35:51 +0300 Subject: [PATCH 19/23] add commands for recreating containers --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index f98f2b3..93c9872 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,14 @@ If you want `least connections`: docker compose -f docker-compose.yml -f docker-compose.leastconn.yml up -d --build ``` +After that, run the load balancing check again, because Docker Compose recreates containers when switching the mode. + +To return to the default `round robin` mode: + +```bash +docker compose up -d --build +``` + ## Monitoring and logs Prometheus collects metrics from: From 83e6164eb4ee67bed6afb2089e60d9e124748955 Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 09:51:04 +0300 Subject: [PATCH 20/23] fix issues, update docker-compose.yml --- README.md | 4 +++- docker-compose.yml | 1 + scripts/smoke_test.sh | 22 +++++++++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 93c9872..5d92356 100644 --- a/README.md +++ b/README.md @@ -118,10 +118,12 @@ Grafana dashboards: Logs are viewed in Grafana Explore through the Loki datasource. +In `Explore`, switch the datasource from `Prometheus` to `Loki` first. + Example Loki query in Grafana Explore: ```logql -{container=~"containerized-observability-app.*"} +{compose_service=~".+"} ``` ## Load test diff --git a/docker-compose.yml b/docker-compose.yml index dbe3c32..5f55ec3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,6 +115,7 @@ services: privileged: true volumes: - /:/rootfs:ro + - /var/run/docker.sock:/var/run/docker.sock:ro - /var/run:/var/run:rw - /sys:/sys:ro - /var/lib/docker:/var/lib/docker:ro diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index 100a96d..c9a01df 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -154,18 +154,25 @@ while time.time() < deadline: seen_jobs = {job for job, _ in targets} node_payload = query("node_cpu_seconds_total") cadvisor_payload = query("container_cpu_usage_seconds_total") + compose_label_payload = query( + 'count(container_last_seen{container_label_com_docker_compose_service!=""})' + ) + compose_label_count = 0 + if compose_label_payload["data"]["result"]: + compose_label_count = int(float(compose_label_payload["data"]["result"][0]["value"][1])) if ( required_jobs.issubset(seen_jobs) and all(value == "1" for value in targets.values()) and node_payload["data"]["result"] and cadvisor_payload["data"]["result"] + and compose_label_count > 0 ): break time.sleep(2) else: raise SystemExit( - "Prometheus did not expose all required jobs and metrics within 60 seconds" + "Prometheus did not expose all required jobs, metrics, and Docker Compose labels within 60 seconds" ) PY @@ -202,21 +209,18 @@ PY echo "Checking Loki ingestion..." retry 30 2 curl -fsS "${LOKI_URL}/ready" >/dev/null -loki_labels="$(curl -fsS "${LOKI_URL}/loki/api/v1/label/container/values")" +loki_labels="$(curl -fsS "${LOKI_URL}/loki/api/v1/label/compose_service/values")" JSON_PAYLOAD="$loki_labels" python3 - <<'PY' import json import os payload = json.loads(os.environ["JSON_PAYLOAD"]) -containers = set(payload["data"]) -required_suffixes = {"-app1-1", "-app2-1", "-app3-1"} -missing = { - suffix for suffix in required_suffixes - if not any(container.endswith(suffix) for container in containers) -} +services = set(payload["data"]) +required_services = {"app1", "app2", "app3", "nginx"} +missing = required_services - services if missing: - raise SystemExit(f"Missing Loki container labels with suffixes: {sorted(missing)}") + raise SystemExit(f"Missing Loki compose_service labels: {sorted(missing)}") PY echo "Checking PostgreSQL backup flow..." From b3be001731bee29291c07dd36a0063d9e1fa0472 Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 10:02:21 +0300 Subject: [PATCH 21/23] fix(grafana): fix dashboard --- .../dashboards/infrastructure-overview.json | 16 ++++++++-------- scripts/smoke_test.sh | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/monitoring/grafana/dashboards/infrastructure-overview.json b/monitoring/grafana/dashboards/infrastructure-overview.json index 69dffed..59756b0 100644 --- a/monitoring/grafana/dashboards/infrastructure-overview.json +++ b/monitoring/grafana/dashboards/infrastructure-overview.json @@ -55,24 +55,24 @@ }, { "id": 4, - "title": "Observed Compose Containers", + "title": "Observed Docker Containers", "type": "stat", "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, "targets": [ { - "expr": "count(container_last_seen{container_label_com_docker_compose_service!=\"\"})" + "expr": "count(container_last_seen{id=~\"/docker/[a-f0-9]+\"})" } ] }, { "id": 5, - "title": "Container CPU Usage (by service)", + "title": "Container CPU Usage (by container)", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "targets": [ { - "expr": "sum by (container_label_com_docker_compose_service) (rate(container_cpu_usage_seconds_total{container_label_com_docker_compose_service!=\"\"}[5m]))", - "legendFormat": "{{container_label_com_docker_compose_service}}" + "expr": "sum by (container_id) (label_replace(rate(container_cpu_usage_seconds_total{id=~\"/docker/[a-f0-9]+\",cpu=\"total\"}[5m]), \"container_id\", \"$1\", \"id\", \"/docker/([a-f0-9]{12}).*\"))", + "legendFormat": "{{container_id}}" } ], "fieldConfig": { @@ -83,13 +83,13 @@ }, { "id": 6, - "title": "Container Memory (by service)", + "title": "Container Memory (by container)", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "targets": [ { - "expr": "sum by (container_label_com_docker_compose_service) (container_memory_working_set_bytes{container_label_com_docker_compose_service!=\"\"})", - "legendFormat": "{{container_label_com_docker_compose_service}}" + "expr": "sum by (container_id) (label_replace(container_memory_working_set_bytes{id=~\"/docker/[a-f0-9]+\"}, \"container_id\", \"$1\", \"id\", \"/docker/([a-f0-9]{12}).*\"))", + "legendFormat": "{{container_id}}" } ], "fieldConfig": { diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index c9a01df..9aab68e 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -154,25 +154,25 @@ while time.time() < deadline: seen_jobs = {job for job, _ in targets} node_payload = query("node_cpu_seconds_total") cadvisor_payload = query("container_cpu_usage_seconds_total") - compose_label_payload = query( - 'count(container_last_seen{container_label_com_docker_compose_service!=""})' + docker_container_payload = query( + 'count(container_last_seen{id=~"/docker/[a-f0-9]+"})' ) - compose_label_count = 0 - if compose_label_payload["data"]["result"]: - compose_label_count = int(float(compose_label_payload["data"]["result"][0]["value"][1])) + docker_container_count = 0 + if docker_container_payload["data"]["result"]: + docker_container_count = int(float(docker_container_payload["data"]["result"][0]["value"][1])) if ( required_jobs.issubset(seen_jobs) and all(value == "1" for value in targets.values()) and node_payload["data"]["result"] and cadvisor_payload["data"]["result"] - and compose_label_count > 0 + and docker_container_count > 0 ): break time.sleep(2) else: raise SystemExit( - "Prometheus did not expose all required jobs, metrics, and Docker Compose labels within 60 seconds" + "Prometheus did not expose all required jobs, metrics, and Docker container IDs within 60 seconds" ) PY From 3c1783ca114ddaad2dbf6a83a39d21788d2c37e3 Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 19:42:19 +0300 Subject: [PATCH 22/23] pipeline fix --- .github/workflows/ci-cd.yml | 2 +- .../grafana/dashboards/infrastructure-overview.json | 10 +++++----- scripts/smoke_test.sh | 9 +-------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index a251e90..97757df 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -47,7 +47,7 @@ jobs: run: chmod +x scripts/*.sh - name: Start full stack - run: docker compose up -d --build + run: docker compose up -d --build --wait --wait-timeout 120 - name: Run smoke test run: ./scripts/smoke_test.sh diff --git a/monitoring/grafana/dashboards/infrastructure-overview.json b/monitoring/grafana/dashboards/infrastructure-overview.json index 59756b0..d4157ae 100644 --- a/monitoring/grafana/dashboards/infrastructure-overview.json +++ b/monitoring/grafana/dashboards/infrastructure-overview.json @@ -60,7 +60,7 @@ "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, "targets": [ { - "expr": "count(container_last_seen{id=~\"/docker/[a-f0-9]+\"})" + "expr": "count(container_last_seen{id=~\".*[a-f0-9]{12,}.*\"})" } ] }, @@ -71,8 +71,8 @@ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "targets": [ { - "expr": "sum by (container_id) (label_replace(rate(container_cpu_usage_seconds_total{id=~\"/docker/[a-f0-9]+\",cpu=\"total\"}[5m]), \"container_id\", \"$1\", \"id\", \"/docker/([a-f0-9]{12}).*\"))", - "legendFormat": "{{container_id}}" + "expr": "sum by (id) (rate(container_cpu_usage_seconds_total{id=~\".*[a-f0-9]{12,}.*\",cpu=\"total\"}[5m]))", + "legendFormat": "{{id}}" } ], "fieldConfig": { @@ -88,8 +88,8 @@ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "targets": [ { - "expr": "sum by (container_id) (label_replace(container_memory_working_set_bytes{id=~\"/docker/[a-f0-9]+\"}, \"container_id\", \"$1\", \"id\", \"/docker/([a-f0-9]{12}).*\"))", - "legendFormat": "{{container_id}}" + "expr": "sum by (id) (container_memory_working_set_bytes{id=~\".*[a-f0-9]{12,}.*\"})", + "legendFormat": "{{id}}" } ], "fieldConfig": { diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index 9aab68e..c90b618 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -154,25 +154,18 @@ while time.time() < deadline: seen_jobs = {job for job, _ in targets} node_payload = query("node_cpu_seconds_total") cadvisor_payload = query("container_cpu_usage_seconds_total") - docker_container_payload = query( - 'count(container_last_seen{id=~"/docker/[a-f0-9]+"})' - ) - docker_container_count = 0 - if docker_container_payload["data"]["result"]: - docker_container_count = int(float(docker_container_payload["data"]["result"][0]["value"][1])) if ( required_jobs.issubset(seen_jobs) and all(value == "1" for value in targets.values()) and node_payload["data"]["result"] and cadvisor_payload["data"]["result"] - and docker_container_count > 0 ): break time.sleep(2) else: raise SystemExit( - "Prometheus did not expose all required jobs, metrics, and Docker container IDs within 60 seconds" + "Prometheus did not expose all required jobs and metrics within 60 seconds" ) PY From 97b9d92ced8f882c076f6ce7f655f9784ab29b75 Mon Sep 17 00:00:00 2001 From: albert-de-swerto Date: Mon, 18 May 2026 21:53:15 +0300 Subject: [PATCH 23/23] README.md Update --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5d92356..63307d6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# Student Project: Containerized Observability App +# Containerized Observability App ## Project Topic **Highly Available Containerized Web App with Centralized Monitoring and Logging** -This is a student project for Docker and observability practice. -We built a simple Flask web app with PostgreSQL, ran multiple replicas with Docker Compose, used NGINX as a reverse proxy, and added monitoring, logging, and CI/CD. +This project is a simple Flask web app with PostgreSQL. +It runs multiple replicas with Docker Compose, uses NGINX as a reverse proxy, and includes monitoring, centralized logging, and CI/CD. ## Stack @@ -21,7 +21,7 @@ We built a simple Flask web app with PostgreSQL, ran multiple replicas with Dock - cAdvisor - GitHub Actions -## What is included +## Features - Flask API - PostgreSQL database