Improve reusable deploy workflow diagnostics for webhook failures and non-2xx responses

.github/workflows/deploy.yml

@@ -29,15 +29,35 @@ jobs:
           echo "Deploying project [${{ inputs.project_name }}] to [${{ inputs.environment }}]"
           echo "---"
 
-          curl --no-buffer --fail-with-body --max-time 300 \
+          response_body_file="$(mktemp)"
+          trap 'rm -f "$response_body_file"' EXIT
+
+          curl_exit=0
+          http_code=$(curl --no-buffer --silent --show-error --max-time 600 \
+            --output "$response_body_file" \
+            --write-out "%{http_code}" \
             -X POST \
             -H "Content-Type: application/json" \
             -d '{
               "push_data": { "tag": "latest" },
               "repository": { "name": "${{ inputs.project_name }}" },
               "environment": "${{ inputs.environment }}"
             }' \
-            "${{ secrets.WEBHOOK_URL }}/hooks/${{ secrets.WEBHOOK_SECRET }}/docker-webhook"
+            "${{ secrets.WEBHOOK_URL }}/hooks/${{ secrets.WEBHOOK_SECRET }}/docker-webhook") || curl_exit=$?
 
+          echo "HTTP status: ${http_code:-N/A}"
+          echo "--- Response body ---"
+          cat "$response_body_file"
           echo ""
+
+          if [ "$curl_exit" -ne 0 ]; then
+            echo "--- Deployment request failed due to network/transport error ---"
+            exit "$curl_exit"
+          fi
+
+          if [ "$http_code" -lt 200 ] || [ "$http_code" -ge 300 ]; then
+            echo "--- Deployment failed ---"
+            exit 1
+          fi
+
           echo "--- Deployment completed successfully ---"

Dockerfile

@@ -2,7 +2,7 @@ FROM thecatlady/webhook:2.8.2
 WORKDIR     /etc/webhook
 COPY assets /etc/webhook/assets
 ENV SHARED_DIR_PATH=/etc/webhook/shared
-RUN apk update && apk add --no-cache docker-cli bash dos2unix
+RUN apk add --no-cache docker-cli bash dos2unix
 RUN find /etc/webhook/assets/scripts -type f -name '*.sh' -exec dos2unix {} \;
 RUN echo https://ftp.halifax.rwth-aachen.de/alpine/v3.17/main >> /etc/apk/repositories
 RUN echo https://ftp.halifax.rwth-aachen.de/alpine/v3.17/community >> /etc/apk/repositories
@@ -11,11 +11,11 @@ ENV BUILD_DEPS="gettext"  \
     RUNTIME_DEPS="libintl"
 
 RUN set -x && \
-    apk add --update $RUNTIME_DEPS && \
+    apk add --no-cache $RUNTIME_DEPS && \
     apk add --virtual build_deps $BUILD_DEPS &&  \
     cp /usr/bin/envsubst /usr/local/bin/envsubst && \
     apk del build_deps
 
-RUN apk update --allow-untrusted && apk add --allow-untrusted --no-cache docker-cli-compose
+RUN apk add --allow-untrusted --no-cache docker-cli-compose
 
-ENTRYPOINT ["/etc/webhook/assets/scripts/entrypoint.sh"]
+ENTRYPOINT ["/etc/webhook/assets/scripts/entrypoint.sh"]

README.md

@@ -16,6 +16,7 @@ An example of the usage of the image can be found in a [test](test/run-project-o
  - `WEBHOOK_SECRET` - you have to create a webhook with a secret placed into its URL `http://<server_ip>/hooks/<webhook_secret>/docker-webhook`. The secret can be then loaded as an environment variable `WEBHOOK_SECRET` in `docker-webhook` service. The secret is used for security purposes.
  - `DOCKER_USERNAME` - an env var that contains your DockerHub username.
  - `DEPLOY_DEFAULT_ENV` - (optional) the default environment to deploy to when no environment is specified in the webhook payload. Defaults to `stage` if not set.
+ - `DEPLOY_HEALTHCHECK_TIMEOUT` - (optional) a non-negative integer number of seconds to wait for Docker health checks after `docker compose up -d`. When set to a value greater than `0`, the deploy script fails if any service exits while waiting, becomes unhealthy, or does not become healthy before the timeout. When omitted or set to `0`, the deploy only relies on the Docker Compose command result, which keeps one-shot jobs supported.
  - `project-whitelist.list` - is a file with target image names being listed (without owner name). The tool will check if the list contains the image name from webhook details and will proceed         only on finding a match. Each image name should be written from the new line. The file should placed in `shared/config` folder and the `shared` folder should be mounted into `/etc/webhook/shared` of the container.
  - `envs` - a folder that loads env vars for each docker compose project. Envs should be stored in `.envs` file a placed in `shared/<image_name>-<environment>/` folder (e.g., `shared/envs/myapp-stage/` or `shared/envs/myapp-prod/`). The `shared` folder should be mounted into `/etc/webhook/shared` of the container. This configuration is optional.
  - `docker-compose.override.yml` - (optional) you can place environment-specific Docker Compose override files in `shared/envs/<image_name>-<environment>/docker-compose.override.yml`. These override files will be automatically applied when deploying to that environment, allowing you to customize ports, volumes, environment variables, or any other Docker Compose settings per environment.
@@ -26,11 +27,13 @@ The webhook service supports deploying to multiple environments (e.g., stage and
 
 - **Default behavior**: By default, webhooks deploy to the `stage` environment (or whatever `DEPLOY_DEFAULT_ENV` is set to).
 - **Production deployments**: To deploy to production, include an `environment` field with value `prod` in your webhook payload.
-- **Environment isolation**: Each environment maintains separate:
-  - Cache directories: `/etc/webhook/cache/<project>-<environment>`
-  - Docker Compose project names: `<project>-<environment>`
-  - Environment variable files: `shared/envs/<project>-<environment>/`
-  - Docker Compose override files: `shared/envs/<project>-<environment>/docker-compose.override.yml`
+ - **Environment isolation**: Each environment maintains separate:
+   - Cache directories: `/etc/webhook/cache/<project>-<environment>`
+   - Docker Compose project names: `<project>-<environment>`
+   - Environment variable files: `shared/envs/<project>-<environment>/`
+   - Docker Compose override files: `shared/envs/<project>-<environment>/docker-compose.override.yml`
+
+If your services define Docker health checks, set `DEPLOY_HEALTHCHECK_TIMEOUT` to make deployments wait for them before reporting success.
 
 **Example webhook payload for production deployment:**
 ```json

assets/scripts/restart-project.sh

@@ -1,25 +1,152 @@
 #!/bin/bash
 
-set -e
+set -euo pipefail
 
-PROJECT_NAME=$1
-ENVIRONMENT=$2
+PROJECT_NAME=${1:?Project name argument is required}
+ENVIRONMENT=${2-}
 
 # If environment is not provided or empty, use default from env var (defaults to "stage")
 if [ -z "$ENVIRONMENT" ]; then
     ENVIRONMENT="${DEPLOY_DEFAULT_ENV:-stage}"
 fi
 
+: "${SHARED_DIR_PATH:?SHARED_DIR_PATH environment variable is required}"
+: "${DOCKER_USERNAME:?DOCKER_USERNAME environment variable is required}"
+
 echo "Deploying project [$PROJECT_NAME] to environment [$ENVIRONMENT]"
 
+PROJECT_DOCKER_COMPOSE_FILE=
+PROJECT_DOCKER_COMPOSE_OVERRIDE=
+COMPOSE_PROJECT_NAME=
+declare -a COMPOSE_ARGS=()
+declare -a COMPOSE_CONTAINER_IDS=()
+HEALTHCHECK_POLL_INTERVAL=2
+
+print_service_logs() {
+    local container_id=$1
+    local service_name=$2
+
+    echo "Logs for service [$service_name]:"
+    docker logs "$container_id" || true
+}
+
+get_compose_container_ids() {
+    local container_ids_output
+
+    container_ids_output=$(docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps -q)
+
+    COMPOSE_CONTAINER_IDS=()
+    if [ -n "$container_ids_output" ]; then
+        mapfile -t COMPOSE_CONTAINER_IDS <<< "$container_ids_output"
+    fi
+
+    if [ "${#COMPOSE_CONTAINER_IDS[@]}" -eq 0 ]; then
+        echo "ERROR! Deploy for [$PROJECT_NAME] environment [$ENVIRONMENT] did not create any containers"
+        exit 1
+    fi
+}
+
+ensure_healthcheck_containers_are_active() {
+    local container_id service_name container_status
+
+    for container_id in "${COMPOSE_CONTAINER_IDS[@]}"; do
+        service_name=$(docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' "$container_id")
+        container_status=$(docker inspect --format '{{.State.Status}}' "$container_id")
+
+        case "$container_status" in
+            running)
+                ;;
+            *)
+                echo "ERROR! Service [$service_name] is in [$container_status] state after deploy"
+                docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps
+                print_service_logs "$container_id" "$service_name"
+                exit 1
+                ;;
+        esac
+    done
+}
+
+wait_for_container_healthchecks() {
+    local healthcheck_timeout=${DEPLOY_HEALTHCHECK_TIMEOUT:-0}
+    local start_time current_time container_id service_name health_status has_healthcheck pending_healthchecks
+
+    if ! [[ "$healthcheck_timeout" =~ ^(0|[1-9][0-9]*)$ ]]; then
+        echo "ERROR! DEPLOY_HEALTHCHECK_TIMEOUT must be a non-negative integer, got [$healthcheck_timeout]"
+        exit 1
+    fi
+
+    if [ "$healthcheck_timeout" -le 0 ]; then
+        return 0
+    fi
+
+    echo "Waiting up to [$healthcheck_timeout] seconds for container health checks"
+    start_time=$(date +%s)
+
+    while true; do
+        has_healthcheck=false
+        pending_healthchecks=false
+
+        get_compose_container_ids
+        ensure_healthcheck_containers_are_active
+
+        for container_id in "${COMPOSE_CONTAINER_IDS[@]}"; do
+            service_name=$(docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' "$container_id")
+            health_status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")
+
+            case "$health_status" in
+                none)
+                    ;;
+                healthy)
+                    has_healthcheck=true
+                    ;;
+                starting)
+                    has_healthcheck=true
+                    pending_healthchecks=true
+                    ;;
+                unhealthy)
+                    echo "ERROR! Service [$service_name] failed its health check after deploy"
+                    docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps
+                    print_service_logs "$container_id" "$service_name"
+                    exit 1
+                    ;;
+                *)
+                    echo "ERROR! Service [$service_name] returned unexpected health status [$health_status]"
+                    docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps
+                    print_service_logs "$container_id" "$service_name"
+                    exit 1
+                    ;;
+            esac
+        done
+
+        if [ "$has_healthcheck" = false ]; then
+            echo "No container health checks configured for [$PROJECT_NAME] environment [$ENVIRONMENT], skipping health verification"
+            return 0
+        fi
+
+        if [ "$pending_healthchecks" = false ]; then
+            echo "All container health checks passed for [$PROJECT_NAME] environment [$ENVIRONMENT]"
+            return 0
+        fi
+
+        current_time=$(date +%s)
+        if [ $((current_time - start_time)) -ge "$healthcheck_timeout" ]; then
+            echo "ERROR! Timed out waiting [$healthcheck_timeout] seconds for container health checks"
+            docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps
+            exit 1
+        fi
+
+        sleep "$HEALTHCHECK_POLL_INTERVAL"
+    done
+}
+
 # Check if project name is present in project-whitelist.list
 ./verify-project.sh "$PROJECT_NAME"
 
 # Use environment-specific paths
-PROJECT_CACHE_PATH=/etc/webhook/cache/${PROJECT_NAME}-${ENVIRONMENT}
-PROJECT_ENVS_PATH=$SHARED_DIR_PATH/envs/${PROJECT_NAME}-${ENVIRONMENT}
-REPO_NAME=$DOCKER_USERNAME/$PROJECT_NAME
-COMPOSE_PROJECT_NAME=${PROJECT_NAME}-${ENVIRONMENT}
+PROJECT_CACHE_PATH="/etc/webhook/cache/${PROJECT_NAME}-${ENVIRONMENT}"
+PROJECT_ENVS_PATH="${SHARED_DIR_PATH}/envs/${PROJECT_NAME}-${ENVIRONMENT}"
+REPO_NAME="${DOCKER_USERNAME}/${PROJECT_NAME}"
+COMPOSE_PROJECT_NAME="${PROJECT_NAME}-${ENVIRONMENT}"
 
 # Create if not exists a directory for project files
 mkdir -p "$PROJECT_CACHE_PATH"
@@ -45,22 +172,22 @@ else
     echo "Env file for project [$PROJECT_NAME] environment [$ENVIRONMENT] was not found"
 fi
 
-PROJECT_DOCKER_COMPOSE_FILE=$PROJECT_CACHE_PATH/docker-compose.yml
-PROJECT_DOCKER_COMPOSE_OVERRIDE=$PROJECT_CACHE_PATH/docker-compose.override.yml
+PROJECT_DOCKER_COMPOSE_FILE="$PROJECT_CACHE_PATH/docker-compose.yml"
+PROJECT_DOCKER_COMPOSE_OVERRIDE="$PROJECT_CACHE_PATH/docker-compose.override.yml"
 
 # RESTART PROCESS
 
 # Build docker compose command with override file if it exists
-COMPOSE_FILES="-f $PROJECT_DOCKER_COMPOSE_FILE"
+COMPOSE_ARGS=(-f "$PROJECT_DOCKER_COMPOSE_FILE")
 if [ -f "$PROJECT_DOCKER_COMPOSE_OVERRIDE" ]; then
     echo "Docker compose override file found for [$PROJECT_NAME] environment [$ENVIRONMENT]"
-    COMPOSE_FILES="$COMPOSE_FILES -f $PROJECT_DOCKER_COMPOSE_OVERRIDE"
+    COMPOSE_ARGS+=(-f "$PROJECT_DOCKER_COMPOSE_OVERRIDE")
 else
     echo "No docker compose override file for [$PROJECT_NAME] environment [$ENVIRONMENT]"
 fi
 
-docker compose $COMPOSE_FILES -p "$COMPOSE_PROJECT_NAME" pull
-docker compose $COMPOSE_FILES -p "$COMPOSE_PROJECT_NAME" down
-docker compose $COMPOSE_FILES -p "$COMPOSE_PROJECT_NAME" up -d
-
+docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" pull
+docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" down
+docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" up -d
 
+wait_for_container_healthchecks