diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c5f4518..7f8918f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,7 +29,13 @@ jobs: echo "Deploying project [${{ inputs.project_name }}] to [${{ inputs.environment }}]" echo "---" - curl --no-buffer --fail-with-body --max-time 300 \ + response_body_file="$(mktemp)" + trap 'rm -f "$response_body_file"' EXIT + + curl_exit=0 + http_code=$(curl --no-buffer --silent --show-error --max-time 600 \ + --output "$response_body_file" \ + --write-out "%{http_code}" \ -X POST \ -H "Content-Type: application/json" \ -d '{ @@ -37,7 +43,21 @@ jobs: "repository": { "name": "${{ inputs.project_name }}" }, "environment": "${{ inputs.environment }}" }' \ - "${{ secrets.WEBHOOK_URL }}/hooks/${{ secrets.WEBHOOK_SECRET }}/docker-webhook" + "${{ secrets.WEBHOOK_URL }}/hooks/${{ secrets.WEBHOOK_SECRET }}/docker-webhook") || curl_exit=$? + echo "HTTP status: ${http_code:-N/A}" + echo "--- Response body ---" + cat "$response_body_file" echo "" + + if [ "$curl_exit" -ne 0 ]; then + echo "--- Deployment request failed due to network/transport error ---" + exit "$curl_exit" + fi + + if [ "$http_code" -lt 200 ] || [ "$http_code" -ge 300 ]; then + echo "--- Deployment failed ---" + exit 1 + fi + echo "--- Deployment completed successfully ---" diff --git a/Dockerfile b/Dockerfile index 896c9e3..385a935 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM thecatlady/webhook:2.8.2 WORKDIR /etc/webhook COPY assets /etc/webhook/assets ENV SHARED_DIR_PATH=/etc/webhook/shared -RUN apk update && apk add --no-cache docker-cli bash dos2unix +RUN apk add --no-cache docker-cli bash dos2unix RUN find /etc/webhook/assets/scripts -type f -name '*.sh' -exec dos2unix {} \; RUN echo https://ftp.halifax.rwth-aachen.de/alpine/v3.17/main >> /etc/apk/repositories RUN echo https://ftp.halifax.rwth-aachen.de/alpine/v3.17/community >> /etc/apk/repositories @@ -11,11 +11,11 @@ ENV BUILD_DEPS="gettext" \ RUNTIME_DEPS="libintl" RUN set -x && \ - apk add --update $RUNTIME_DEPS && \ + apk add --no-cache $RUNTIME_DEPS && \ apk add --virtual build_deps $BUILD_DEPS && \ cp /usr/bin/envsubst /usr/local/bin/envsubst && \ apk del build_deps -RUN apk update --allow-untrusted && apk add --allow-untrusted --no-cache docker-cli-compose +RUN apk add --allow-untrusted --no-cache docker-cli-compose -ENTRYPOINT ["/etc/webhook/assets/scripts/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/etc/webhook/assets/scripts/entrypoint.sh"] diff --git a/README.md b/README.md index 609343c..f5ba93c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ An example of the usage of the image can be found in a [test](test/run-project-o - `WEBHOOK_SECRET` - you have to create a webhook with a secret placed into its URL `http:///hooks//docker-webhook`. The secret can be then loaded as an environment variable `WEBHOOK_SECRET` in `docker-webhook` service. The secret is used for security purposes. - `DOCKER_USERNAME` - an env var that contains your DockerHub username. - `DEPLOY_DEFAULT_ENV` - (optional) the default environment to deploy to when no environment is specified in the webhook payload. Defaults to `stage` if not set. + - `DEPLOY_HEALTHCHECK_TIMEOUT` - (optional) a non-negative integer number of seconds to wait for Docker health checks after `docker compose up -d`. When set to a value greater than `0`, the deploy script fails if any service exits while waiting, becomes unhealthy, or does not become healthy before the timeout. When omitted or set to `0`, the deploy only relies on the Docker Compose command result, which keeps one-shot jobs supported. - `project-whitelist.list` - is a file with target image names being listed (without owner name). The tool will check if the list contains the image name from webhook details and will proceed only on finding a match. Each image name should be written from the new line. The file should placed in `shared/config` folder and the `shared` folder should be mounted into `/etc/webhook/shared` of the container. - `envs` - a folder that loads env vars for each docker compose project. Envs should be stored in `.envs` file a placed in `shared/-/` folder (e.g., `shared/envs/myapp-stage/` or `shared/envs/myapp-prod/`). The `shared` folder should be mounted into `/etc/webhook/shared` of the container. This configuration is optional. - `docker-compose.override.yml` - (optional) you can place environment-specific Docker Compose override files in `shared/envs/-/docker-compose.override.yml`. These override files will be automatically applied when deploying to that environment, allowing you to customize ports, volumes, environment variables, or any other Docker Compose settings per environment. @@ -26,11 +27,13 @@ The webhook service supports deploying to multiple environments (e.g., stage and - **Default behavior**: By default, webhooks deploy to the `stage` environment (or whatever `DEPLOY_DEFAULT_ENV` is set to). - **Production deployments**: To deploy to production, include an `environment` field with value `prod` in your webhook payload. -- **Environment isolation**: Each environment maintains separate: - - Cache directories: `/etc/webhook/cache/-` - - Docker Compose project names: `-` - - Environment variable files: `shared/envs/-/` - - Docker Compose override files: `shared/envs/-/docker-compose.override.yml` + - **Environment isolation**: Each environment maintains separate: + - Cache directories: `/etc/webhook/cache/-` + - Docker Compose project names: `-` + - Environment variable files: `shared/envs/-/` + - Docker Compose override files: `shared/envs/-/docker-compose.override.yml` + +If your services define Docker health checks, set `DEPLOY_HEALTHCHECK_TIMEOUT` to make deployments wait for them before reporting success. **Example webhook payload for production deployment:** ```json diff --git a/assets/scripts/restart-project.sh b/assets/scripts/restart-project.sh index d53d678..cf750f2 100755 --- a/assets/scripts/restart-project.sh +++ b/assets/scripts/restart-project.sh @@ -1,25 +1,152 @@ #!/bin/bash -set -e +set -euo pipefail -PROJECT_NAME=$1 -ENVIRONMENT=$2 +PROJECT_NAME=${1:?Project name argument is required} +ENVIRONMENT=${2-} # If environment is not provided or empty, use default from env var (defaults to "stage") if [ -z "$ENVIRONMENT" ]; then ENVIRONMENT="${DEPLOY_DEFAULT_ENV:-stage}" fi +: "${SHARED_DIR_PATH:?SHARED_DIR_PATH environment variable is required}" +: "${DOCKER_USERNAME:?DOCKER_USERNAME environment variable is required}" + echo "Deploying project [$PROJECT_NAME] to environment [$ENVIRONMENT]" +PROJECT_DOCKER_COMPOSE_FILE= +PROJECT_DOCKER_COMPOSE_OVERRIDE= +COMPOSE_PROJECT_NAME= +declare -a COMPOSE_ARGS=() +declare -a COMPOSE_CONTAINER_IDS=() +HEALTHCHECK_POLL_INTERVAL=2 + +print_service_logs() { + local container_id=$1 + local service_name=$2 + + echo "Logs for service [$service_name]:" + docker logs "$container_id" || true +} + +get_compose_container_ids() { + local container_ids_output + + container_ids_output=$(docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps -q) + + COMPOSE_CONTAINER_IDS=() + if [ -n "$container_ids_output" ]; then + mapfile -t COMPOSE_CONTAINER_IDS <<< "$container_ids_output" + fi + + if [ "${#COMPOSE_CONTAINER_IDS[@]}" -eq 0 ]; then + echo "ERROR! Deploy for [$PROJECT_NAME] environment [$ENVIRONMENT] did not create any containers" + exit 1 + fi +} + +ensure_healthcheck_containers_are_active() { + local container_id service_name container_status + + for container_id in "${COMPOSE_CONTAINER_IDS[@]}"; do + service_name=$(docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' "$container_id") + container_status=$(docker inspect --format '{{.State.Status}}' "$container_id") + + case "$container_status" in + running) + ;; + *) + echo "ERROR! Service [$service_name] is in [$container_status] state after deploy" + docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps + print_service_logs "$container_id" "$service_name" + exit 1 + ;; + esac + done +} + +wait_for_container_healthchecks() { + local healthcheck_timeout=${DEPLOY_HEALTHCHECK_TIMEOUT:-0} + local start_time current_time container_id service_name health_status has_healthcheck pending_healthchecks + + if ! [[ "$healthcheck_timeout" =~ ^(0|[1-9][0-9]*)$ ]]; then + echo "ERROR! DEPLOY_HEALTHCHECK_TIMEOUT must be a non-negative integer, got [$healthcheck_timeout]" + exit 1 + fi + + if [ "$healthcheck_timeout" -le 0 ]; then + return 0 + fi + + echo "Waiting up to [$healthcheck_timeout] seconds for container health checks" + start_time=$(date +%s) + + while true; do + has_healthcheck=false + pending_healthchecks=false + + get_compose_container_ids + ensure_healthcheck_containers_are_active + + for container_id in "${COMPOSE_CONTAINER_IDS[@]}"; do + service_name=$(docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' "$container_id") + health_status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id") + + case "$health_status" in + none) + ;; + healthy) + has_healthcheck=true + ;; + starting) + has_healthcheck=true + pending_healthchecks=true + ;; + unhealthy) + echo "ERROR! Service [$service_name] failed its health check after deploy" + docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps + print_service_logs "$container_id" "$service_name" + exit 1 + ;; + *) + echo "ERROR! Service [$service_name] returned unexpected health status [$health_status]" + docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps + print_service_logs "$container_id" "$service_name" + exit 1 + ;; + esac + done + + if [ "$has_healthcheck" = false ]; then + echo "No container health checks configured for [$PROJECT_NAME] environment [$ENVIRONMENT], skipping health verification" + return 0 + fi + + if [ "$pending_healthchecks" = false ]; then + echo "All container health checks passed for [$PROJECT_NAME] environment [$ENVIRONMENT]" + return 0 + fi + + current_time=$(date +%s) + if [ $((current_time - start_time)) -ge "$healthcheck_timeout" ]; then + echo "ERROR! Timed out waiting [$healthcheck_timeout] seconds for container health checks" + docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" ps + exit 1 + fi + + sleep "$HEALTHCHECK_POLL_INTERVAL" + done +} + # Check if project name is present in project-whitelist.list ./verify-project.sh "$PROJECT_NAME" # Use environment-specific paths -PROJECT_CACHE_PATH=/etc/webhook/cache/${PROJECT_NAME}-${ENVIRONMENT} -PROJECT_ENVS_PATH=$SHARED_DIR_PATH/envs/${PROJECT_NAME}-${ENVIRONMENT} -REPO_NAME=$DOCKER_USERNAME/$PROJECT_NAME -COMPOSE_PROJECT_NAME=${PROJECT_NAME}-${ENVIRONMENT} +PROJECT_CACHE_PATH="/etc/webhook/cache/${PROJECT_NAME}-${ENVIRONMENT}" +PROJECT_ENVS_PATH="${SHARED_DIR_PATH}/envs/${PROJECT_NAME}-${ENVIRONMENT}" +REPO_NAME="${DOCKER_USERNAME}/${PROJECT_NAME}" +COMPOSE_PROJECT_NAME="${PROJECT_NAME}-${ENVIRONMENT}" # Create if not exists a directory for project files mkdir -p "$PROJECT_CACHE_PATH" @@ -45,22 +172,22 @@ else echo "Env file for project [$PROJECT_NAME] environment [$ENVIRONMENT] was not found" fi -PROJECT_DOCKER_COMPOSE_FILE=$PROJECT_CACHE_PATH/docker-compose.yml -PROJECT_DOCKER_COMPOSE_OVERRIDE=$PROJECT_CACHE_PATH/docker-compose.override.yml +PROJECT_DOCKER_COMPOSE_FILE="$PROJECT_CACHE_PATH/docker-compose.yml" +PROJECT_DOCKER_COMPOSE_OVERRIDE="$PROJECT_CACHE_PATH/docker-compose.override.yml" # RESTART PROCESS # Build docker compose command with override file if it exists -COMPOSE_FILES="-f $PROJECT_DOCKER_COMPOSE_FILE" +COMPOSE_ARGS=(-f "$PROJECT_DOCKER_COMPOSE_FILE") if [ -f "$PROJECT_DOCKER_COMPOSE_OVERRIDE" ]; then echo "Docker compose override file found for [$PROJECT_NAME] environment [$ENVIRONMENT]" - COMPOSE_FILES="$COMPOSE_FILES -f $PROJECT_DOCKER_COMPOSE_OVERRIDE" + COMPOSE_ARGS+=(-f "$PROJECT_DOCKER_COMPOSE_OVERRIDE") else echo "No docker compose override file for [$PROJECT_NAME] environment [$ENVIRONMENT]" fi -docker compose $COMPOSE_FILES -p "$COMPOSE_PROJECT_NAME" pull -docker compose $COMPOSE_FILES -p "$COMPOSE_PROJECT_NAME" down -docker compose $COMPOSE_FILES -p "$COMPOSE_PROJECT_NAME" up -d - +docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" pull +docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" down +docker compose "${COMPOSE_ARGS[@]}" -p "$COMPOSE_PROJECT_NAME" up -d +wait_for_container_healthchecks diff --git a/test/test-restart-project.sh b/test/test-restart-project.sh new file mode 100755 index 0000000..008c268 --- /dev/null +++ b/test/test-restart-project.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +set -euo pipefail + +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +TMP_ROOT=$(mktemp -d) +trap 'if [[ -n "${TMP_ROOT:-}" && -d "$TMP_ROOT" ]]; then rm -rf "$TMP_ROOT"; fi' EXIT + +SCRIPT_DIR="$TMP_ROOT/scripts" +MOCK_BIN="$TMP_ROOT/bin" +SHARED_DIR="$TMP_ROOT/shared" +WEBHOOK_ROOT="$TMP_ROOT/webhook" +ESCAPED_WEBHOOK_ROOT=$(printf '%s\n' "$WEBHOOK_ROOT" | sed 's#[/&]#\\&#g') + +mkdir -p "$SCRIPT_DIR" "$MOCK_BIN" "$SHARED_DIR/envs" "$SHARED_DIR/configs" "$WEBHOOK_ROOT/cache" + +sed "s#/etc/webhook#$ESCAPED_WEBHOOK_ROOT#g" "$REPO_ROOT/assets/scripts/restart-project.sh" > "$SCRIPT_DIR/restart-project.sh" + +cat > "$SCRIPT_DIR/verify-project.sh" <<'EOF' +#!/bin/bash +exit 0 +EOF + +cat > "$SCRIPT_DIR/extract-file.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +mkdir -p "$3/docker" +cat > "$3/docker/docker-compose.yml" <<'YAML' +services: + app: + image: example/app:latest +YAML +EOF + +cat > "$MOCK_BIN/docker" <<'EOF' +#!/bin/bash +set -euo pipefail + +cmd=$1 +shift + +case "$cmd" in + pull|create|cp|rm) + exit 0 + ;; + logs) + echo "mock logs for $1" + exit 0 + ;; + inspect) + format=$2 + case "$format" in + *com.docker.compose.service*) + echo "app" + ;; + *'.State.Status'*) + echo "${MOCK_CONTAINER_STATUS:-running}" + ;; + *'.State.Health'*) + echo "${MOCK_HEALTH_STATUS:-none}" + ;; + *) + exit 1 + ;; + esac + ;; + compose) + subcmd='' + while [ "$#" -gt 0 ]; do + case "$1" in + -f|-p) + shift 2 + ;; + *) + subcmd=$1 + shift + break + ;; + esac + done + + case "$subcmd" in + pull|down|up) + exit 0 + ;; + ps) + if [ "${1:-}" = "-q" ]; then + if [ "${MOCK_NO_CONTAINERS:-0}" = "1" ]; then + exit 0 + fi + echo "container-1" + else + echo "NAME STATUS" + fi + exit 0 + ;; + *) + echo "unsupported compose subcommand: $subcmd" >&2 + exit 1 + ;; + esac + ;; + *) + echo "unsupported docker command: $cmd" >&2 + exit 1 + ;; +esac +EOF + +cat > "$MOCK_BIN/chown" <<'EOF' +#!/bin/bash +exit 0 +EOF + +chmod +x \ + "$SCRIPT_DIR/restart-project.sh" \ + "$SCRIPT_DIR/verify-project.sh" \ + "$SCRIPT_DIR/extract-file.sh" \ + "$MOCK_BIN/docker" \ + "$MOCK_BIN/chown" + +printf 'project\n' > "$SHARED_DIR/configs/project-whitelist.list" + +run_restart_project() { + local expected_exit=$1 + shift + + set +e + ( + cd "$SCRIPT_DIR" + PATH="$MOCK_BIN:$PATH" \ + SHARED_DIR_PATH="$SHARED_DIR" \ + DOCKER_USERNAME=tester \ + "$@" \ + ./restart-project.sh project stage + ) >/dev/null 2>&1 + local status=$? + set -e + + if [ "$status" -ne "$expected_exit" ]; then + echo "Expected exit code $expected_exit, got $status" >&2 + exit 1 + fi +} + +bash -n "$SCRIPT_DIR/restart-project.sh" +run_restart_project 0 env DEPLOY_HEALTHCHECK_TIMEOUT=5 MOCK_CONTAINER_STATUS=running MOCK_HEALTH_STATUS=healthy +run_restart_project 0 env MOCK_CONTAINER_STATUS=running MOCK_HEALTH_STATUS=none +run_restart_project 0 env MOCK_CONTAINER_STATUS=exited MOCK_HEALTH_STATUS=none +run_restart_project 1 env DEPLOY_HEALTHCHECK_TIMEOUT=5 MOCK_CONTAINER_STATUS=exited MOCK_HEALTH_STATUS=none +run_restart_project 1 env DEPLOY_HEALTHCHECK_TIMEOUT=5 MOCK_CONTAINER_STATUS=running MOCK_HEALTH_STATUS=unhealthy +run_restart_project 1 env DEPLOY_HEALTHCHECK_TIMEOUT=1 MOCK_CONTAINER_STATUS=running MOCK_HEALTH_STATUS=starting +run_restart_project 1 env DEPLOY_HEALTHCHECK_TIMEOUT=5 MOCK_NO_CONTAINERS=1 + +echo "restart-project.sh tests passed" diff --git a/test/test.sh b/test/test.sh index e085cce..f089fac 100755 --- a/test/test.sh +++ b/test/test.sh @@ -6,6 +6,8 @@ set -e image_name=$(basename "$(dirname "$(pwd)")") +./test-restart-project.sh + docker build -t "generaltao725/$image_name:test" $(dirname $(pwd)) for dir in $(find -mindepth 1 -maxdepth 1 -type d | sort); do