From 61b8d4ea613ef4473161afa14b1037ef8a4baaab Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 1 Mar 2026 05:51:41 +0000 Subject: [PATCH] refactor: Update Docker image handling and registry configuration - Modified `services.py` to support prebuilt images from GHCR and custom registries, enhancing flexibility in image management. - Updated `pull-images.sh` and `push-images.sh` scripts to streamline image pulling and pushing processes, allowing for backward compatibility with DockerHub. - Refactored Docker Compose configurations to align with new image naming conventions, ensuring consistency across services. - Enhanced error messages and documentation for improved clarity on registry usage and image management. --- .../advanced-docker-compose-build.yml | 75 +++++++++---------- scripts/pull-images.sh | 35 +++++---- scripts/push-images.sh | 29 ++++--- services.py | 18 ++--- 4 files changed, 81 insertions(+), 76 deletions(-) diff --git a/.github/workflows/advanced-docker-compose-build.yml b/.github/workflows/advanced-docker-compose-build.yml index 93e72d68..21ff96e4 100644 --- a/.github/workflows/advanced-docker-compose-build.yml +++ b/.github/workflows/advanced-docker-compose-build.yml @@ -13,7 +13,6 @@ on: - "backends/advanced/**" - "extras/asr-services/**" - "extras/speaker-recognition/**" - - "extras/openmemory-mcp/**" - ".github/workflows/advanced-docker-compose-build.yml" release: types: [ published ] @@ -87,7 +86,6 @@ jobs: copy_env . copy_env ../../extras/asr-services copy_env ../../extras/speaker-recognition - copy_env ../../extras/openmemory-mcp - name: Create .env from secret (if provided) if: env.ADVANCED_ENV != '' @@ -140,22 +138,21 @@ jobs: set -euo pipefail docker compose version OWNER_LC=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]') - + # CUDA variants from pyproject.toml CUDA_VARIANTS=("cpu" "cu121" "cu126" "cu128") - + # Base services (no CUDA variants, no profiles) + # Image names must match what docker-compose.yml expects (chronicle-backend, chronicle-webui) base_service_specs=( - "chronicle-backend|advanced-chronicle-backend|docker-compose.yml|." - "workers|advanced-workers|docker-compose.yml|." - "webui|advanced-webui|docker-compose.yml|." - "openmemory-mcp|openmemory-mcp|../../extras/openmemory-mcp/docker-compose.yml|../../extras/openmemory-mcp" + "chronicle-backend|chronicle-backend|docker-compose.yml|." + "webui|chronicle-webui|docker-compose.yml|." ) - + # Build and push base services for spec in "${base_service_specs[@]}"; do IFS='|' read -r svc svc_repo compose_file project_dir <<< "$spec" - + echo "::group::Building and pushing $svc_repo" if [ "$compose_file" = "docker-compose.yml" ] && [ "$project_dir" = "." ]; then docker compose build --pull "$svc" @@ -173,7 +170,7 @@ jobs: echo "::endgroup::" continue fi - + # Tag and push with version target_image="$REGISTRY/$OWNER_LC/$svc_repo:$VERSION" latest_image="$REGISTRY/$OWNER_LC/$svc_repo:latest" @@ -181,21 +178,21 @@ jobs: docker tag "$img_id" "$target_image" echo "Tagging $img_id as $latest_image" docker tag "$img_id" "$latest_image" - + echo "Pushing $target_image" docker push "$target_image" echo "Pushing $latest_image" docker push "$latest_image" - + # Clean up local tags docker image rm -f "$target_image" || true docker image rm -f "$latest_image" || true echo "::endgroup::" - + # Aggressive cleanup to save space docker system prune -af || true done - + # Build and push parakeet-asr with CUDA variants (cu121, cu126, cu128) echo "::group::Building and pushing parakeet-asr CUDA variants" cd ../../extras/asr-services @@ -203,32 +200,32 @@ jobs: echo "Building parakeet-asr-${cuda_variant}" export PYTORCH_CUDA_VERSION="${cuda_variant}" docker compose build parakeet-asr - + img_id=$(docker compose images -q parakeet-asr | head -n1) if [ -n "${img_id:-}" ]; then - target_image="$REGISTRY/$OWNER_LC/parakeet-asr-${cuda_variant}:$VERSION" - latest_image="$REGISTRY/$OWNER_LC/parakeet-asr-${cuda_variant}:latest" + target_image="$REGISTRY/$OWNER_LC/chronicle-asr-nemo-${cuda_variant}:$VERSION" + latest_image="$REGISTRY/$OWNER_LC/chronicle-asr-nemo-${cuda_variant}:latest" echo "Tagging $img_id as $target_image" docker tag "$img_id" "$target_image" echo "Tagging $img_id as $latest_image" docker tag "$img_id" "$latest_image" - + echo "Pushing $target_image" docker push "$target_image" echo "Pushing $latest_image" docker push "$latest_image" - + # Clean up local tags docker image rm -f "$target_image" || true docker image rm -f "$latest_image" || true fi - + # Aggressive cleanup to save space docker system prune -af || true done cd - > /dev/null echo "::endgroup::" - + # Build and push speaker-recognition with all CUDA variants (including CPU) # Note: speaker-service has profiles, but we can build it directly by setting PYTORCH_CUDA_VERSION echo "::group::Building and pushing speaker-recognition variants" @@ -238,32 +235,32 @@ jobs: export PYTORCH_CUDA_VERSION="${cuda_variant}" # Build speaker-service directly (profiles only affect 'up', not 'build') docker compose build speaker-service - + img_id=$(docker compose images -q speaker-service | head -n1) if [ -n "${img_id:-}" ]; then - target_image="$REGISTRY/$OWNER_LC/speaker-recognition-${cuda_variant}:$VERSION" - latest_image="$REGISTRY/$OWNER_LC/speaker-recognition-${cuda_variant}:latest" + target_image="$REGISTRY/$OWNER_LC/chronicle-speaker-${cuda_variant}:$VERSION" + latest_image="$REGISTRY/$OWNER_LC/chronicle-speaker-${cuda_variant}:latest" echo "Tagging $img_id as $target_image" docker tag "$img_id" "$target_image" echo "Tagging $img_id as $latest_image" docker tag "$img_id" "$latest_image" - + echo "Pushing $target_image" docker push "$target_image" echo "Pushing $latest_image" docker push "$latest_image" - + # Clean up local tags docker image rm -f "$target_image" || true docker image rm -f "$latest_image" || true fi - + # Aggressive cleanup to save space docker system prune -af || true done cd - > /dev/null echo "::endgroup::" - + # Summary echo "::group::Build Summary" echo "Built and pushed images with version tag: ${VERSION}" @@ -291,25 +288,23 @@ jobs: ### Core Services \`\`\`bash - docker pull ghcr.io/${OWNER_LC}/advanced-chronicle-backend:${VERSION} - docker pull ghcr.io/${OWNER_LC}/advanced-workers:${VERSION} - docker pull ghcr.io/${OWNER_LC}/advanced-webui:${VERSION} - docker pull ghcr.io/${OWNER_LC}/openmemory-mcp:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-backend:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-webui:${VERSION} \`\`\` ### Parakeet ASR (pick your CUDA version) \`\`\`bash - docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu121:${VERSION} - docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu126:${VERSION} - docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu128:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu121:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu126:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu128:${VERSION} \`\`\` ### Speaker Recognition (pick your variant) \`\`\`bash - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cpu:${VERSION} - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu121:${VERSION} - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu126:${VERSION} - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu128:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cpu:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu121:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu126:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu128:${VERSION} \`\`\` EOF ) diff --git a/scripts/pull-images.sh b/scripts/pull-images.sh index caf86bbe..5a4cdb0d 100755 --- a/scripts/pull-images.sh +++ b/scripts/pull-images.sh @@ -1,11 +1,15 @@ #!/usr/bin/env bash -# pull-images.sh — Pull Chronicle images from DockerHub and retag them locally +# pull-images.sh — Pull Chronicle images and retag them locally # -# Usage: -# DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0 +# Usage (GHCR — default, no account needed): +# ./scripts/pull-images.sh v1.0.0 +# +# Usage (custom registry): +# CHRONICLE_REGISTRY=ghcr.io/myuser/ ./scripts/pull-images.sh v1.0.0 +# DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0 # backward compat # # After pulling, start with the prebuilt images: -# DOCKERHUB_USERNAME=myuser ./start.sh --use-prebuilt v1.0.0 +# ./start.sh --use-prebuilt v1.0.0 set -euo pipefail @@ -23,20 +27,21 @@ warn() { echo -e "${YELLOW}⚠️ $*${RESET}"; } error() { echo -e "${RED}❌ $*${RESET}" >&2; } # ── Validate inputs ──────────────────────────────────────────────────────────── -if [[ -z "${DOCKERHUB_USERNAME:-}" ]]; then - error "DOCKERHUB_USERNAME env var is required." - echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0" >&2 - exit 1 -fi - TAG="${1:-}" if [[ -z "$TAG" ]]; then error "TAG argument is required." - echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0" >&2 + echo " Example: ./scripts/pull-images.sh v1.0.0" >&2 exit 1 fi -REGISTRY="${DOCKERHUB_USERNAME}/" +# Registry precedence: CHRONICLE_REGISTRY > DOCKERHUB_USERNAME > default GHCR +if [[ -n "${CHRONICLE_REGISTRY:-}" ]]; then + REGISTRY="${CHRONICLE_REGISTRY}" +elif [[ -n "${DOCKERHUB_USERNAME:-}" ]]; then + REGISTRY="${DOCKERHUB_USERNAME}/" +else + REGISTRY="ghcr.io/simpleopensoftware/" +fi # ── Image inventory ──────────────────────────────────────────────────────────── # Format: "local-image-name:registry-image-name" @@ -85,7 +90,7 @@ for entry in "${IMAGES[@]}"; do success " Ready as ${LOCAL_TAG}" PULLED+=("${entry##*:}") else - warn " Not found on DockerHub — skipping (this service may not have been pushed)" + warn " Not found in registry — skipping (this service may not have been pushed)" FAILED+=("${entry##*:}") fi echo "" @@ -115,8 +120,8 @@ if [[ ${#PULLED[@]} -gt 0 ]]; then success "Pulled ${#PULLED[@]} image(s) tagged as ${TAG}" echo "" echo "Start services with prebuilt images:" - echo -e " ${BOLD}DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./start.sh --use-prebuilt ${TAG}${RESET}" + echo -e " ${BOLD}./start.sh --use-prebuilt ${TAG}${RESET}" fi if [[ ${#FAILED[@]} -gt 0 ]]; then - warn "${#FAILED[@]} image(s) not found on DockerHub (these services will fall back to local builds)" + warn "${#FAILED[@]} image(s) not found in registry (these services will fall back to local builds)" fi diff --git a/scripts/push-images.sh b/scripts/push-images.sh index f91c2d82..1737ba08 100755 --- a/scripts/push-images.sh +++ b/scripts/push-images.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash -# push-images.sh — Tag and push locally-built Chronicle images to DockerHub +# push-images.sh — Tag and push locally-built Chronicle images to a registry # # Usage: -# DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 "stable before refactor" +# CHRONICLE_PUSH_REGISTRY=ghcr.io/myuser/ ./scripts/push-images.sh v1.0.0 "stable before refactor" +# DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 "description" # backward compat # # Requirements: # - Images must already be built locally (run ./start.sh --build first) -# - DOCKERHUB_USERNAME env var must be set -# - Must be logged in to DockerHub (docker login) +# - CHRONICLE_PUSH_REGISTRY or DOCKERHUB_USERNAME env var must be set +# - Must be logged in to the target registry (docker login) set -euo pipefail @@ -25,23 +26,27 @@ warn() { echo -e "${YELLOW}⚠️ $*${RESET}"; } error() { echo -e "${RED}❌ $*${RESET}" >&2; } # ── Validate inputs ──────────────────────────────────────────────────────────── -if [[ -z "${DOCKERHUB_USERNAME:-}" ]]; then - error "DOCKERHUB_USERNAME env var is required." - echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 'description'" >&2 +# Registry precedence: CHRONICLE_PUSH_REGISTRY > DOCKERHUB_USERNAME +if [[ -n "${CHRONICLE_PUSH_REGISTRY:-}" ]]; then + REGISTRY="${CHRONICLE_PUSH_REGISTRY}" +elif [[ -n "${DOCKERHUB_USERNAME:-}" ]]; then + REGISTRY="${DOCKERHUB_USERNAME}/" +else + error "CHRONICLE_PUSH_REGISTRY or DOCKERHUB_USERNAME env var is required." + echo " GHCR: CHRONICLE_PUSH_REGISTRY=ghcr.io/yourusername/ ./scripts/push-images.sh v1.0.0" >&2 + echo " DockerHub: DOCKERHUB_USERNAME=yourusername ./scripts/push-images.sh v1.0.0" >&2 exit 1 fi TAG="${1:-}" if [[ -z "$TAG" ]]; then error "TAG argument is required." - echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 'description'" >&2 + echo " Example: CHRONICLE_PUSH_REGISTRY=ghcr.io/myuser/ ./scripts/push-images.sh v1.0.0 'description'" >&2 exit 1 fi DESCRIPTION="${2:-}" -REGISTRY="${DOCKERHUB_USERNAME}/" - # ── Image inventory ──────────────────────────────────────────────────────────── # Format: "local-image-name:registry-image-name" # local-image-name = what docker-compose builds to locally (with empty CHRONICLE_REGISTRY) @@ -181,8 +186,8 @@ if [[ ${#PUSHED[@]} -gt 0 ]]; then success "Pushed ${#PUSHED[@]} image(s) as ${TAG}" echo "" echo "To restore this snapshot:" - echo " DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./scripts/pull-images.sh ${TAG}" - echo " DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./start.sh --use-prebuilt ${TAG}" + echo " CHRONICLE_REGISTRY=${REGISTRY} ./scripts/pull-images.sh ${TAG}" + echo " CHRONICLE_REGISTRY=${REGISTRY} ./start.sh --use-prebuilt ${TAG}" fi if [[ ${#SKIPPED[@]} -gt 0 ]]; then warn "${#SKIPPED[@]} image(s) were skipped (not found locally)" diff --git a/services.py b/services.py index aac9adc8..d9147b0c 100755 --- a/services.py +++ b/services.py @@ -638,7 +638,7 @@ def main(): start_parser.add_argument( "--use-prebuilt", metavar="TAG", - help="Use prebuilt images from DockerHub instead of building (requires prior pull-images.sh run)", + help="Use prebuilt images from GHCR (or custom registry via CHRONICLE_REGISTRY env var)", ) # Stop command @@ -703,16 +703,16 @@ def main(): return if args.use_prebuilt: - dockerhub_user = os.environ.get("DOCKERHUB_USERNAME", "") - if not dockerhub_user: - console.print( - "[red]❌ DOCKERHUB_USERNAME env var required with --use-prebuilt[/red]" - ) - return - os.environ["CHRONICLE_REGISTRY"] = f"{dockerhub_user}/" + if os.environ.get("CHRONICLE_REGISTRY"): + registry = os.environ["CHRONICLE_REGISTRY"] + elif os.environ.get("DOCKERHUB_USERNAME"): + registry = f"{os.environ['DOCKERHUB_USERNAME']}/" + else: + registry = "ghcr.io/simpleopensoftware/" + os.environ["CHRONICLE_REGISTRY"] = registry os.environ["CHRONICLE_TAG"] = args.use_prebuilt console.print( - f"[cyan]ℹ️ Using prebuilt images: {dockerhub_user}/*:{args.use_prebuilt}[/cyan]" + f"[cyan]ℹ️ Using prebuilt images: {registry}*:{args.use_prebuilt}[/cyan]" ) build_flag = False else: