Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 35 additions & 40 deletions .github/workflows/advanced-docker-compose-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ on:
- "backends/advanced/**"
- "extras/asr-services/**"
- "extras/speaker-recognition/**"
- "extras/openmemory-mcp/**"
- ".github/workflows/advanced-docker-compose-build.yml"
release:
types: [ published ]
Expand Down Expand Up @@ -87,7 +86,6 @@ jobs:
copy_env .
copy_env ../../extras/asr-services
copy_env ../../extras/speaker-recognition
copy_env ../../extras/openmemory-mcp

- name: Create .env from secret (if provided)
if: env.ADVANCED_ENV != ''
Expand Down Expand Up @@ -140,22 +138,21 @@ jobs:
set -euo pipefail
docker compose version
OWNER_LC=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]')

# CUDA variants from pyproject.toml
CUDA_VARIANTS=("cpu" "cu121" "cu126" "cu128")

# Base services (no CUDA variants, no profiles)
# Image names must match what docker-compose.yml expects (chronicle-backend, chronicle-webui)
base_service_specs=(
"chronicle-backend|advanced-chronicle-backend|docker-compose.yml|."
"workers|advanced-workers|docker-compose.yml|."
"webui|advanced-webui|docker-compose.yml|."
"openmemory-mcp|openmemory-mcp|../../extras/openmemory-mcp/docker-compose.yml|../../extras/openmemory-mcp"
"chronicle-backend|chronicle-backend|docker-compose.yml|."
"webui|chronicle-webui|docker-compose.yml|."
)

# Build and push base services
for spec in "${base_service_specs[@]}"; do
IFS='|' read -r svc svc_repo compose_file project_dir <<< "$spec"

echo "::group::Building and pushing $svc_repo"
if [ "$compose_file" = "docker-compose.yml" ] && [ "$project_dir" = "." ]; then
docker compose build --pull "$svc"
Expand All @@ -173,62 +170,62 @@ jobs:
echo "::endgroup::"
continue
fi

# Tag and push with version
target_image="$REGISTRY/$OWNER_LC/$svc_repo:$VERSION"
latest_image="$REGISTRY/$OWNER_LC/$svc_repo:latest"
echo "Tagging $img_id as $target_image"
docker tag "$img_id" "$target_image"
echo "Tagging $img_id as $latest_image"
docker tag "$img_id" "$latest_image"

echo "Pushing $target_image"
docker push "$target_image"
echo "Pushing $latest_image"
docker push "$latest_image"

# Clean up local tags
docker image rm -f "$target_image" || true
docker image rm -f "$latest_image" || true
echo "::endgroup::"

# Aggressive cleanup to save space
docker system prune -af || true
done

# Build and push parakeet-asr with CUDA variants (cu121, cu126, cu128)
echo "::group::Building and pushing parakeet-asr CUDA variants"
cd ../../extras/asr-services
for cuda_variant in cu121 cu126 cu128; do
echo "Building parakeet-asr-${cuda_variant}"
export PYTORCH_CUDA_VERSION="${cuda_variant}"
docker compose build parakeet-asr

img_id=$(docker compose images -q parakeet-asr | head -n1)
if [ -n "${img_id:-}" ]; then
target_image="$REGISTRY/$OWNER_LC/parakeet-asr-${cuda_variant}:$VERSION"
latest_image="$REGISTRY/$OWNER_LC/parakeet-asr-${cuda_variant}:latest"
target_image="$REGISTRY/$OWNER_LC/chronicle-asr-nemo-${cuda_variant}:$VERSION"
latest_image="$REGISTRY/$OWNER_LC/chronicle-asr-nemo-${cuda_variant}:latest"
echo "Tagging $img_id as $target_image"
docker tag "$img_id" "$target_image"
echo "Tagging $img_id as $latest_image"
docker tag "$img_id" "$latest_image"

echo "Pushing $target_image"
docker push "$target_image"
echo "Pushing $latest_image"
docker push "$latest_image"

# Clean up local tags
docker image rm -f "$target_image" || true
docker image rm -f "$latest_image" || true
fi

# Aggressive cleanup to save space
docker system prune -af || true
done
cd - > /dev/null
echo "::endgroup::"

# Build and push speaker-recognition with all CUDA variants (including CPU)
# Note: speaker-service has profiles, but we can build it directly by setting PYTORCH_CUDA_VERSION
echo "::group::Building and pushing speaker-recognition variants"
Expand All @@ -238,32 +235,32 @@ jobs:
export PYTORCH_CUDA_VERSION="${cuda_variant}"
# Build speaker-service directly (profiles only affect 'up', not 'build')
docker compose build speaker-service

img_id=$(docker compose images -q speaker-service | head -n1)
if [ -n "${img_id:-}" ]; then
target_image="$REGISTRY/$OWNER_LC/speaker-recognition-${cuda_variant}:$VERSION"
latest_image="$REGISTRY/$OWNER_LC/speaker-recognition-${cuda_variant}:latest"
target_image="$REGISTRY/$OWNER_LC/chronicle-speaker-${cuda_variant}:$VERSION"
latest_image="$REGISTRY/$OWNER_LC/chronicle-speaker-${cuda_variant}:latest"
echo "Tagging $img_id as $target_image"
docker tag "$img_id" "$target_image"
echo "Tagging $img_id as $latest_image"
docker tag "$img_id" "$latest_image"

echo "Pushing $target_image"
docker push "$target_image"
echo "Pushing $latest_image"
docker push "$latest_image"

# Clean up local tags
docker image rm -f "$target_image" || true
docker image rm -f "$latest_image" || true
fi

# Aggressive cleanup to save space
docker system prune -af || true
done
cd - > /dev/null
echo "::endgroup::"

# Summary
echo "::group::Build Summary"
echo "Built and pushed images with version tag: ${VERSION}"
Expand Down Expand Up @@ -291,25 +288,23 @@ jobs:

### Core Services
\`\`\`bash
docker pull ghcr.io/${OWNER_LC}/advanced-chronicle-backend:${VERSION}
docker pull ghcr.io/${OWNER_LC}/advanced-workers:${VERSION}
docker pull ghcr.io/${OWNER_LC}/advanced-webui:${VERSION}
docker pull ghcr.io/${OWNER_LC}/openmemory-mcp:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-backend:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-webui:${VERSION}
\`\`\`

### Parakeet ASR (pick your CUDA version)
\`\`\`bash
docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu121:${VERSION}
docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu126:${VERSION}
docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu128:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu121:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu126:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu128:${VERSION}
\`\`\`

### Speaker Recognition (pick your variant)
\`\`\`bash
docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cpu:${VERSION}
docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu121:${VERSION}
docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu126:${VERSION}
docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu128:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cpu:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu121:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu126:${VERSION}
docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu128:${VERSION}
\`\`\`
EOF
)
Expand Down
35 changes: 20 additions & 15 deletions scripts/pull-images.sh
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
#!/usr/bin/env bash
# pull-images.sh — Pull Chronicle images from DockerHub and retag them locally
# pull-images.sh — Pull Chronicle images and retag them locally
#
# Usage:
# DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0
# Usage (GHCR — default, no account needed):
# ./scripts/pull-images.sh v1.0.0
#
# Usage (custom registry):
# CHRONICLE_REGISTRY=ghcr.io/myuser/ ./scripts/pull-images.sh v1.0.0
# DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0 # backward compat
#
# After pulling, start with the prebuilt images:
# DOCKERHUB_USERNAME=myuser ./start.sh --use-prebuilt v1.0.0
# ./start.sh --use-prebuilt v1.0.0

set -euo pipefail

Expand All @@ -23,20 +27,21 @@ warn() { echo -e "${YELLOW}⚠️ $*${RESET}"; }
error() { echo -e "${RED}❌ $*${RESET}" >&2; }

# ── Validate inputs ────────────────────────────────────────────────────────────
if [[ -z "${DOCKERHUB_USERNAME:-}" ]]; then
error "DOCKERHUB_USERNAME env var is required."
echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0" >&2
exit 1
fi

TAG="${1:-}"
if [[ -z "$TAG" ]]; then
error "TAG argument is required."
echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0" >&2
echo " Example: ./scripts/pull-images.sh v1.0.0" >&2
exit 1
fi

REGISTRY="${DOCKERHUB_USERNAME}/"
# Registry precedence: CHRONICLE_REGISTRY > DOCKERHUB_USERNAME > default GHCR
if [[ -n "${CHRONICLE_REGISTRY:-}" ]]; then
REGISTRY="${CHRONICLE_REGISTRY}"
elif [[ -n "${DOCKERHUB_USERNAME:-}" ]]; then
REGISTRY="${DOCKERHUB_USERNAME}/"
else
REGISTRY="ghcr.io/simpleopensoftware/"
fi

# ── Image inventory ────────────────────────────────────────────────────────────
# Format: "local-image-name:registry-image-name"
Expand Down Expand Up @@ -85,7 +90,7 @@ for entry in "${IMAGES[@]}"; do
success " Ready as ${LOCAL_TAG}"
PULLED+=("${entry##*:}")
else
warn " Not found on DockerHub — skipping (this service may not have been pushed)"
warn " Not found in registry — skipping (this service may not have been pushed)"
FAILED+=("${entry##*:}")
fi
echo ""
Expand Down Expand Up @@ -115,8 +120,8 @@ if [[ ${#PULLED[@]} -gt 0 ]]; then
success "Pulled ${#PULLED[@]} image(s) tagged as ${TAG}"
echo ""
echo "Start services with prebuilt images:"
echo -e " ${BOLD}DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./start.sh --use-prebuilt ${TAG}${RESET}"
echo -e " ${BOLD}./start.sh --use-prebuilt ${TAG}${RESET}"
fi
if [[ ${#FAILED[@]} -gt 0 ]]; then
warn "${#FAILED[@]} image(s) not found on DockerHub (these services will fall back to local builds)"
warn "${#FAILED[@]} image(s) not found in registry (these services will fall back to local builds)"
fi
29 changes: 17 additions & 12 deletions scripts/push-images.sh
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
#!/usr/bin/env bash
# push-images.sh — Tag and push locally-built Chronicle images to DockerHub
# push-images.sh — Tag and push locally-built Chronicle images to a registry
#
# Usage:
# DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 "stable before refactor"
# CHRONICLE_PUSH_REGISTRY=ghcr.io/myuser/ ./scripts/push-images.sh v1.0.0 "stable before refactor"
# DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 "description" # backward compat
#
# Requirements:
# - Images must already be built locally (run ./start.sh --build first)
# - DOCKERHUB_USERNAME env var must be set
# - Must be logged in to DockerHub (docker login)
# - CHRONICLE_PUSH_REGISTRY or DOCKERHUB_USERNAME env var must be set
# - Must be logged in to the target registry (docker login)

set -euo pipefail

Expand All @@ -25,23 +26,27 @@ warn() { echo -e "${YELLOW}⚠️ $*${RESET}"; }
error() { echo -e "${RED}❌ $*${RESET}" >&2; }

# ── Validate inputs ────────────────────────────────────────────────────────────
if [[ -z "${DOCKERHUB_USERNAME:-}" ]]; then
error "DOCKERHUB_USERNAME env var is required."
echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 'description'" >&2
# Registry precedence: CHRONICLE_PUSH_REGISTRY > DOCKERHUB_USERNAME
if [[ -n "${CHRONICLE_PUSH_REGISTRY:-}" ]]; then
REGISTRY="${CHRONICLE_PUSH_REGISTRY}"
elif [[ -n "${DOCKERHUB_USERNAME:-}" ]]; then
REGISTRY="${DOCKERHUB_USERNAME}/"
else
error "CHRONICLE_PUSH_REGISTRY or DOCKERHUB_USERNAME env var is required."
echo " GHCR: CHRONICLE_PUSH_REGISTRY=ghcr.io/yourusername/ ./scripts/push-images.sh v1.0.0" >&2
echo " DockerHub: DOCKERHUB_USERNAME=yourusername ./scripts/push-images.sh v1.0.0" >&2
exit 1
fi

TAG="${1:-}"
if [[ -z "$TAG" ]]; then
error "TAG argument is required."
echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 'description'" >&2
echo " Example: CHRONICLE_PUSH_REGISTRY=ghcr.io/myuser/ ./scripts/push-images.sh v1.0.0 'description'" >&2
exit 1
fi

DESCRIPTION="${2:-}"

REGISTRY="${DOCKERHUB_USERNAME}/"

# ── Image inventory ────────────────────────────────────────────────────────────
# Format: "local-image-name:registry-image-name"
# local-image-name = what docker-compose builds to locally (with empty CHRONICLE_REGISTRY)
Expand Down Expand Up @@ -181,8 +186,8 @@ if [[ ${#PUSHED[@]} -gt 0 ]]; then
success "Pushed ${#PUSHED[@]} image(s) as ${TAG}"
echo ""
echo "To restore this snapshot:"
echo " DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./scripts/pull-images.sh ${TAG}"
echo " DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./start.sh --use-prebuilt ${TAG}"
echo " CHRONICLE_REGISTRY=${REGISTRY} ./scripts/pull-images.sh ${TAG}"
echo " CHRONICLE_REGISTRY=${REGISTRY} ./start.sh --use-prebuilt ${TAG}"
fi
if [[ ${#SKIPPED[@]} -gt 0 ]]; then
warn "${#SKIPPED[@]} image(s) were skipped (not found locally)"
Expand Down
18 changes: 9 additions & 9 deletions services.py
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,7 @@ def main():
start_parser.add_argument(
"--use-prebuilt",
metavar="TAG",
help="Use prebuilt images from DockerHub instead of building (requires prior pull-images.sh run)",
help="Use prebuilt images from GHCR (or custom registry via CHRONICLE_REGISTRY env var)",
)

# Stop command
Expand Down Expand Up @@ -703,16 +703,16 @@ def main():
return

if args.use_prebuilt:
dockerhub_user = os.environ.get("DOCKERHUB_USERNAME", "")
if not dockerhub_user:
console.print(
"[red]❌ DOCKERHUB_USERNAME env var required with --use-prebuilt[/red]"
)
return
os.environ["CHRONICLE_REGISTRY"] = f"{dockerhub_user}/"
if os.environ.get("CHRONICLE_REGISTRY"):
registry = os.environ["CHRONICLE_REGISTRY"]
elif os.environ.get("DOCKERHUB_USERNAME"):
registry = f"{os.environ['DOCKERHUB_USERNAME']}/"
else:
registry = "ghcr.io/simpleopensoftware/"
os.environ["CHRONICLE_REGISTRY"] = registry
os.environ["CHRONICLE_TAG"] = args.use_prebuilt
console.print(
f"[cyan]ℹ️ Using prebuilt images: {dockerhub_user}/*:{args.use_prebuilt}[/cyan]"
f"[cyan]ℹ️ Using prebuilt images: {registry}*:{args.use_prebuilt}[/cyan]"
)
build_flag = False
else:
Expand Down