From 2c2d575dc4c7250f1d7c80406f50d0715fb7d2f2 Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Thu, 2 Jul 2026 16:36:13 +0100 Subject: [PATCH 1/6] feat: parallelize bundle and image processing in prepare-restricted-environment.sh Parallelize all network-bound operations in the script for an estimated ~8x speedup in disconnected environment workflows: - Add MAX_PARALLEL variable (default 10) with --max-parallel CLI flag - Add reusable throttle_parallel() and wait_for_pids() helper functions - Parallelize process_bundles() with per-worker sed files for race-free render.yaml updates and inner parallel image mirroring - Parallelize process_bundles_from_dir() with same pattern, converting find|while-read to mapfile-based iteration for PID tracking - Parallelize mirror_extra_images() and mirror_extra_images_from_dir() with pre-created namespaces to avoid TOCTOU races - Parallelize ocp_prepare_internal_registry() namespace/policy/secret setup - Add top-level fork-join so extra image mirroring runs concurrently with bundle processing - Update traps to clean up background jobs on exit/signal Setting MAX_PARALLEL=1 restores fully sequential behavior for debugging. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 708 ++++++++++++------ 1 file changed, 489 insertions(+), 219 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index f072b3f7b..3d42e68bb 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -17,6 +17,13 @@ IS_HOSTED_CONTROL_PLANE="" NAMESPACE_OPERATOR="rhdh-operator" INDEX_IMAGE="registry.redhat.io/redhat/redhat-operator-index:v4.18" FILTERED_VERSIONS=(*) +CATALOG_PULL_SECRET="" + +MAX_PARALLEL="${MAX_PARALLEL:-10}" +if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then + echo "[ERROR] MAX_PARALLEL must be a positive integer, got: '$MAX_PARALLEL'" >&2 + exit 1 +fi # assume mikefarah version of yq is already available on the path; if 1, then install the version shown INSTALL_YQ=0 @@ -53,6 +60,38 @@ function errorf() { logf "ERROR" "\033[0;31m" "$1" } +function throttle_parallel() { + local -n __tp_pids=$1 + while true; do + local running=0 + for pid in ${__tp_pids[@]+"${__tp_pids[@]}"}; do + if kill -0 "$pid" 2>/dev/null; then + running=$((running + 1)) + fi + done + if [[ $running -lt $MAX_PARALLEL ]]; then + break + fi + sleep 0.2 + done +} + +function wait_for_pids() { + local -n __wfp_pids=$1 + local context="${2:-background job}" + local failed=0 + for pid in ${__wfp_pids[@]+"${__wfp_pids[@]}"}; do + if ! wait "$pid"; then + failed=$((failed + 1)) + fi + done + __wfp_pids=() + if [[ $failed -gt 0 ]]; then + errorf "${failed} ${context}(s) failed" + return 1 + fi +} + function check_tool() { if ! command -v "$1" >/dev/null; then errorf "Error: Required tool '$1' is not installed." @@ -112,6 +151,7 @@ Options: --to-registry together. --oc-mirror-path : Path to the oc-mirror binary (default: 'oc-mirror'). --oc-mirror-flags : Additional flags to pass to all oc-mirror commands. + --max-parallel : Maximum number of parallel image operations (default: 10, env: MAX_PARALLEL) --install-yq : Install yq $YQ_VERSION from https://github.com/mikefarah/yq (not the jq python wrapper) Examples: @@ -265,7 +305,15 @@ while [[ "$#" -gt 0 ]]; do OC_MIRROR_FLAGS="$2" shift 1 ;; - '--install-yq') + '--max-parallel') + MAX_PARALLEL="$2" + if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then + errorf "MAX_PARALLEL must be a positive integer, got: '$MAX_PARALLEL'" + exit 1 + fi + shift 1 + ;; + '--install-yq') INSTALL_YQ=1 ;; '-h' | '--help') usage @@ -382,10 +430,13 @@ fi if [[ -n "${TO_DIR}" ]]; then mkdir -p "${TO_DIR}" TMPDIR="${TO_DIR}" + trap "jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT + trap "exit 1" INT TERM else TMPDIR=$(mktemp -d) # shellcheck disable=SC2064 - trap "rm -fr $TMPDIR || true" EXIT + trap "rm -fr $TMPDIR || true; jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT + trap "exit 1" INT TERM fi pushd "${TMPDIR}" >/dev/null debugf ">>> WORKING DIR: $TMPDIR <<<" @@ -410,37 +461,66 @@ function ocp_prepare_internal_registry() { # https://access.redhat.com/solutions/6022011 oc patch configs.imageregistry.operator.openshift.io/cluster --patch '{"spec":{"disableRedirect":true}}' --type=merge >&2 my_registry=$(oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}') - skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 - podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 + + skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & + local skopeo_login_pid=$! + podman login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" >&2 & + local podman_login_pid=$! + + local ns_pids=() for ns in rhdh-operator openshift4 rhdh rhel9 oc-mirror; do - # To be able to push images under this scope in the internal image registry - if ! oc get namespace "$ns" &>/dev/null; then - oc create namespace "$ns" >&2 - fi - oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true + ( + if ! oc get namespace "$ns" &>/dev/null; then + oc create namespace "$ns" >&2 + fi + oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true + ) & + ns_pids+=($!) done + wait_for_pids ns_pids "namespace setup" + + if ! wait "$skopeo_login_pid"; then + errorf "skopeo login failed" + return 1 + fi + if ! wait "$podman_login_pid"; then + errorf "podman login failed" + return 1 + fi + + local secret_pids=() for ns in rhdh-operator openshift-marketplace; do - if oc -n ${ns} get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then - oc -n ${ns} delete secret internal-reg-ext-auth-for-rhdh >&2 - fi - oc -n ${ns} create secret docker-registry internal-reg-ext-auth-for-rhdh \ - --docker-server="${my_registry}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry-ext.example.com" >&2 - if oc -n ${ns} get secret internal-reg-auth-for-rhdh &>/dev/null; then - oc -n ${ns} delete secret internal-reg-auth-for-rhdh >&2 - fi - oc -n ${ns} create secret docker-registry internal-reg-auth-for-rhdh \ - --docker-server="${internal_registry_url}" \ - --docker-username=kubeadmin \ - --docker-password="$(oc whoami -t)" \ - --docker-email="admin@internal-registry.example.com" >&2 - oc adm policy add-cluster-role-to-user system:image-signer system:serviceaccount:${ns}:default >&2 || true + ( + if oc -n "${ns}" get secret internal-reg-ext-auth-for-rhdh &>/dev/null; then + oc -n "${ns}" delete secret internal-reg-ext-auth-for-rhdh >&2 + fi + oc -n "${ns}" create secret docker-registry internal-reg-ext-auth-for-rhdh \ + --docker-server="${my_registry}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry-ext.example.com" >&2 + if oc -n "${ns}" get secret internal-reg-auth-for-rhdh &>/dev/null; then + oc -n "${ns}" delete secret internal-reg-auth-for-rhdh >&2 + fi + oc -n "${ns}" create secret docker-registry internal-reg-auth-for-rhdh \ + --docker-server="${internal_registry_url}" \ + --docker-username=kubeadmin \ + --docker-password="$(oc whoami -t)" \ + --docker-email="admin@internal-registry.example.com" >&2 + oc adm policy add-cluster-role-to-user system:image-signer "system:serviceaccount:${ns}:default" >&2 || true + ) & + secret_pids+=($!) done - oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 || true - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 || true - oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 || true + wait_for_pids secret_pids "secret setup" + + local policy_pids=() + oc policy add-role-to-user system:image-puller system:serviceaccount:openshift-marketplace:default -n openshift-marketplace >&2 & + policy_pids+=($!) + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:default -n rhdh-operator >&2 & + policy_pids+=($!) + oc policy add-role-to-user system:image-puller system:serviceaccount:rhdh-operator:rhdh-operator -n rhdh-operator >&2 & + policy_pids+=($!) + wait_for_pids policy_pids "policy setup" } function buildRegistryUrl() { @@ -536,16 +616,37 @@ function extract_last_two_elements() { } function mirror_extra_images() { - debugf "Extra images: " "${EXTRA_IMAGES[@]}" + if [[ ${#EXTRA_IMAGES[@]} -eq 0 ]]; then + return + fi + debugf "Extra images (${#EXTRA_IMAGES[@]}, max ${MAX_PARALLEL} parallel): ${EXTRA_IMAGES[*]}" + + if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + for img in "${EXTRA_IMAGES[@]}"; do + local lastTwo + if [[ "$img" == *"@sha256:"* ]]; then + lastTwo=$(extract_last_two_elements "${img%@*}") + elif [[ "$img" == *":"* ]]; then + lastTwo=$(extract_last_two_elements "${img%:*}") + else + lastTwo=$(extract_last_two_elements "${img}") + fi + local projectNameForOcpReg=${lastTwo%%/*} + oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" + done + fi + + local pids=() for img in "${EXTRA_IMAGES[@]}"; do + local imgDir imgTag lastTwo targetImg if [[ "$img" == *"@sha256:"* ]]; then - imgDigest="${img##*@sha256:}" + local imgDigest="${img##*@sha256:}" imgDir="./extraImages/${img%@*}/sha256_$imgDigest" lastTwo=$(extract_last_two_elements "${img%@*}") targetImg="$(buildRegistryUrl)/${lastTwo}:$imgDigest" elif [[ "$img" == *":"* ]]; then - imgDir="./extraImages/${img%:*}/tag_$imgTag" imgTag="${img##*:}" + imgDir="./extraImages/${img%:*}/tag_$imgTag" lastTwo=$(extract_last_two_elements "${img%:*}") targetImg="$(buildRegistryUrl)/${lastTwo}:$imgTag" else @@ -555,66 +656,104 @@ function mirror_extra_images() { fi if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - mirror_image_to_registry "$img" "$targetImg" + throttle_parallel pids + mirror_image_to_registry "$img" "$targetImg" & + pids+=($!) else if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - mirror_image_to_archive "$img" "$imgDir" + throttle_parallel pids + mirror_image_to_archive "$img" "$imgDir" & + pids+=($!) fi fi done + wait_for_pids pids "extra image" debugf "... done." } function mirror_extra_images_from_dir() { - BASE_DIR="${FROM_DIR}/extraImages" + local BASE_DIR="${FROM_DIR}/extraImages" debugf "Extra images from ${BASE_DIR}..." - if [ -d "${BASE_DIR}" ]; then - # Iterate over all directories named "sha256_*" - find "$BASE_DIR" -type d -name "sha256_*" | while read -r sha256_dir; do - relative_path=${sha256_dir#"$BASE_DIR/"} - sha256_hash=${sha256_dir##*/sha256_} + if [ ! -d "${BASE_DIR}" ]; then + return + fi + + local sha256_dirs=() + mapfile -t sha256_dirs < <(find "$BASE_DIR" -type d -name "sha256_*" 2>/dev/null) + local tag_dirs=() + mapfile -t tag_dirs < <(find "$BASE_DIR" -type d -name "tag_*" 2>/dev/null) + + if [[ ${#sha256_dirs[@]} -eq 0 ]] && [[ ${#tag_dirs[@]} -eq 0 ]]; then + return + fi + + infof "Pushing ${#sha256_dirs[@]} digest + ${#tag_dirs[@]} tag extra images from dir (max ${MAX_PARALLEL} parallel)..." + + if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then + local ns_set=() + for sha256_dir in "${sha256_dirs[@]}"; do + local relative_path=${sha256_dir#"$BASE_DIR/"} + local parent_path parent_path=$(dirname "$relative_path") - debugf "parent_path: $parent_path" + local lastTwo lastTwo=$(extract_last_two_elements "${parent_path}") - extraImg="${lastTwo}:${sha256_hash}" - debugf "Extra-image: $extraImg" - if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - targetImg="$(buildRegistryUrl)/${extraImg%@*}" - push_image_from_archive "$sha256_dir" "$targetImg" - fi + ns_set+=("${lastTwo%%/*}") done - - # Iterate over all directories named "tag_*" - find "$BASE_DIR" -type d -name "tag_*" | while read -r tag_dir; do - relative_path=${tag_dir#"$BASE_DIR/"} - tag_hash=${tag_dir##*/tag_} + for tag_dir in "${tag_dirs[@]}"; do + local relative_path=${tag_dir#"$BASE_DIR/"} + local parent_path parent_path=$(dirname "$relative_path") - debugf "parent_path: $parent_path" + local lastTwo lastTwo=$(extract_last_two_elements "${parent_path}") - extraImg="${lastTwo}:${tag_hash}" - debugf "Extra-image: $extraImg" - if [[ -n "$TO_REGISTRY" ]]; then - if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then - # Create the corresponding project if it doesn't exist - projectNameForOcpReg=${lastTwo%%/*} - oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" - fi - targetImg="$(buildRegistryUrl)/${extraImg%:*}" - push_image_from_archive "$tag_dir" "$targetImg" - fi + ns_set+=("${lastTwo%%/*}") done + local unique_ns + unique_ns=$(printf '%s\n' "${ns_set[@]}" | sort -u) + while IFS= read -r ns; do + if [[ -n "$ns" ]] && ! oc get namespace "$ns" &>/dev/null; then + oc create namespace "$ns" + fi + done <<<"$unique_ns" fi + + local pids=() + + for sha256_dir in "${sha256_dirs[@]}"; do + local relative_path=${sha256_dir#"$BASE_DIR/"} + local sha256_hash=${sha256_dir##*/sha256_} + local parent_path + parent_path=$(dirname "$relative_path") + local lastTwo + lastTwo=$(extract_last_two_elements "${parent_path}") + local extraImg="${lastTwo}:${sha256_hash}" + if [[ -n "$TO_REGISTRY" ]]; then + local targetImg + targetImg="$(buildRegistryUrl)/${extraImg%@*}" + throttle_parallel pids + push_image_from_archive "$sha256_dir" "$targetImg" & + pids+=($!) + fi + done + + for tag_dir in "${tag_dirs[@]}"; do + local relative_path=${tag_dir#"$BASE_DIR/"} + local tag_hash=${tag_dir##*/tag_} + local parent_path + parent_path=$(dirname "$relative_path") + local lastTwo + lastTwo=$(extract_last_two_elements "${parent_path}") + local extraImg="${lastTwo}:${tag_hash}" + if [[ -n "$TO_REGISTRY" ]]; then + local targetImg + targetImg="$(buildRegistryUrl)/${extraImg%:*}" + throttle_parallel pids + push_image_from_archive "$tag_dir" "$targetImg" & + pids+=($!) + fi + done + + wait_for_pids pids "extra image from dir" } function replaceInternalRegIfNeeded() { @@ -630,96 +769,151 @@ function replaceInternalRegIfNeeded() { echo "$img" } -function process_bundles() { +function process_single_bundle() { + set -euo pipefail - for bundleImg in $(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do - debugf "bundleImg=$bundleImg" - originalBundleImg="$bundleImg" - bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") - digest="${bundleImg##*@sha256:}" - if skopeo inspect "docker://$bundleImg" &>/dev/null; then - mkdir -p "bundles/$digest" - debugf "\t copying and unpacking image $bundleImg locally..." - if [ ! -d "./bundles/${digest}/src" ]; then - skopeo copy --remove-signatures "docker://$bundleImg" "oci:./bundles/${digest}/src:latest" + local bundleImg="$1" + local originalBundleImg="$2" + local digest="$3" + local sed_commands_dir="$4" + local bundle_id="$5" + + local bundle_dir="bundles/${digest}" + mkdir -p "${bundle_dir}" + + if ! skopeo copy --remove-signatures "docker://$bundleImg" "oci:./${bundle_dir}/src:latest" 2>"${bundle_dir}/copy.err"; then + debugf "bundle #${bundle_id}: skopeo copy failed, skipping (see ${bundle_dir}/copy.err)" >&2 + return 0 + fi + debugf "bundle #${bundle_id}: pulled ${bundleImg}" >&2 + + umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless + + for file in "./${bundle_dir}/unpacked/rootfs/manifests"/*; do + if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then + "$YQ" eval ' + (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += + {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} + ' -i "$file" + + local all_related_images=() + local images + mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) + if ((${#images[@]})); then + all_related_images+=("${images[@]}") fi - if [ ! -d "./bundles/${digest}/unpacked" ]; then - umoci unpack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" --rootless + local related_images + related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) + if [[ -n "$related_images" ]]; then + while IFS= read -r img; do + [[ -n "$img" ]] && all_related_images+=("$img") + done <<<"$related_images" fi - debugf "\t inspecting related images referenced in bundle image $bundleImg..." - for file in "./bundles/${digest}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - debugf "\t Adding imagePullSecrets to the CSV file so we can pull from private registries" - "$YQ" eval ' - (.spec.install.spec.deployments[] | select(.name == "rhdh-operator").spec.template.spec) += - {"imagePullSecrets": [{"name": "internal-reg-auth-for-rhdh"},{"name": "internal-reg-ext-auth-for-rhdh"},{"name": "reg-pull-secret"}]} - ' -i "$file" - - all_related_images=() - debugf "\t finding related images in $file to mirror..." - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") - fi - # TODO(rm3l): we should use spec.relatedImages instead, but it seems to be incomplete in some bundles - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - while IFS= read -r img; do - [[ -n "$img" ]] && all_related_images+=("$img") - done <<<"$related_images" - fi - for relatedImage in "${all_related_images[@]}"; do - imgDir="./images/" - if [[ "$relatedImage" == *"@sha256:"* ]]; then - relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - lastTwo=$(extract_last_two_elements "${relatedImage%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - lastTwo=$(extract_last_two_elements "${relatedImage%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - lastTwo=$(extract_last_two_elements "${relatedImage}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" - fi + local csv_sed_file="${sed_commands_dir}/${digest}_csv.sed" + : > "$csv_sed_file" + local inner_pids=() + + for relatedImage in "${all_related_images[@]}"; do + local imgDir="./images/" + local lastTwo targetImg internalTargetImg + if [[ "$relatedImage" == *"@sha256:"* ]]; then + local relatedImageDigest="${relatedImage##*@sha256:}" + imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" + lastTwo=$(extract_last_two_elements "${relatedImage%@*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" + elif [[ "$relatedImage" == *":"* ]]; then + local relatedImageTag="${relatedImage##*:}" + imgDir+="${relatedImage%:*}/tag_$relatedImageTag" + lastTwo=$(extract_last_two_elements "${relatedImage%:*}") + targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" + else + imgDir+="${relatedImage}/tag_latest" + lastTwo=$(extract_last_two_elements "${relatedImage}") + targetImg="$(buildRegistryUrl)/${lastTwo}:latest" + internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" + fi - if [[ -n "$TO_REGISTRY" ]]; then - mirror_image_to_registry "$relatedImage" "$targetImg" - debugf "replacing $relatedImage in file '${file}' => $internalTargetImg" - sed -i 's#'"$relatedImage"'#'"$internalTargetImg"'#g' "$file" - else - if [ ! -d "$imgDir" ]; then - mkdir -p "${imgDir}" - mirror_image_to_archive "$relatedImage" "$imgDir" - fi - fi - done + if [[ -n "$TO_REGISTRY" ]]; then + echo "s#${relatedImage}#${internalTargetImg}#g" >> "$csv_sed_file" + throttle_parallel inner_pids + mirror_image_to_registry "$relatedImage" "$targetImg" & + inner_pids+=($!) + else + if [ ! -d "$imgDir" ]; then + mkdir -p "${imgDir}" + throttle_parallel inner_pids + mirror_image_to_archive "$relatedImage" "$imgDir" & + inner_pids+=($!) + fi fi done - if [[ -n "$TO_REGISTRY" ]]; then - # repack the image with the changes - debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." - umoci repack --image "./bundles/${digest}/src:latest" "./bundles/${digest}/unpacked" - - # Push the bundle to the mirror registry - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." - skopeo copy --remove-signatures --dest-tls-verify=false "oci:./bundles/${digest}/src:latest" "docker://${newBundleImage}" + wait_for_pids inner_pids "related image mirror" - sed -i "s#${originalBundleImg}#${newBundleImageInternal}#g" "./rhdh/rhdh/render.yaml" + if [[ -s "$csv_sed_file" ]]; then + sed -i -f "$csv_sed_file" "$file" fi fi done + if [[ -n "$TO_REGISTRY" ]]; then + umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" + + local newBundleImage + newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local newBundleImageInternal + newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 + skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" + + echo "s#${originalBundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" + fi +} + +function process_bundles() { + + local bundle_images + bundle_images=$(grep -E '^image: .*operator-bundle' "${TMPDIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) + + local total_bundles + total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') + infof "Processing ${total_bundles} bundles (max ${MAX_PARALLEL} parallel)..." + + local sed_commands_dir="${TMPDIR}/sed_commands_bundles" + mkdir -p "$sed_commands_dir" + + local bundle_count=0 + local pids=() + + for bundleImg in $bundle_images; do + bundle_count=$((bundle_count + 1)) + local originalBundleImg="$bundleImg" + bundleImg=$(replaceInternalRegIfNeeded "$bundleImg") + local digest="${bundleImg##*@sha256:}" + debugf "bundle #${bundle_count}/${total_bundles}: $originalBundleImg => $bundleImg" + + throttle_parallel pids + + process_single_bundle "$bundleImg" "$originalBundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & + pids+=($!) + done + + wait_for_pids pids "bundle" + + local sed_files + sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' 2>/dev/null || true) + if [[ -n "$sed_files" ]]; then + local combined_sed="${TMPDIR}/combined_bundle_sed.txt" + find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' -exec cat {} + > "$combined_sed" + local replacement_count + replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') + infof "Applying ${replacement_count} image ref replacements to render.yaml..." + sed -i -f "$combined_sed" "./rhdh/rhdh/render.yaml" + fi + if [ ! -f "rhdh/rhdh.Dockerfile" ]; then debugf "\t Regenerating Dockerfile so the index can be rebuilt..." opm generate dockerfile rhdh/rhdh @@ -737,6 +931,97 @@ function process_bundles() { fi } +function process_single_bundle_from_dir() { + set -euo pipefail + + local bundleImg="$1" + local digest="$2" + local sed_commands_dir="$3" + local bundle_id="$4" + + if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then + warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" >&2 + return 0 + fi + if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then + warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" >&2 + return 0 + fi + + debugf "bundle #${bundle_id}: handling from ${FROM_DIR}/bundles/${digest}..." >&2 + + for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do + if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then + local all_related_images=() + local images + mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) + if ((${#images[@]})); then + all_related_images+=("${images[@]}") + fi + local related_images + related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) + if [[ -n "$related_images" ]]; then + while IFS= read -r img; do + [[ -n "$img" ]] && all_related_images+=("$img") + done <<<"$related_images" + fi + + local csv_sed_file="${sed_commands_dir}/${digest}_csv.sed" + : > "$csv_sed_file" + local inner_pids=() + + for relatedImage in "${all_related_images[@]}"; do + local imgDir="${FROM_DIR}/images/" + local targetImg targetImgInternal + if [[ "$relatedImage" == *"@sha256:"* ]]; then + local relatedImageDigest="${relatedImage##*@sha256:}" + imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" + targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" + targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" + elif [[ "$relatedImage" == *":"* ]]; then + local relatedImageTag="${relatedImage##*:}" + imgDir+="${relatedImage%:*}/tag_$relatedImageTag" + targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" + targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" + else + imgDir+="${relatedImage}/tag_latest" + targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage}"):latest" + targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage}"):latest" + fi + if [ ! -d "$imgDir" ]; then + warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" >&2 + continue + fi + if [[ -n "$TO_REGISTRY" ]]; then + echo "s#${relatedImage}#${targetImgInternal}#g" >> "$csv_sed_file" + throttle_parallel inner_pids + push_image_from_archive "$imgDir" "$targetImg" & + inner_pids+=($!) + fi + done + + wait_for_pids inner_pids "related image push" + + if [[ -s "$csv_sed_file" ]]; then + sed -i -f "$csv_sed_file" "$file" + fi + fi + done + + if [[ -n "$TO_REGISTRY" ]]; then + umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" + + local newBundleImage + newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local newBundleImageInternal + newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 + skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" + + echo "s#${bundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" + fi +} + function process_bundles_from_dir() { if [ ! -f "${FROM_DIR}/rhdh/rhdh.Dockerfile" ]; then @@ -748,79 +1033,43 @@ function process_bundles_from_dir() { cp -r "${FROM_DIR}/${d}" "${TMPDIR}/${d}" done - for bundleImg in $(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq); do - debugf "bundleImg=$bundleImg" - digest="${bundleImg##*@sha256:}" - if [ ! -d "${FROM_DIR}/bundles/${digest}/src" ]; then - warnf "missing src image for bundle digest: ${FROM_DIR}/bundles/${digest}/src" - continue - fi - if [ ! -d "${FROM_DIR}/bundles/${digest}/unpacked" ]; then - warnf "missing unpacked image for bundle digest: ${FROM_DIR}/bundles/${digest}/unpacked" - continue - fi + local bundle_images + bundle_images=$(grep -E '^image: .*operator-bundle' "${FROM_DIR}/rhdh/rhdh/render.yaml" | awk '{print $2}' | uniq) - debugf "Handling bundle image from ${FROM_DIR}/bundles/${digest}..." + local total_bundles + total_bundles=$(echo "$bundle_images" | wc -l | tr -d ' ') + infof "Processing ${total_bundles} bundles from dir (max ${MAX_PARALLEL} parallel)..." - debugf "\t inspecting related images referenced in bundle image $bundleImg..." - for file in "${TMPDIR}/bundles/${digest}/unpacked/rootfs/manifests"/*; do - if [[ "$file" == *.clusterserviceversion.yaml || "$file" == *.csv.yaml ]]; then - all_related_images=() - debugf "\t finding related images in $file to mirror..." - mapfile -t images < <(grep -E 'image: ' "$file" | awk -F ': ' '{print $2}' | uniq) - if ((${#images[@]})); then - all_related_images+=("${images[@]}") - fi - # TODO(rm3l): we should use spec.relatedImages instead, but it seems to be incomplete in some bundles - related_images=$("$YQ" '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | select(.name | test("^RELATED_IMAGE_")).value' "$file" || true) - if [[ -n "$related_images" ]]; then - all_related_images+=("$related_images") - fi - for relatedImage in "${all_related_images[@]}"; do - imgDir="${FROM_DIR}/images/" - if [[ "$relatedImage" == *"@sha256:"* ]]; then - relatedImageDigest="${relatedImage##*@sha256:}" - imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - elif [[ "$relatedImage" == *":"* ]]; then - relatedImageTag="${relatedImage##*:}" - imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - else - imgDir+="${relatedImage}/tag_latest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage}"):latest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage}"):latest" - fi - if [ ! -d "$imgDir" ]; then - warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" - continue - fi - if [[ -n "$TO_REGISTRY" ]]; then - push_image_from_archive "$imgDir" "$targetImg" - debugf "replacing $relatedImage in file '${file}' => $targetImgInternal" - sed -i 's#'"$relatedImage"'#'"$targetImgInternal"'#g' "$file" - fi - done - fi - done + local sed_commands_dir="${TMPDIR}/sed_commands_bundles_from_dir" + mkdir -p "$sed_commands_dir" - if [[ -n "$TO_REGISTRY" ]]; then - # repack the image with the changes - debugf "\t Repacking image ./bundles/${digest}/src => ./bundles/${digest}/unpacked..." - umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" + local bundle_count=0 + local pids=() - # Push the bundle to the mirror registry - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - debugf "\t Pushing updated bundle image: ./bundles/${digest}/src => ${newBundleImage}..." - skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" + for bundleImg in $bundle_images; do + bundle_count=$((bundle_count + 1)) + local digest="${bundleImg##*@sha256:}" + debugf "bundle #${bundle_count}/${total_bundles}: $bundleImg" - sed -i "s#${bundleImg}#${newBundleImageInternal}#g" "${TMPDIR}/rhdh/rhdh/render.yaml" - fi + throttle_parallel pids + + process_single_bundle_from_dir "$bundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & + pids+=($!) done + wait_for_pids pids "bundle from dir" + + local sed_files + sed_files=$(find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' 2>/dev/null || true) + if [[ -n "$sed_files" ]]; then + local combined_sed="${TMPDIR}/combined_bundle_from_dir_sed.txt" + find "$sed_commands_dir" -maxdepth 1 -name '*.sed' ! -name '*_csv.sed' -exec cat {} + > "$combined_sed" + local replacement_count + replacement_count=$(wc -l < "$combined_sed" | tr -d ' ') + infof "Applying ${replacement_count} image ref replacements to render.yaml..." + sed -i -f "$combined_sed" "${TMPDIR}/rhdh/rhdh/render.yaml" + fi + if [[ -n "$TO_REGISTRY" ]]; then pushd "${TMPDIR}/rhdh" my_operator_index="$(buildCatalogImageUrl)" @@ -1051,13 +1300,34 @@ EOF fi else if [[ -z "${FROM_DIR}" ]]; then + mirror_extra_images & + extra_mirror_pid=$! + render_index process_bundles + + if ! wait "$extra_mirror_pid"; then + errorf "mirror_extra_images failed" + exit 1 + fi else + mirror_extra_images_from_dir & + extra_from_dir_pid=$! + + mirror_extra_images & + extra_mirror_pid=$! + process_bundles_from_dir - mirror_extra_images_from_dir + + if ! wait "$extra_from_dir_pid"; then + errorf "mirror_extra_images_from_dir failed" + exit 1 + fi + if ! wait "$extra_mirror_pid"; then + errorf "mirror_extra_images failed" + exit 1 + fi fi - mirror_extra_images # create OLM resources manifestsTargetDir="${TMPDIR}" From c250c6511db3d1156e977ddb3496875f2c528907 Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Thu, 2 Jul 2026 17:01:35 +0100 Subject: [PATCH 2/6] fix: resolve shellcheck warnings in prepare-restricted-environment.sh Remove unused CATALOG_PULL_SECRET variable (SC2034) and replace useless echo wrapping oc command output in buildRegistryUrl (SC2005). Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .rhdh/scripts/prepare-restricted-environment.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 3d42e68bb..39cbe94e8 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -17,7 +17,6 @@ IS_HOSTED_CONTROL_PLANE="" NAMESPACE_OPERATOR="rhdh-operator" INDEX_IMAGE="registry.redhat.io/redhat/redhat-operator-index:v4.18" FILTERED_VERSIONS=(*) -CATALOG_PULL_SECRET="" MAX_PARALLEL="${MAX_PARALLEL:-10}" if ! [[ "$MAX_PARALLEL" =~ ^[0-9]+$ ]] || [[ "$MAX_PARALLEL" -lt 1 ]]; then @@ -532,7 +531,7 @@ function buildRegistryUrl() { if [[ "${input}" == "internal" ]]; then echo "image-registry.openshift-image-registry.svc:5000" else - echo "$(oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}')" + oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}' fi else echo "${TO_REGISTRY}" From e4549a190055e3058d9e0cc7e31bf95bfb0b7ffa Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 10:06:01 +0100 Subject: [PATCH 3/6] refactor: replace throttle_parallel with semaphore-based concurrency in prepare-restricted-environment.sh - Introduced semaphore functions (sem_init, sem_acquire, sem_release) to manage parallel execution. - Removed the throttle_parallel function and its usages, enhancing clarity and control over concurrent processes. - Updated traps to ensure proper cleanup of background jobs and temporary directories. - Adjusted image mirroring and processing functions to utilize the new semaphore mechanism for improved synchronization. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 85 +++++++++++-------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 39cbe94e8..1196d1e41 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -59,22 +59,27 @@ function errorf() { logf "ERROR" "\033[0;31m" "$1" } -function throttle_parallel() { - local -n __tp_pids=$1 - while true; do - local running=0 - for pid in ${__tp_pids[@]+"${__tp_pids[@]}"}; do - if kill -0 "$pid" 2>/dev/null; then - running=$((running + 1)) - fi - done - if [[ $running -lt $MAX_PARALLEL ]]; then - break - fi - sleep 0.2 +SEM_FD="" + +function sem_init() { + local fifo + fifo=$(mktemp -u) + mkfifo "$fifo" + exec {SEM_FD}<>"$fifo" + rm -f "$fifo" + for ((i = 0; i < MAX_PARALLEL; i++)); do + printf '\n' >&"${SEM_FD}" done } +function sem_acquire() { + IFS= read -r -u "${SEM_FD}" +} + +function sem_release() { + printf '\n' >&"${SEM_FD}" +} + function wait_for_pids() { local -n __wfp_pids=$1 local context="${2:-background job}" @@ -429,17 +434,19 @@ fi if [[ -n "${TO_DIR}" ]]; then mkdir -p "${TO_DIR}" TMPDIR="${TO_DIR}" - trap "jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT - trap "exit 1" INT TERM + trap '{ jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null; } || true' EXIT + trap 'exit 1' INT TERM else TMPDIR=$(mktemp -d) # shellcheck disable=SC2064 - trap "rm -fr $TMPDIR || true; jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT - trap "exit 1" INT TERM + trap "rm -fr \"$TMPDIR\" || true; { jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null; } || true" EXIT + trap 'exit 1' INT TERM fi pushd "${TMPDIR}" >/dev/null debugf ">>> WORKING DIR: $TMPDIR <<<" +sem_init + if (( INSTALL_YQ )); then YQ=$HOME/.local/bin/yq_mf YQ_BINARY=yq_linux_amd64 @@ -655,13 +662,11 @@ function mirror_extra_images() { fi if [[ -n "$TO_REGISTRY" ]]; then - throttle_parallel pids mirror_image_to_registry "$img" "$targetImg" & pids+=($!) else if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - throttle_parallel pids mirror_image_to_archive "$img" "$imgDir" & pids+=($!) fi @@ -729,7 +734,6 @@ function mirror_extra_images_from_dir() { if [[ -n "$TO_REGISTRY" ]]; then local targetImg targetImg="$(buildRegistryUrl)/${extraImg%@*}" - throttle_parallel pids push_image_from_archive "$sha256_dir" "$targetImg" & pids+=($!) fi @@ -745,8 +749,7 @@ function mirror_extra_images_from_dir() { local extraImg="${lastTwo}:${tag_hash}" if [[ -n "$TO_REGISTRY" ]]; then local targetImg - targetImg="$(buildRegistryUrl)/${extraImg%:*}" - throttle_parallel pids + targetImg="$(buildRegistryUrl)/${extraImg}" push_image_from_archive "$tag_dir" "$targetImg" & pids+=($!) fi @@ -780,10 +783,13 @@ function process_single_bundle() { local bundle_dir="bundles/${digest}" mkdir -p "${bundle_dir}" + sem_acquire if ! skopeo copy --remove-signatures "docker://$bundleImg" "oci:./${bundle_dir}/src:latest" 2>"${bundle_dir}/copy.err"; then + sem_release debugf "bundle #${bundle_id}: skopeo copy failed, skipping (see ${bundle_dir}/copy.err)" >&2 return 0 fi + sem_release debugf "bundle #${bundle_id}: pulled ${bundleImg}" >&2 umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless @@ -837,13 +843,11 @@ function process_single_bundle() { if [[ -n "$TO_REGISTRY" ]]; then echo "s#${relatedImage}#${internalTargetImg}#g" >> "$csv_sed_file" - throttle_parallel inner_pids mirror_image_to_registry "$relatedImage" "$targetImg" & inner_pids+=($!) else if [ ! -d "$imgDir" ]; then mkdir -p "${imgDir}" - throttle_parallel inner_pids mirror_image_to_archive "$relatedImage" "$imgDir" & inner_pids+=($!) fi @@ -866,7 +870,9 @@ function process_single_bundle() { local newBundleImageInternal newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 - skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" + sem_acquire + skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } + sem_release echo "s#${originalBundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" fi @@ -894,8 +900,6 @@ function process_bundles() { local digest="${bundleImg##*@sha256:}" debugf "bundle #${bundle_count}/${total_bundles}: $originalBundleImg => $bundleImg" - throttle_parallel pids - process_single_bundle "$bundleImg" "$originalBundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & pids+=($!) done @@ -993,7 +997,6 @@ function process_single_bundle_from_dir() { fi if [[ -n "$TO_REGISTRY" ]]; then echo "s#${relatedImage}#${targetImgInternal}#g" >> "$csv_sed_file" - throttle_parallel inner_pids push_image_from_archive "$imgDir" "$targetImg" & inner_pids+=($!) fi @@ -1015,7 +1018,9 @@ function process_single_bundle_from_dir() { local newBundleImageInternal newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 - skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" + sem_acquire + skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } + sem_release echo "s#${bundleImg}#${newBundleImageInternal}#g" > "${sed_commands_dir}/${digest}.sed" fi @@ -1050,8 +1055,6 @@ function process_bundles_from_dir() { local digest="${bundleImg##*@sha256:}" debugf "bundle #${bundle_count}/${total_bundles}: $bundleImg" - throttle_parallel pids - process_single_bundle_from_dir "$bundleImg" "$digest" "$sed_commands_dir" "$bundle_count" & pids+=($!) done @@ -1082,30 +1085,42 @@ function process_bundles_from_dir() { } function mirror_image_to_registry() { + sem_acquire + local rc=0 local src_image src_image=$(replaceInternalRegIfNeeded "$1") local dest_image dest_image=$2 - + echo "Mirroring $src_image to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" || rc=$? + sem_release + return $rc } function mirror_image_to_archive() { + sem_acquire + local rc=0 local src_image src_image=$(replaceInternalRegIfNeeded "$1") local archive_path archive_path="$2" debugf "Saving $src_image to $archive_path..." - skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" + skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" || rc=$? + sem_release + return $rc } function push_image_from_archive() { + sem_acquire + local rc=0 local archive_path=$1 local dest_image=$2 echo "Pushing $archive_path to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" || rc=$? + sem_release + return $rc } check_tool "yq" From 33e3f78cdf699f95862318f2cdcee1454bd1527a Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 10:40:20 +0100 Subject: [PATCH 4/6] refactor: enhance registry URL handling and streamline image processing in prepare-restricted-environment.sh - Introduced caching for registry URLs to optimize repeated calls. - Replaced direct calls to buildRegistryUrl with cached values in image processing functions. - Added a new helper function _last_two to simplify extraction of last two path elements. - Updated various functions to utilize the new caching mechanism, improving performance and readability. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 159 +++++++++++------- 1 file changed, 99 insertions(+), 60 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 1196d1e41..ebf9bde07 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -545,11 +545,27 @@ function buildRegistryUrl() { fi } +CACHED_REGISTRY_URL="" +CACHED_REGISTRY_URL_INTERNAL="" + +function cache_registry_urls() { + if [[ -n "$TO_REGISTRY" ]]; then + CACHED_REGISTRY_URL=$(buildRegistryUrl) + CACHED_REGISTRY_URL_INTERNAL=$(buildRegistryUrl "internal") + fi +} + function buildCatalogImageUrl() { if [[ -n "$TO_REGISTRY" ]]; then tag=${INDEX_IMAGE##*:} [[ "$INDEX_IMAGE" == "$tag" ]] && tag="latest" - echo "$(buildRegistryUrl "${1:-external}")/${2:-rhdh/index}:${tag}" + local reg_url + if [[ "${1:-external}" == "internal" ]]; then + reg_url="${CACHED_REGISTRY_URL_INTERNAL}" + else + reg_url="${CACHED_REGISTRY_URL}" + fi + echo "${reg_url}/${2:-rhdh/index}:${tag}" else echo "" fi @@ -599,7 +615,7 @@ function render_index() { >"${local_index_file}" fi - debugf "Got $(cat "${local_index_file}" | wc -l) lines of JSON from the index!" + debugf "Got $(wc -l < "${local_index_file}") lines of JSON from the index!" if [ ! -s "${local_index_file}" ]; then errorf "[ERROR] 'opm render $INDEX_IMAGE' returned an empty output, which likely means that this index Image does not contain the rhdh operator." @@ -621,6 +637,18 @@ function extract_last_two_elements() { fi } +function _last_two() { + local -n __lt_result=$1 + local __lt_input="$2" + if [[ "$__lt_input" == */*/* ]]; then + local __lt_last="${__lt_input##*/}" + local __lt_rest="${__lt_input%/*}" + __lt_result="${__lt_rest##*/}/${__lt_last}" + else + __lt_result="$__lt_input" + fi +} + function mirror_extra_images() { if [[ ${#EXTRA_IMAGES[@]} -eq 0 ]]; then return @@ -629,13 +657,13 @@ function mirror_extra_images() { if [[ -n "$TO_REGISTRY" ]] && [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then for img in "${EXTRA_IMAGES[@]}"; do - local lastTwo + local lastTwo="" if [[ "$img" == *"@sha256:"* ]]; then - lastTwo=$(extract_last_two_elements "${img%@*}") + _last_two lastTwo "${img%@*}" elif [[ "$img" == *":"* ]]; then - lastTwo=$(extract_last_two_elements "${img%:*}") + _last_two lastTwo "${img%:*}" else - lastTwo=$(extract_last_two_elements "${img}") + _last_two lastTwo "${img}" fi local projectNameForOcpReg=${lastTwo%%/*} oc get namespace "${projectNameForOcpReg}" &>/dev/null || oc create namespace "${projectNameForOcpReg}" @@ -644,21 +672,21 @@ function mirror_extra_images() { local pids=() for img in "${EXTRA_IMAGES[@]}"; do - local imgDir imgTag lastTwo targetImg + local imgDir imgTag lastTwo="" targetImg if [[ "$img" == *"@sha256:"* ]]; then local imgDigest="${img##*@sha256:}" imgDir="./extraImages/${img%@*}/sha256_$imgDigest" - lastTwo=$(extract_last_two_elements "${img%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$imgDigest" + _last_two lastTwo "${img%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgDigest" elif [[ "$img" == *":"* ]]; then imgTag="${img##*:}" imgDir="./extraImages/${img%:*}/tag_$imgTag" - lastTwo=$(extract_last_two_elements "${img%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$imgTag" + _last_two lastTwo "${img%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$imgTag" else imgDir="./extraImages/${img}/tag_latest" - lastTwo=$(extract_last_two_elements "${img}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" + _last_two lastTwo "${img}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" fi if [[ -n "$TO_REGISTRY" ]]; then @@ -700,16 +728,16 @@ function mirror_extra_images_from_dir() { local relative_path=${sha256_dir#"$BASE_DIR/"} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" ns_set+=("${lastTwo%%/*}") done for tag_dir in "${tag_dirs[@]}"; do local relative_path=${tag_dir#"$BASE_DIR/"} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" ns_set+=("${lastTwo%%/*}") done local unique_ns @@ -728,12 +756,12 @@ function mirror_extra_images_from_dir() { local sha256_hash=${sha256_dir##*/sha256_} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" local extraImg="${lastTwo}:${sha256_hash}" if [[ -n "$TO_REGISTRY" ]]; then local targetImg - targetImg="$(buildRegistryUrl)/${extraImg%@*}" + targetImg="${CACHED_REGISTRY_URL}/${extraImg%@*}" push_image_from_archive "$sha256_dir" "$targetImg" & pids+=($!) fi @@ -744,12 +772,12 @@ function mirror_extra_images_from_dir() { local tag_hash=${tag_dir##*/tag_} local parent_path parent_path=$(dirname "$relative_path") - local lastTwo - lastTwo=$(extract_last_two_elements "${parent_path}") + local lastTwo="" + _last_two lastTwo "${parent_path}" local extraImg="${lastTwo}:${tag_hash}" if [[ -n "$TO_REGISTRY" ]]; then local targetImg - targetImg="$(buildRegistryUrl)/${extraImg}" + targetImg="${CACHED_REGISTRY_URL}/${extraImg}" push_image_from_archive "$tag_dir" "$targetImg" & pids+=($!) fi @@ -821,24 +849,24 @@ function process_single_bundle() { for relatedImage in "${all_related_images[@]}"; do local imgDir="./images/" - local lastTwo targetImg internalTargetImg + local lastTwo="" targetImg internalTargetImg if [[ "$relatedImage" == *"@sha256:"* ]]; then local relatedImageDigest="${relatedImage##*@sha256:}" imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - lastTwo=$(extract_last_two_elements "${relatedImage%@*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageDigest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageDigest" + _last_two lastTwo "${relatedImage%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" elif [[ "$relatedImage" == *":"* ]]; then local relatedImageTag="${relatedImage##*:}" imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - lastTwo=$(extract_last_two_elements "${relatedImage%:*}") - targetImg="$(buildRegistryUrl)/${lastTwo}:$relatedImageTag" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:$relatedImageTag" + _last_two lastTwo "${relatedImage%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" else imgDir+="${relatedImage}/tag_latest" - lastTwo=$(extract_last_two_elements "${relatedImage}") - targetImg="$(buildRegistryUrl)/${lastTwo}:latest" - internalTargetImg="$(buildRegistryUrl "internal")/${lastTwo}:latest" + _last_two lastTwo "${relatedImage}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" + internalTargetImg="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" fi if [[ -n "$TO_REGISTRY" ]]; then @@ -865,10 +893,10 @@ function process_single_bundle() { if [[ -n "$TO_REGISTRY" ]]; then umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" - local newBundleImage - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - local newBundleImageInternal - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local bundleLastTwo="" + _last_two bundleLastTwo "${bundleImg%@*}" + local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" + local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 sem_acquire skopeo copy --remove-signatures --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } @@ -975,21 +1003,24 @@ function process_single_bundle_from_dir() { for relatedImage in "${all_related_images[@]}"; do local imgDir="${FROM_DIR}/images/" - local targetImg targetImgInternal + local lastTwo="" targetImg targetImgInternal if [[ "$relatedImage" == *"@sha256:"* ]]; then local relatedImageDigest="${relatedImage##*@sha256:}" imgDir+="${relatedImage%@*}/sha256_$relatedImageDigest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%@*}"):$relatedImageDigest" + _last_two lastTwo "${relatedImage%@*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageDigest" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageDigest" elif [[ "$relatedImage" == *":"* ]]; then local relatedImageTag="${relatedImage##*:}" imgDir+="${relatedImage%:*}/tag_$relatedImageTag" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage%:*}"):$relatedImageTag" + _last_two lastTwo "${relatedImage%:*}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:$relatedImageTag" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:$relatedImageTag" else imgDir+="${relatedImage}/tag_latest" - targetImg="$(buildRegistryUrl)/$(extract_last_two_elements "${relatedImage}"):latest" - targetImgInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${relatedImage}"):latest" + _last_two lastTwo "${relatedImage}" + targetImg="${CACHED_REGISTRY_URL}/${lastTwo}:latest" + targetImgInternal="${CACHED_REGISTRY_URL_INTERNAL}/${lastTwo}:latest" fi if [ ! -d "$imgDir" ]; then warnf "Skipping related image $relatedImage not found mirrored in dir: $FROM_DIR/images" >&2 @@ -1013,10 +1044,10 @@ function process_single_bundle_from_dir() { if [[ -n "$TO_REGISTRY" ]]; then umoci repack --image "${TMPDIR}/bundles/${digest}/src:latest" "${TMPDIR}/bundles/${digest}/unpacked" - local newBundleImage - newBundleImage="$(buildRegistryUrl)/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" - local newBundleImageInternal - newBundleImageInternal="$(buildRegistryUrl "internal")/$(extract_last_two_elements "${bundleImg%@*}"):${digest}" + local bundleLastTwo="" + _last_two bundleLastTwo "${bundleImg%@*}" + local newBundleImage="${CACHED_REGISTRY_URL}/${bundleLastTwo}:${digest}" + local newBundleImageInternal="${CACHED_REGISTRY_URL_INTERNAL}/${bundleLastTwo}:${digest}" debugf "bundle #${bundle_id}: pushing ${newBundleImage}" >&2 sem_acquire skopeo copy --preserve-digests --remove-signatures --dest-tls-verify=false "oci:${TMPDIR}/bundles/${digest}/src:latest" "docker://${newBundleImage}" || { sem_release; return 1; } @@ -1087,10 +1118,14 @@ function process_bundles_from_dir() { function mirror_image_to_registry() { sem_acquire local rc=0 - local src_image - src_image=$(replaceInternalRegIfNeeded "$1") - local dest_image - dest_image=$2 + local src_image="$1" + if [[ "${IS_CI_INDEX_IMAGE}" == "true" ]]; then + for reg in registry.stage.redhat.io registry.redhat.io; do + src_image="${src_image/$reg\/rhdh/quay.io\/rhdh}" + done + src_image="${src_image/registry-proxy.engineering.redhat.com\/rh-osbs\/rhdh-/quay.io\/rhdh\/}" + fi + local dest_image=$2 echo "Mirroring $src_image to $dest_image..." skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false docker://"$src_image" docker://"$dest_image" || rc=$? @@ -1101,10 +1136,14 @@ function mirror_image_to_registry() { function mirror_image_to_archive() { sem_acquire local rc=0 - local src_image - src_image=$(replaceInternalRegIfNeeded "$1") - local archive_path - archive_path="$2" + local src_image="$1" + if [[ "${IS_CI_INDEX_IMAGE}" == "true" ]]; then + for reg in registry.stage.redhat.io registry.redhat.io; do + src_image="${src_image/$reg\/rhdh/quay.io\/rhdh}" + done + src_image="${src_image/registry-proxy.engineering.redhat.com\/rh-osbs\/rhdh-/quay.io\/rhdh\/}" + fi + local archive_path="$2" debugf "Saving $src_image to $archive_path..." skopeo copy --preserve-digests --remove-signatures --all --preserve-digests --dest-tls-verify=false docker://"$src_image" dir:"$archive_path" || rc=$? @@ -1138,7 +1177,7 @@ detect_ocp_and_set_env_var if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then ocp_prepare_internal_registry fi - +cache_registry_urls manifestsTargetDir="${TMPDIR}" if [[ -n "${FROM_DIR}" ]]; then @@ -1203,7 +1242,7 @@ EOF mirror_image_to_archive "registry.redhat.io/ubi9/ubi:latest" "${TO_DIR}/rhdh-catalog" fi if [[ -n "$TO_REGISTRY" ]]; then - registryUrl=$(buildRegistryUrl) + registryUrl="${CACHED_REGISTRY_URL}" if [[ "${TO_REGISTRY}" == "OCP_INTERNAL" ]]; then registryUrl+="/oc-mirror" fi @@ -1233,7 +1272,7 @@ EOF exit 1 fi if [[ -n "${TO_REGISTRY}" ]]; then - registryUrl=$(buildRegistryUrl) + registryUrl="${CACHED_REGISTRY_URL}" if [[ "${TO_REGISTRY}" == "OCP_INTERNAL" ]]; then registryUrl+="/oc-mirror" fi @@ -1411,7 +1450,7 @@ EOF if [[ -n "${TO_REGISTRY}" ]]; then # IDMS will only work on regular OCP clusters. It doesn't work on ROSA or clusters with hosted control planes like on IBM Cloud. - registry_url_internal=$(buildRegistryUrl) + registry_url_internal="${CACHED_REGISTRY_URL}" cat <"${manifestsTargetDir}/imageDigestMirrorSet.yaml" apiVersion: config.openshift.io/v1 kind: ImageDigestMirrorSet From a53505f8f6531c64e86d7760b2a7c87f8ac6fffb Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 10:56:25 +0100 Subject: [PATCH 5/6] docs: update --max-parallel option description in prepare-restricted-environment.sh - Enhanced the documentation for the --max-parallel option to clarify its impact on disk usage and recommend lowering the value when disk space is limited. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .rhdh/scripts/prepare-restricted-environment.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index ebf9bde07..979a7d511 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -155,7 +155,9 @@ Options: --to-registry together. --oc-mirror-path : Path to the oc-mirror binary (default: 'oc-mirror'). --oc-mirror-flags : Additional flags to pass to all oc-mirror commands. - --max-parallel : Maximum number of parallel image operations (default: 10, env: MAX_PARALLEL) + --max-parallel : Maximum number of parallel image operations (default: 10, env: MAX_PARALLEL). + Lower this value if you are running low on disk space, as fewer concurrent + downloads will reduce peak disk usage. --install-yq : Install yq $YQ_VERSION from https://github.com/mikefarah/yq (not the jq python wrapper) Examples: From a001c0cb2de59b94dbc5b5d79f79d9b502a84b2a Mon Sep 17 00:00:00 2001 From: Fortune-Ndlovu Date: Fri, 3 Jul 2026 14:05:59 +0100 Subject: [PATCH 6/6] feat: implement retry mechanism for image push in prepare-restricted-environment.sh - Added a retry mechanism for the push_image_from_archive function to handle transient failures during image pushing. - Introduced a locking mechanism to prevent duplicate pushes for the same destination image. - Enhanced error handling with exponential backoff for retries. Ref: RHDHBUGS-3432 Signed-off-by: Fortune-Ndlovu --- .../scripts/prepare-restricted-environment.sh | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.rhdh/scripts/prepare-restricted-environment.sh b/.rhdh/scripts/prepare-restricted-environment.sh index 979a7d511..085bf34e0 100755 --- a/.rhdh/scripts/prepare-restricted-environment.sh +++ b/.rhdh/scripts/prepare-restricted-environment.sh @@ -1154,12 +1154,31 @@ function mirror_image_to_archive() { } function push_image_from_archive() { - sem_acquire - local rc=0 local archive_path=$1 local dest_image=$2 + + local lock_key="${dest_image//\//__}" + lock_key="${lock_key//:/_}" + if ! mkdir "$TMPDIR/.push_locks/$lock_key" 2>/dev/null; then + debugf "Skipping duplicate push: $dest_image (already handled)" + return 0 + fi + + sem_acquire + local rc=0 + local max_retries=3 + local retry_delay=5 echo "Pushing $archive_path to $dest_image..." - skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" || rc=$? + for ((attempt=1; attempt<=max_retries; attempt++)); do + rc=0 + skopeo copy --preserve-digests --remove-signatures --all --dest-tls-verify=false dir:"$archive_path" docker://"$dest_image" || rc=$? + if [[ $rc -eq 0 ]]; then break; fi + if ((attempt < max_retries)); then + warnf "Push failed (attempt $attempt/$max_retries), retrying in ${retry_delay}s... [$dest_image]" + sleep "$retry_delay" + retry_delay=$((retry_delay * 2)) + fi + done sem_release return $rc } @@ -1180,6 +1199,7 @@ if [[ "${IS_OPENSHIFT}" = "true" && "${TO_REGISTRY}" = "OCP_INTERNAL" ]]; then ocp_prepare_internal_registry fi cache_registry_urls +mkdir -p "$TMPDIR/.push_locks" manifestsTargetDir="${TMPDIR}" if [[ -n "${FROM_DIR}" ]]; then