Skip to content

Reduce Copilot propagation API pressure #58

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
woksin merged 1 commit into from
Jun 17, 2026
Merged

Reduce Copilot propagation API pressure #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 18 additions & 27 deletions .github/scripts/bootstrap-copilot-sync.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,14 @@ sync_b64="bmFtZTogU3luYyBDb3BpbG90IEluc3RydWN0aW9ucwoKb246CiAgd29ya2Zsb3dfZGlzcG
# paths:
# - ".ai/**"
# - ".claude/**"
# - ".agents/**"
# - "AGENTS.md"
# - ".github/copilot-instructions.md"
# - ".github/instructions/**"
# - ".github/agents/**"
# - ".github/skills"
# - ".github/skills/**"
# - ".github/prompts"
# - ".github/prompts/**"
# - ".github/hooks/**"
# workflow_dispatch:
Expand All @@ -136,26 +140,20 @@ sync_b64="bmFtZTogU3luYyBDb3BpbG90IEluc3RydWN0aW9ucwoKb246CiAgd29ya2Zsb3dfZGlzcG
# with:
# event_name: ${{ github.event_name }}
# secrets: inherit
propagate_b64="bmFtZTogUHJvcGFnYXRlIENvcGlsb3QgSW5zdHJ1Y3Rpb25zCgpvbjoKICBwdXNoOgogICAgYnJhbmNoZXM6IFsibWFpbiJdCiAgICBwYXRoczoKICAgICAgLSAiLmFpLyoqIgogICAgICAtICIuY2xhdWRlLyoqIgogICAgICAtICIuZ2l0aHViL2NvcGlsb3QtaW5zdHJ1Y3Rpb25zLm1kIgogICAgICAtICIuZ2l0aHViL2luc3RydWN0aW9ucy8qKiIKICAgICAgLSAiLmdpdGh1Yi9hZ2VudHMvKioiCiAgICAgIC0gIi5naXRodWIvc2tpbGxzLyoqIgogICAgICAtICIuZ2l0aHViL3Byb21wdHMvKioiCiAgICAgIC0gIi5naXRodWIvaG9va3MvKioiCiAgd29ya2Zsb3dfZGlzcGF0Y2g6Cgpqb2JzOgogIHByb3BhZ2F0ZToKICAgIHVzZXM6IENyYXRpcy9Xb3JrZmxvd3MvLmdpdGh1Yi93b3JrZmxvd3MvcHJvcGFnYXRlLWNvcGlsb3QtaW5zdHJ1Y3Rpb25zLnltbEBtYWluCiAgICB3aXRoOgogICAgICBldmVudF9uYW1lOiAke3sgZ2l0aHViLmV2ZW50X25hbWUgfX0KICAgIHNlY3JldHM6IGluaGVyaXQK"
propagate_b64="bmFtZTogUHJvcGFnYXRlIENvcGlsb3QgSW5zdHJ1Y3Rpb25zCgpvbjoKICBwdXNoOgogICAgYnJhbmNoZXM6IFsibWFpbiJdCiAgICBwYXRoczoKICAgICAgLSAiLmFpLyoqIgogICAgICAtICIuY2xhdWRlLyoqIgogICAgICAtICIuYWdlbnRzLyoqIgogICAgICAtICJBR0VOVFMubWQiCiAgICAgIC0gIi5naXRodWIvY29waWxvdC1pbnN0cnVjdGlvbnMubWQiCiAgICAgIC0gIi5naXRodWIvaW5zdHJ1Y3Rpb25zLyoqIgogICAgICAtICIuZ2l0aHViL2FnZW50cy8qKiIKICAgICAgLSAiLmdpdGh1Yi9za2lsbHMiCiAgICAgIC0gIi5naXRodWIvc2tpbGxzLyoqIgogICAgICAtICIuZ2l0aHViL3Byb21wdHMiCiAgICAgIC0gIi5naXRodWIvcHJvbXB0cy8qKiIKICAgICAgLSAiLmdpdGh1Yi9ob29rcy8qKiIKICB3b3JrZmxvd19kaXNwYXRjaDoKCmpvYnM6CiAgcHJvcGFnYXRlOgogICAgdXNlczogQ3JhdGlzL1dvcmtmbG93cy8uZ2l0aHViL3dvcmtmbG93cy9wcm9wYWdhdGUtY29waWxvdC1pbnN0cnVjdGlvbnMueW1sQG1haW4KICAgIHdpdGg6CiAgICAgIGV2ZW50X25hbWU6ICR7eyBnaXRodWIuZXZlbnRfbmFtZSB9fQogICAgc2VjcmV0czogaW5oZXJpdAo="

# Fetch the Copilot setup tree from Cratis/AI once; reused for every repo.
# Prepare the Copilot setup from Cratis/AI once; reused for every repo.
ai_copilot_source_dir=$(mktemp -d)
ai_copilot_files=""
ai_tree_error=$(mktemp)
ai_tree_raw=$(gh_api_with_retry "repos/Cratis/AI/git/trees/main?recursive=1" 2>"$ai_tree_error" || true)
if [ -n "$ai_tree_raw" ]; then
ai_copilot_files=$(echo "$ai_tree_raw" | jq -c \
'[.tree[] | select(.type == "blob") |
select(.path | test("^(\\.github/(copilot-instructions\\.md$|instructions/|agents/|skills/|prompts/|hooks/)|\\.ai/[^/]+(/.*)?|\\.claude/[^/]+(/.*)?)")) |
{path: .path, sha: .sha, mode: .mode}]' 2>/dev/null || true)
if SOURCE_REPO="Cratis/AI" OUTPUT_DIR="$ai_copilot_source_dir" \
bash .github/scripts/prepare-copilot-source-artifact.sh; then
ai_copilot_files=$(jq -c '.' "${ai_copilot_source_dir}/copilot-files.json" 2>/dev/null || true)
fi
if [ -z "$ai_copilot_files" ] || [ "$ai_copilot_files" = "[]" ]; then
ai_tree_api_error=$(cat "$ai_tree_error" 2>/dev/null || true)
echo "⚠ No Copilot setup files found in Cratis/AI; second commit will be skipped"
[ -n "$ai_tree_api_error" ] && echo " API error: $ai_tree_api_error"
else
echo "✓ Found $(echo "$ai_copilot_files" | jq 'length') Copilot setup file(s) in Cratis/AI"
fi
rm -f "$ai_tree_error"

# ================================================================
# Pre-flight: verify PAT has write permission on target repositories
Expand Down Expand Up @@ -299,7 +297,8 @@ echo "$repos" | jq -r '.[]' | while read -r repo; do
# instructions/, agents/, skills/, prompts/, and hooks/ sub-directories.
files_to_delete=$(echo "$subtree" | jq -r \
'.tree[] | select(.type == "blob") |
select(.path | test("^(\\.github/(copilot-instructions\\.md$|instructions/|agents/|skills/|prompts/|hooks/)|\\.ai/[^/]+(/.*)?|\\.claude/[^/]+(/.*)?)")) |
select(.path | test("^(AGENTS\\.md$|\\.agents(/|$)|\\.github/(copilot-instructions\\.md$|instructions(/|$)|agents(/|$)|skills(/|$)|prompts(/|$)|hooks(/|$))|\\.ai/|\\.claude/)")) |
select(.path != ".claude/settings.local.json") |
.path' 2>/dev/null || true)

# Check whether Copilot files from Cratis/AI are already present in
Expand Down Expand Up @@ -332,7 +331,8 @@ echo "$repos" | jq -r '.[]' | while read -r repo; do
# repeated jq invocations inside the loop.
repo_copilot_shas=$(echo "$subtree" | jq -r \
'[.tree[] | select(.type == "blob") |
select(.path | test("^(\\.github/(copilot-instructions\\.md$|instructions/|agents/|skills/|prompts/|hooks/)|\\.ai/[^/]+(/.*)?|\\.claude/[^/]+(/.*)?)"))] |
select(.path | test("^(AGENTS\\.md$|\\.agents(/|$)|\\.github/(copilot-instructions\\.md$|instructions(/|$)|agents(/|$)|skills(/|$)|prompts(/|$)|hooks(/|$))|\\.ai/|\\.claude/)")) |
select(.path != ".claude/settings.local.json")] |
.[] | .path + "\t" + .sha + "\t" + .mode' 2>/dev/null || true)
ai_path_sha_set=$(echo "$ai_copilot_files" | jq -r '.[] | .path + "\t" + .sha + "\t" + (.mode // "100644")' 2>/dev/null || true)

Expand Down Expand Up @@ -442,23 +442,14 @@ echo "$repos" | jq -r '.[]' | while read -r repo; do
while IFS=$'\t' read -r ai_path ai_sha ai_mode; do
[ -z "$ai_path" ] && continue

# Fetch blob content from Cratis/AI (returned as base64 by the API)
ai_blob_error=$(mktemp)
ai_blob_content=$(gh_api_with_retry "repos/Cratis/AI/git/blobs/$ai_sha" \
--jq '.content' 2>"$ai_blob_error" || true)

if [ -z "$ai_blob_content" ]; then
ai_blob_api_error=$(cat "$ai_blob_error" 2>/dev/null || true)
echo " ⚠ Could not fetch blob for $ai_path from Cratis/AI; skipping second commit"
[ -n "$ai_blob_api_error" ] && echo " API error: $ai_blob_api_error"
rm -f "$ai_blob_error"
ai_blob_file="${ai_copilot_source_dir}/blobs/${ai_sha}.b64"
if [ ! -f "$ai_blob_file" ]; then
echo " ⚠ Prepared source artifact is missing blob for $ai_path ($ai_sha); skipping second commit"
ai_copy_failed=true
break
fi
rm -f "$ai_blob_error"

# Strip embedded newlines that the API inserts into base64 output
clean_ai_b64=$(echo "$ai_blob_content" | tr -d '\n')
clean_ai_b64=$(tr -d '\n' < "$ai_blob_file")

target_blob_error=$(mktemp)
_target_blob_resp=$(gh_api_with_retry -X POST "repos/Cratis/$repo/git/blobs" \
Expand Down
23 changes: 18 additions & 5 deletions .github/scripts/copilot-sync-ignore-filter.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,13 @@ _apply_copilot_sync_ignore() {

echo "ℹ Found .copilot-sync-ignore in ${source_repo}"
local ignore_blob
ignore_blob=$(gh api "repos/${source_repo}/git/blobs/${ignore_sha}" \
--jq '.content' 2>/dev/null || true)
if declare -F gh_api_with_retry >/dev/null 2>&1; then
ignore_blob=$(gh_api_with_retry "repos/${source_repo}/git/blobs/${ignore_sha}" \
--jq '.content' 2>/dev/null || true)
else
ignore_blob=$(gh api "repos/${source_repo}/git/blobs/${ignore_sha}" \
--jq '.content' 2>/dev/null || true)
fi
local ignore_content
ignore_content=$(echo "$ignore_blob" | base64 -d 2>/dev/null || true)

Expand All @@ -33,16 +38,24 @@ _apply_copilot_sync_ignore() {
# * → [^/]* (match within a single directory)
# ? → [^/] (match a single character)
# . → \. (literal dot)
# Patterns without a .github/ prefix get one prepended automatically.
# Patterns without an explicit root prefix default to .github/ for backward
# compatibility. Use .ai/, .claude/, .agents/, or AGENTS.md explicitly to
# target those propagated AI surfaces.
local combined_regex=""
local pattern regex
while IFS= read -r pattern || [ -n "$pattern" ]; do
pattern=$(echo "$pattern" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
[ -z "$pattern" ] && continue
[[ "$pattern" == \#* ]] && continue

# Normalize: ensure .github/ prefix
[[ "$pattern" != .github/* ]] && pattern=".github/${pattern}"
# Normalize: default unscoped patterns to .github/ for backward compatibility.
if [[ "$pattern" != .github/* && \
"$pattern" != .ai/* && \
"$pattern" != .claude/* && \
"$pattern" != .agents/* && \
"$pattern" != AGENTS.md ]]; then
pattern=".github/${pattern}"
fi

# Convert glob → regex (order matters: ** before *)
regex=$(printf '%s' "$pattern" \
Expand Down
114 changes: 114 additions & 0 deletions .github/scripts/github-api-retry.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/usr/bin/env bash
# Shared gh api wrapper with bounded retry handling for GitHub API rate limits.
#
# Source this file from scripts that already run with set -euo pipefail.
# The wrapper preserves gh api stdout on success and stderr on failure.

_gh_api_retry_after_seconds() {
local text="$1"
local retry_after

retry_after=$(printf '%s\n' "$text" | awk '
BEGIN { IGNORECASE = 1 }
/^[[:space:]]*retry-after:[[:space:]]*[0-9]+/ {
gsub("\r", "")
print $2
exit
}')

if [[ "$retry_after" =~ ^[0-9]+$ ]]; then
echo "$retry_after"
return 0
fi

return 1
}

_gh_api_is_rate_limited() {
local text="$1"

printf '%s\n' "$text" | grep -qiE \
'API rate limit exceeded|secondary rate limit|rate_limit|abuse detection|Retry-After|exceeded a secondary rate limit'
}

gh_api_with_retry() {
local max_attempts="${GH_API_MAX_ATTEMPTS:-8}"
local base_delay="${GH_API_BASE_DELAY_SECONDS:-15}"
local max_delay="${GH_API_MAX_DELAY_SECONDS:-300}"
local attempt=1
local response=""
local err=""
local cached_input_file=""
local args=("$@")
local i

[[ "$max_attempts" =~ ^[0-9]+$ ]] || max_attempts=8
[[ "$base_delay" =~ ^[0-9]+$ ]] || base_delay=15
[[ "$max_delay" =~ ^[0-9]+$ ]] || max_delay=300

# gh api --input - consumes stdin. Cache it once so retries can replay it.
for ((i = 0; i < ${#args[@]}; i++)); do
if [ "${args[$i]}" = "--input" ]; then
local next_index=$((i + 1))
if [ "$next_index" -lt "${#args[@]}" ] && [ "${args[$next_index]}" = "-" ]; then
cached_input_file=$(mktemp)
cat > "$cached_input_file"
args[$next_index]="$cached_input_file"
break
fi
elif [ "${args[$i]}" = "--input=-" ]; then
cached_input_file=$(mktemp)
cat > "$cached_input_file"
args[$i]="--input=${cached_input_file}"
break
fi
done

while [ "$attempt" -le "$max_attempts" ]; do
local out_file
local err_file
out_file=$(mktemp)
err_file=$(mktemp)

if gh api "${args[@]}" >"$out_file" 2>"$err_file"; then
cat "$out_file"
rm -f "$out_file" "$err_file" "$cached_input_file"
return 0
fi

response=$(cat "$out_file" 2>/dev/null || true)
err=$(cat "$err_file" 2>/dev/null || true)
rm -f "$out_file" "$err_file"

if _gh_api_is_rate_limited "$(printf '%s\n%s' "$response" "$err")" && [ "$attempt" -lt "$max_attempts" ]; then
local wait_seconds
wait_seconds=$(_gh_api_retry_after_seconds "$(printf '%s\n%s' "$response" "$err")" || true)

if [ -z "$wait_seconds" ]; then
wait_seconds=$((attempt * base_delay))
if [ "$wait_seconds" -gt "$max_delay" ]; then
wait_seconds="$max_delay"
fi
fi

if [ "$wait_seconds" -lt 1 ]; then
wait_seconds=1
fi

echo " GitHub API rate limit hit; waiting ${wait_seconds} seconds before retry (attempt ${attempt}/${max_attempts})" >&2
sleep "$wait_seconds"
attempt=$((attempt + 1))
continue
fi

[ -n "$response" ] && echo "$response"
[ -n "$err" ] && echo "$err" >&2
rm -f "$cached_input_file"
return 1
done

[ -n "$response" ] && echo "$response"
[ -n "$err" ] && echo "$err" >&2
rm -f "$cached_input_file"
return 1
}
Loading