From 8c7cd784faf16988ce1a8bf7ef40d5093717b0b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 24 Mar 2026 05:23:17 +0000 Subject: [PATCH 01/74] Add Bitbucket Server to GitHub Enterprise migration tool Python CLI (bb2gh) with three subcommands: - migrate: bulk clone repos via SSH, push --mirror to GitHub - sync: continuous fetch+push every 60s for smooth transition - migrate-prs: recreate open Bitbucket PRs on GitHub with comments Includes Bitbucket Server REST API client, GitHub Enterprise client (PyGithub), state tracking, Docker support, and 15 passing tests. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- Dockerfile | 21 +++ README.md | 94 +++++++++- bb2gh/__init__.py | 1 + bb2gh/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 222 bytes .../bitbucket_client.cpython-311.pyc | Bin 0 -> 4962 bytes bb2gh/__pycache__/config.cpython-311.pyc | Bin 0 -> 3192 bytes .../__pycache__/github_client.cpython-311.pyc | Bin 0 -> 4956 bytes bb2gh/__pycache__/migrator.cpython-311.pyc | Bin 0 -> 7250 bytes bb2gh/__pycache__/pr_migrator.cpython-311.pyc | Bin 0 -> 7635 bytes bb2gh/__pycache__/state.cpython-311.pyc | Bin 0 -> 5515 bytes bb2gh/__pycache__/syncer.cpython-311.pyc | Bin 0 -> 6934 bytes bb2gh/bitbucket_client.py | 82 ++++++++ bb2gh/cli.py | 83 +++++++++ bb2gh/config.py | 52 ++++++ bb2gh/github_client.py | 86 +++++++++ bb2gh/migrator.py | 141 ++++++++++++++ bb2gh/pr_migrator.py | 169 +++++++++++++++++ bb2gh/state.py | 79 ++++++++ bb2gh/syncer.py | 126 +++++++++++++ config.yaml.example | 31 +++ docker-compose.yml | 35 ++++ requirements.txt | 5 + setup.py | 21 +++ tests/__init__.py | 0 tests/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 155 bytes ...test_migrator.cpython-311-pytest-9.0.2.pyc | Bin 0 -> 9929 bytes ...t_pr_migrator.cpython-311-pytest-9.0.2.pyc | Bin 0 -> 18766 bytes .../test_syncer.cpython-311-pytest-9.0.2.pyc | Bin 0 -> 4245 bytes tests/test_migrator.py | 102 ++++++++++ tests/test_pr_migrator.py | 176 ++++++++++++++++++ tests/test_syncer.py | 78 ++++++++ 31 files changed, 1381 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 bb2gh/__init__.py create mode 100644 bb2gh/__pycache__/__init__.cpython-311.pyc create mode 100644 bb2gh/__pycache__/bitbucket_client.cpython-311.pyc create mode 100644 bb2gh/__pycache__/config.cpython-311.pyc create mode 100644 bb2gh/__pycache__/github_client.cpython-311.pyc create mode 100644 bb2gh/__pycache__/migrator.cpython-311.pyc create mode 100644 bb2gh/__pycache__/pr_migrator.cpython-311.pyc create mode 100644 bb2gh/__pycache__/state.cpython-311.pyc create mode 100644 bb2gh/__pycache__/syncer.cpython-311.pyc create mode 100644 bb2gh/bitbucket_client.py create mode 100644 bb2gh/cli.py create mode 100644 bb2gh/config.py create mode 100644 bb2gh/github_client.py create mode 100644 bb2gh/migrator.py create mode 100644 bb2gh/pr_migrator.py create mode 100644 bb2gh/state.py create mode 100644 bb2gh/syncer.py create mode 100644 config.yaml.example create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-311.pyc create mode 100644 tests/__pycache__/test_migrator.cpython-311-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_pr_migrator.cpython-311-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_syncer.cpython-311-pytest-9.0.2.pyc create mode 100644 tests/test_migrator.py create mode 100644 tests/test_pr_migrator.py create mode 100644 tests/test_syncer.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..84501e3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + openssh-client \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +RUN pip install --no-cache-dir -e . + +# SSH config for non-interactive git clone +RUN mkdir -p /root/.ssh && \ + echo "StrictHostKeyChecking no" >> /root/.ssh/config + +ENTRYPOINT ["bb2gh"] +CMD ["--help"] diff --git a/README.md b/README.md index e4afa9e..b977a76 100644 --- a/README.md +++ b/README.md @@ -1 +1,93 @@ -# bitbucket--github \ No newline at end of file +# bb2gh — Bitbucket Server to GitHub Enterprise Migration + +A Python CLI tool for migrating repositories and pull requests from self-hosted Bitbucket Server (Data Center) to GitHub Enterprise, with continuous sync support for a smooth transition. + +## Features + +- **Bulk migration** — Clone all repos from Bitbucket, push to GitHub (branches, tags, full history) +- **Continuous sync** — Fetch from Bitbucket and push to GitHub every 60 seconds +- **PR migration** — Recreate open Bitbucket PRs on GitHub with title, description, comments, and reviewers +- **Idempotent** — Safe to re-run; skips already-migrated repos and PRs +- **Docker support** — Run the sync as a long-lived service + +## Quick Start + +### 1. Configure + +```bash +cp config.yaml.example config.yaml +# Edit config.yaml with your Bitbucket and GitHub credentials +``` + +### 2. Install + +```bash +pip install -r requirements.txt +pip install -e . +``` + +### 3. Migrate + +```bash +# Bulk migrate all repos +bb2gh --config config.yaml migrate + +# Start continuous sync (runs until interrupted) +bb2gh --config config.yaml sync + +# Migrate open PRs (dry-run first) +bb2gh --config config.yaml migrate-prs --dry-run +bb2gh --config config.yaml migrate-prs +``` + +### Docker + +```bash +# Copy your config +mkdir config && cp config.yaml config/ + +# Run bulk migration +docker compose --profile migrate run migrate + +# Start continuous sync +docker compose up -d sync + +# Migrate PRs +docker compose --profile migrate-prs run migrate-prs +``` + +## Configuration + +See `config.yaml.example` for all options. Key settings: + +| Setting | Description | +|---------|-------------| +| `bitbucket.base_url` | Bitbucket Server URL | +| `bitbucket.token` | Personal access token (or set `BB_TOKEN` env var) | +| `bitbucket.ssh_url` | SSH base URL for git clone | +| `bitbucket.projects` | Optional list of projects to migrate (omit for all) | +| `github.base_url` | GitHub Enterprise API URL | +| `github.token` | GitHub PAT with repo + admin:org (or set `GH_TOKEN` env var) | +| `github.org` | Target GitHub organization | +| `sync.interval_seconds` | Sync frequency (default: 60) | +| `user_mapping` | Bitbucket → GitHub username mapping for PR reviewers | + +## How It Works + +1. **`migrate`** — For each Bitbucket repo: creates a GitHub repo, bare-clones via SSH, cleans hidden refs, and pushes `--mirror` +2. **`sync`** — Loops every N seconds: `git fetch origin --prune` then `git push github --mirror` for each migrated repo +3. **`migrate-prs`** — For each open PR: creates a GitHub PR with metadata header, migrates comments, and assigns reviewers + +## PR Migration Notes + +- PRs are created under the service account (original author is noted in the PR body) +- Inline/file-level comments are migrated as regular PR comments +- Reviewer assignments use the `user_mapping` config (falls back to same username) +- Merged/closed PRs are not migrated (only open PRs) + +## Testing + +```bash +pip install pytest +python -m pytest tests/ -v +``` diff --git a/bb2gh/__init__.py b/bb2gh/__init__.py new file mode 100644 index 0000000..7cc069d --- /dev/null +++ b/bb2gh/__init__.py @@ -0,0 +1 @@ +"""Bitbucket Server to GitHub Enterprise migration tool.""" diff --git a/bb2gh/__pycache__/__init__.cpython-311.pyc b/bb2gh/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8a524037c766ec60f7a749bb1b32f1190f6248c GIT binary patch literal 222 zcmZ3^%ge<81efIwW!eDg#~=<2FhUuhK}x1Gq%cG=q%a0EXfjoqI%Sq5l_qDWmM8?L z7L}zIDU{?ZxM!AllqM;-=9Q!t6%=I_rz+%TrWYlaWaj4qrSo(2{4^PFvB$@!1oEL$HVWxhMe0rm&j16)_TCvs z;0zKEQN^y1%0`hu-A46OnrQQohd$(`s?=X#BTHB#trV%M)Kyub=$%^qrbs=qPYu!FD8udGVxq%9P6eP zS2IQ&zA@*!p{p1S=ClrHt=Tj-a;jydQ!$?-V$;G##*wB}Q&+M$>G)s~MI@|$l?5)Q zjp4U?*QJ^f#yI10~=>nEc1k??W%R z;<3KcvA$J)t+jmO^dI^Qkt^;CBf%dcU8P8OF%m09Vx(VgX|Ev0e~T$!9(|)1E|0>y zIzNz7>v;xP&{f8TfX+vk>z|;kWZ882o350spo<(<5ZF8OA`3j>FO0iEYkhtnYrd4zrp#0rZ-5+!MrRH9v9gwa2kQZs;xq zuWISCHH*{PnORv)CgqGeqnmPiT2||V9tMpRP-?QC(lTiSP9{f48ViQ?BjNqvCF6)0 z6{`!TBR*7<;N#;?z%a*D)4B)zoYs`SNU;e^#g-$Oq=|_RuNeu;37ER2>C7~D9yhD_$U0Wpnr56U7uc0=GkILwA2w@70XiFwsd?;I=3oIv`!AO}BTt96JEL2j(frNVVxcoy?7Uj)yt+D44z_Ox&uj(H6ob)HFj@#kfp+8i zlk4Ap{Pg41k++eP8;egC^U{m6#YlfC(q9N3@V*Uo{%-M`#m!(|D~5VYp9hGyOQ$cb-dex<_2-pN5C)SENz{h|(FSqxok;*2#Py+dA2>&uXY4X#)rEkK z@?L1v=tyhtJDVE=5RwO*>l-sn%k>|ep3@T+xKIlGO0RPwBL#L$CG>%uKc1;lz2q=S z!498<_4q?TV=T2VJ`1=>#Hrb`m5?a4g}GDv?b$FLC9C@Ytf03YN1y(r(AML=tK1Kv z&h?Rv@h9W?OT}<^DcoHQ^^`(A1*ykXV7&@F1^xzDH7n3&XgcDT+4oW-+o-t49;lJa zMLjiQJ(D&K3la-NqmhDWtbX~xZot!CJSRd{0z%vk^2IneGFqyX%Dlehs*%{(i-`bL z6kV1(ai9|ETo0@PJrrGVjxzv zeZ4;nP@Cb8KMtvw1>aBzK54jN1@8961l@1@(2Gb!TNU|dR>#Caka1 z56L9^?nujmEgW11HI-FDzFt0%%6%H?s0+x%1xV`AjOpoj8yWu;xa5R5p@hPMq#{~@ z;4Xs61e^g76WyFdLX#RA6T?t5<0xPzxjN7@qgr>gR4qyP=1%U9$-2o;$m%5k4F;xp zzSlLdi@OGP@v4E{4l!_Vg&4R(47_GFY;pj|$QW8omx^nzK!nIE`VVU~m8?4{h_Kk$N05=HBHt-%JD}8oR%$ z9YMeTkXF?P({qinZL{+A+< zc^an|WR;%qa(arCCht7r$+6tRU8ZjsmD=tA(go40V;jyRL18dJCc6a^&RZt*3UOJ?g&ThwXh(w60pzt6~iK zxB=J@nnBV8$xW1eyP`M&MOjE|*(AY(it<%fO;%@oilU_xih_^88d?p{<4dMz;NmrdTS(6j79E2D z9pe$w7s)Ji$O=qsyQhU+X~@;>CwL4d$eqBP2LOqIValkxfV$tJwiWtUMxhn@_tt-6 jHB$1Q*t}NspDX##t%w!zF2jJp2g85U@&6n$ONIF#s``m5 literal 0 HcmV?d00001 diff --git a/bb2gh/__pycache__/config.cpython-311.pyc b/bb2gh/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7d17299e5019b206bdf0f0b27c73e7ee6241b22 GIT binary patch literal 3192 zcmb_eO-vg{6rTODSp&uf8ppArF_1ZP5hW%l>F2Sn-KigJdBFo}DbvdniGXNv)-* z2QYf1&kUh7oglL1HmBpH=z7UCBS4x3RG&-=#ajB%X(Y2t^w-P`TB0a4gDALCXtM^l zf{##&smBFLCWZLQ2ZfP99nrE}=>Uh(jvuZlR9!$U^IZ#`Q-E6A#Yv1r&)MaAf zXu?|M&zUOy2HFO}Sf2^7ss64+4}5NV3!hsmHXHiP)%y%KTXh{qFjZ})4JpH0<;|?) z85%_X-}2UcK;GJ_HiedI3=o|~=P(VtX_fZ;k!fSf*g)1SwC($5`ZsT=Wf83VwA%mG zulBuuK{Oo|EdfCt=VzP_90BVJ zw-m?oyddF~Ru2?7zb7hT9LGv)QWh~E^)D?CUm}LK~$dmG@nD}yK z6cv`o{w2kT2tbdo*c29c3 z@u=t_;d=D!4-RUn^aKKDLSaw2t}ZMq91kX?1kbNn#?)%;u1t`%E^k2vSw+vy?Q6%r z_1*TB5M?@+J^j4HcGtbxp6}=@bo6D$N({`ZES0<>Lazy_&K=Zy^XAP#(1%?HHH_ok}+dPqX4i=n)Dx_xEe*60WVAp2P9=a=mr4~T7cHUXKwY0XF#d%9_ z!P1*yi`_jN*VeD)jt_73=DRNzx-SwB97TKA2Di>_Iv#$Nw~rL;BP0d@=yq+4ua9qf zpIpj!oiB8qC&EW^>MfJXxpwF@^BX(Vv_%ay1@%o47-XAKx6nVJ1Lq6sBXr%KrczW2 zEmF9nc1soL6!ks&Vb6B?4R!lS3eF$Za+anARz+c8!=v}17*b3_6k`nvY%MwpyAOqH zgrb%lleeG7WHn$?SXX+cBv~d$9xUESN>~(J+Mn-&l0l9{7hI|^=x{`bYz-@Y`*ieG zRMrM9f?m-grEI? z^^D!n!ww)T*2)5{RH)-b@=qYE=*3}o3DKrcD2-;uixyipn6vc2{oLlt^?0|x00MX3 zHeIky=d9Bu23dRSR#fK+qU}{Nyi{haXz9rIX409oiZHn?=XxgL&fBI6wyB(TiXe2? zE{)FP1mLO)aCINR#P(ROX9DiL?MlIRC1<^&0WejS9S8<_`zcJ~hr$QdDdvxgJg+qK zd^9d3Bc$KT^EZ7&-6`uVgxe_VTq-6crvNy40i%~)acHCG(nud1#uL`auqlnO8L2}m;W&S9$ z%Sa+9M+jViHG&u@0?0?5To=y4hZHc-i*5AO6Dbf7v48;sJ@rPzE)e9@_hy$XQkI(3 zL-OtXyqS6Pe!h9jzXSpu1j?&JPt$+wB;;S%Xcp0_y!-=H)`>=DiN{_jnkApEKgq_-PXL6o=2J%?pL3 zayDn`bdjbFU5Q`0sH8G!J!dLQG_km-)7Zap)>M?YJitC*NLoSrd3HIaFPiCmF6FGt z9?e}|ehkfZq7#st%<>vHD`@;JGAn9+O}fR+N}2~+9?c8XtNDQXG#RL@bpY+qIzhsK z<+ryw!z3z>|K1i!Nzi;Drzuxf>|y14o+?n?ah5ztpFjbPSgNU3lo9I!{9fJ!vQA7_ zWrUQt68}BduC5YMU5*l2aO)e*qrxiNr6jbQ0cPV~tds<3xsqrEj>w~8eaVEOs|A{| zx-$8>bWY7v)y&_}b5V)nWQ(7hw?cEe2?KM9T)N0kWeL!-d_dG(BCAJ5OEmP%b?U?i%qeyykVHg{-!f{dVr?Ql==pApToD&3T($=6P8iCZGe!e?ZOi+YQH zIz#n@sVfPE>Wg_JZRY8UlFljSysosmRji`}odwHP-7L_Y!B}vTl20z^DKn;AyuRI+ z$iTYVilQ&44bylRR^Oh)`qH^MC7;t{?y~35RncXwPE9vbG|f=N@&W>V2@+azqG0Bk z(-jA7p|xgoqZAX$urW4lr1AX4x16wN;ek8rfC$d#I-9Ym*Whpd-qy0dNeQlA_vhO|^ zHU17{jWjv}au}oI)K=u!X5?7Cr@wyiSiSFXec*6ICfy@1h}7Nn3Q76X|M*EDa_92x z%ay(l9%?^iAJ0_%mumh?75^nzqI~(sXTd`aBFf=yQV)i{S-yMX_E+Vvo`v?WcmBDv za_~2gyZ>at^UG0t4dSjK)&Trkn??)&N8P-`ecQ>_Sz(S*O3zXG!Ui{iQ6!vQV1 z;(_zJsU&i?x1ndD9c*WgMR{guItUv;-8GwmVknM%aBv5bHBF7@GWvk5k>|n4H_KZ= zWizO3bXS8%Yr&%x-%%Qd!IT^D&~RbuAvE6PP^+GjXOD{nz^o*eB)E>iGh5)yYqm(R zfnNJs(fE>gJ2OCFY+nNa%?s)j67NF{xwx=c$YemDUler1gop*CXbCPy$J^i#r*p=H z8`Bs9XbN@}O#pHXF{3I7idaXk~EP^&F``)Ua=F(OoD>7+qv8`w8EVovyft3ZGxW52IwWRBUmEJ zXo)BQ)M0~SpRG(b@5r!*S{ybIhyX?qgqUx-NQ4Tf1K524h~>!@vPqo+UZLw+va3(} zHu4-nGK2&jl^z0O@r%^J`ySPbg<+5taS#XB2M}WhEut|Ydvk8@T{xn!j&ThLU{fCn zb-Mx3Uk#4df}>@z9@w`P7~BjDZk&B6KKkIXRtXGN1D9)o%VqwVuj@|d?aoSf>>>Yy z{Am2KUiD4Ze3KR5WIedAEZecu>a8vyd<%QNQS68%aj)2EhYq}`cfa^|rfV3F3}}?N zra=Ux^4TmV!)6{DtpO5?iY*pp*98t+-Hv9{hEdQ}XL{6YqtOVQ)RGseYFpYiDH@WZ z@-45!Zs3j2?Af$vS3=WNnCCD647M7xeQ*LM8=nAahp|>G-ue9Y=Xb}ef#F(UxXjnX z-CN=4W;l9(s2U!tg~!U?dT@Wa>vfa4a5n+C0VWd+NbINI0LxeZ57R;Jx2N$S*K5my zIAUTrIAz0qIcoOc6LLL()}0w1o&z#G3{s~+hIee$)3)BOnV2TRdY+x@K{m#%dTx4f zUAxwjjY-AqRR?FO{$*NU(y3u1Zkvn!Z?JXdO(>I)cq#eoO|-`J9zoaGhRLZYhj8M| zC=P;e>l)y7b8GN!06G9HgsS53xkNex`Y!0q>Tz7(p3nrIZGf{g zEbmf+=HTXTjR_iq`SdsvHyY?+Xuci^A;;NWsfvTU#ezE=$5s9b1W?USLj4WG$&tE0 zTs~KCx#RZ^jt1|MyXyX~E&tG_f9U>7#XnT_kJtR;7614%f7s5+K77>w_+r&RRr60( znqNC56XdQ2X%gh0x;<47A%FaZzy~0|1Jd|;!V)k67QzG!Gtj>sk4MRqXvgUR;mL`E zr+b8_JrdM+Pq{vY32%h3e2HIhlQ@QpM=$}ycZLfyGgqeJzXC|@m1|cg**kcG+?q9C)gfmeWgYXvY5bD4_+ZDws7n}%ryy2oA!2bR5A@eMKngBNTX z0p6KtC%(&KWT>jup{m)uR>)x8uc}`Z5*cU2tEyT)rK%Kjb(YTCF8&TS-$!y13BK|E zm9QP`#2>MdB9Y-|l!2_dpSpOaBA#i8g7g-6Lz8rbCCJy?$p|kU19xZ=Ul3Vb^0p-P zHi%2QLK66nONImqUIJbxwvZ@VRHh%n!}8+W55%D5nPc||`SVM+MxT?eHTJ2KcPhJmpUVT~-kLmc e_f%CLs>wrZo`&bQ90!NkoxWiGul5*avi%oI2zxsK literal 0 HcmV?d00001 diff --git a/bb2gh/__pycache__/migrator.cpython-311.pyc b/bb2gh/__pycache__/migrator.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe4c4c1ba7d4a2ba8f9b49ea335e173aee2c63d3 GIT binary patch literal 7250 zcma)BU2NM{mcEooi4^ri+wxx$Cv@T@RuW5Xv(2V<;-vX;>?X}*>r6ZGU{))Ow5(W` zG$fTDRUQaIfNBZ@>j}Duv7NzcicYFQ^5BPkFc$leot+H&NE!qzaA3f|q7Qvjku4Bp zU-n#5q9i9Bl%@_Z?>+b2AKr7$caFdE`8)`&UHkr&{J&0w{s+G_E@!>)yam?6xkW5!`Voik3sDUd00k(!|h1Z67aT6E92&9W;+FETR>fgDIAZknTExgz_B zK2Ph{vrcub7fA3r_wZJbX6H-h`X3~6Sj+ex2 zMouajDJjbQoRnGQ&n1=FTzo-P_-RqPB}%-K;V&hX$=ob|F|CMFR!YiZ_**>a)o=0W zA;JjQozAC{Vp@SJFZ7wu&D!tOv=URqxWk&PN!>v47Crw0#K86t38H}VP_g7tOa_aT z?IpHUBEcaz%dOJv#)6}LV;%w5W1Eve3Ldw9-6R{Of}=nPB>YiNT#qQ|0YB?nau%F7 z?Ki=_V18C9+BQd5gS+)9&;=y8`|T=_V~<^??dq1oBPc(%-ygw9j~j;w?U*^!U~8Qn ztistmBCX>+f|WkDS9_0W<@+&nfdjni2@c7VhvX+lSqh?K?&k)E@6;2Gs(2!k#nFlZsIaCI&j_MH ziIS9&3>QobN(Ln>QZj3Fo{yzK>NTs^MVkd?12ZIJQbLxnj=^LX1;ZtYaxSIFc;k4u zUWOy{nME;@lSL^q+t8Ge5tuO?_Q>q)TZ#FIrC=E;oV{z%QJ7W~nxz2{;fJ666i5+m zvs~%$%7tHti|4mU&nH*zU0E3@JHMj8petR!_kHQp$goa^RWiKg>s|iGD$TBRm9MXn z>lbTIhu`%b0`k9@6ensfMEiAj-|D*??g7<3utod-+54$i?H;ZSRet&8@Y;!gKfQT; zV&nLPc6?GlKB>`DIz6S*Q(OK%-G8*|?bN*kt8#hb%l9$scTPvZ1EK?-8geimknZhV zKE28A-C*}>EU&Y?O7hQP4Jh8Km{l0Y}W7EYtv=#KLSx=j>_vaJkw{KnO;G#Ah>K3{Sa9A*2kH~8?gop;Hpu+7$j zv)ynAe=&F>_1g=aYgUra`&q zWPTXaNH&*BMUEUZt0Ie{ln_TG@s~MKR^-T$5UUKGN4TAt6` z2EMsJ5~gBZ{T6>@34h7F%MzLQw@b@=a#;bq1@4u1j0pLj3t~!CM4@F)*evAxCQK87 zU6f#Q;HjAggE93@>cB(w0s&KzO(m67GA+tr3UJwQro^;_RZ$Za+fsjlFnU% z!WP%_(EGrv4!pf4YuuR5jj87SVykOF?|K8K4BcBP?%;pmSNFZMeq7_u=-e5#al>+e z8Lx*y`%cu@<{IR^uvzyyyV1V{hMiv#!_NO2jhs8;_|uUi=X`JtW8&g z3v8W@?&E7FwNZrv{Iv~~b*mMHkS;0JaKETyqTE4!Ng z%UBy;xdf^}x9=E%XV9|dW*6Xu9dDx2C8of@IDc636g+~f#e%X0Pnm1>YnH&GvhAZS zc?-@m*5p^N0=<)0+W#ZgL!15Jw6ub&f=giXJ^-Jt<)48yKiC|s0IS8o3im2_3!X>t zeIMK381{my_oS9{V5JW=dx94m0^8iW!@*{2vji3&uVm+p+;zEbu+v}IcKWYv@?QVN z*aB5>72NHh56=|W&RKT;uNcfJ^CCY7W}TmH3k|{^4>oEO8HpET@p&+-*$kgdH-m#w zv;EC5e_j&7Qk!y2CI~AonbCsD@K%^V2c~o+p30UIHu)GUBW$Yu0NIiJjy4T(;$12p9^LB-VCuM=)o;pGW zhr|QKy(z{O?AQ!aPGR_)jxCCLetIF9&0>yN3O<8yw;tPBpgD3-w!Ca_(@WN;34)+C zf8u>R6vWHGE*8cRY$?a^i;Tw*1+#1+@=&*gchO*HXQQ(*S&ZhSltIIbl34(ciY=zW zCgxk(;Kgh+Tu{nL2?<+LgT9@S7NSB@G8`G%pci8cBD~0^-1?Gb!-sb;+IThHwLvD+ za~Z>9wwM7dgdCz`ut_;uU!Y(F8^x#$8w)#x?v(Mv0{wG6lnDz1KI#@LI3d{upe%TNF-@L0K~-@xx#+jhdg( zD&p@bgVljk)pA*2!q;}Fr0c--Cj8`06Gwa>M4tZD9=-o)<-;dYjXA9|r;F#dnC_K1 zy=S;`=t*S##DAXFm?@o^f_iVq%AwC9ns>kM-Cw-;4e7dndBycvpfaSA5si%KWCRM^ zfo?r8s0QlibKS9da&qJ3q;~Q>{p5RaO0|QR^@Ep-liS@r#jDj|ck%sdSFm_xyLVp= z1*!ceDX};JVLxTn;IYbrMvvjXR=qM@rOou)l^l208&LU8wf$ zS^e<0-qI8l{*C@uy4+Lg*ZPj@eaA~v+kxO_fZqu4<=vH}7C5N~PS%jq*N@=^`+!yV zj6Y?+zM*j+>D)&u_feG_D0gVw{uN)9>#LD&-~KIbs2be0damp!U#yI+Gf&_7T7WpG zqZ8tsj!uYkfB>x8SH_TP#lPYQM6(B~eK5NQfx$L8euLxH!O`{0U-xU=w9ZYd+;o-O zQzkTSaK(%NY7}(9ux0?3_5Bc{*iSCsyIiHuR_UHqU%Bud<)k2T0|H9`LL=|4)zg(@ zDtkgDPsor$LVq6GeG!q*(9w%w_p=cKXx#QT&}^qoo?3VZsv1|>Mmvof{1k`+S)g9^ zF0W6Y;KWM1|INAuvOs>000&T?qu-HBG(=Z0_Ny287g}&Vaqk>UX_S@fu$R_Q$p4`4MXh6!5BcAdOaB zDsVyv=1Gw4I|evrcXK5>_Y<&%uV3!Ir~Nj}EUiAsDPV~P~;$|mRtIStCJ zhCo`6UP$E*mOoyb(#Ub099PM4 z@OfRm4~HKNg9F;Hb&lwrBWmZI`sUo%KL>*Ep9@x@c)3ctKDm1D>dHHnqw7lXszzSW z$qOoZ;d}6#U7xveUI6@ebZxg9{t4Vq8S4pD>r*=fFd+mlwpV8JX2z-fj*qE{3)@$dr;qfOzpc4cje>F$k~m^SuJu-kDRmWp5CyYKuc5h_=hUY z+B+J3TBlE|^l6yE%Hh?M&&i(qph%s(0nCMh@!aph^zH4#I4@4s3KC zC{Jl!VZAF{yi#*HDc|2*LQA5@hZ8*(snXsOiPJnjzz=vNUuk}oTD?=*qxw#)PpRaD zMo#GD1QfPNuj)H$-D~|SIi`_gIynXfm=WdKBzrf=-c`1I!-jS^$!s2hk;=#Ga0Kdt zKo;C4?0B}j>tX~wiv%VJ=U)&3G(=ntbr|kwRLI1mQNxW>@SxSE2AL<_;OrbYoJ;(l zA1gBOS<2j~!x4dX-c8)kfXZC4viWp0iV<)$YH2J^lbLE|f?ewZ@ z86lSv$E92FioJ^b3E>FbjM6D47O}hJfMjKI}Jzxq*;R z_o`cJNp`Z#m8GYu>Q_~-s$aeLd(Z#iaybbo!^gjm{YNuF{1R?YK5Mq^|~!?Td~jDNy0*iY2rWd#N(Cpdt*1ed`6mYj799JD-wbTSQS=u2XcfgIao{g_#p%D`r9-6VF$ayrV`4&v zE)K>lrRK~weM5!?=Pib$I`sj?=kde8!Zc3^h5XulxNcG4Tw2YxV5zHH>*`dkzHa?1 zW9T|Q3ixbwZ5rD4T3cXh^}KnurYKMm9RKeTPJw+2dw*u`zp=G$Q;7Gh#R;whymrB@ zSbt9l9)$wxRcN3-#RAl?_c!VN0lmLju|a?ETi-gP*pWkFgcijPJx9K^rd-J$%gGsWL31K;H7Uj9B)YHJS5fk7abDJ}%i?{FlH#dF%@PwdTQnsvC6Pv@ zBoS$>5R+Ep(fiZU6;Y!SnA(4tSWYC@5*j;?L{Kqd6o^KoLl^np-in?J2UNSa*Ljh@ zkxZd^@jQR$EjgH`c@X+!+H&`F+BsMeMH)=o2Xzq#)1EH*txmLTvQSxDHPYN zi=rHIYL-=`+2)gph1jBIgXi!FVSA!cSQL$3MUj|*+_=YH*|BC@5~G5MPz&@*n3*@r z+Ybkqk}KjMKEA=Z>fH{97xn8oI5&4@aS1jYF~<-T9U6nH}a|zIxH44qe$Bn%W(jdbv>eGEyA+ zl{)mRqARMpqWW477alBZ-pyYsx=+D^nTaxEb%aa7qh%}nmI>V8yXqQ!AJiKAoWD$0 z+4hfh*EI&t)HOyf*BWITapF|jULB>cP__~@TjmJHQzl5dzr?r;{=U2&i18^heOcSS zKk%gIQBUsZmZj+LQ}MG8w@_xXM74TDhRN|v`R;$uI z05{8E(C-=Y!Vx0ut&ksxM#5OW}exwLCfd z=xlCk>u3Lb_8-q~kMCS5_KmB34wP_-AwL0cr4i(u>mF@T<~AGo%5H@+wz=oe)$f2-ghG(JUUFl#GO&g}eerZVIE_-63%$#ogO z1&T*_4-Geih7SE^l%Vp2SR^n^$V2^b5y-^@w_}vJdkr9hwH`|34-bSM48#s-D+S=h zgaYp#rBKSjQM5v%X3%rf>DMo((2d}YBsQ)q7@pDTp{eL3U>nELPKyRo6LtSI&`0nbf2I3CKnK14%#~6g_+OI-6F*RjiWdrJx(@v}&?kg!zEy%yh}t`b zy7|@)=z>MY7P?Y^kE*L<4b=r0GEdg<`(GZw#E_bY@+`pA$QdHyTIa+M8o=ZXIA4A?C)Ct zHnAbB5#LyE6KkZx!oAnlIfbi-_3JKZJ8Ce0-3@1rt^A>a=T}^f@}kZL$=-c3FiLSZ zjw1=|w+t_}F$ofIH3W@ai9G4SU0 zs&8(S+?e&zqSMbU@e)73;dkO3(1722S&-1 z%&$b3!4mWHOVPxlC>ax_KblTff+kUb*c>7xej%Ak2m|~~D#4%95qp*46u%Hdk~}bt zu(e7bxrUPSq9ox-ug&0bi0_sLyCseK;@a3Wgypn^!|-&+cr+G=>0!I^m?SqcBYbxu zMvKXDG2L}zIi~vyJ{pIJ2BJD6Qh>oYV(J#o!wLvnH_eXo2RN>CQ~=Bs0b%2o4&ND_ zxy{e~a{4ZROFxIo6m>K3r@JLSd=cgfrBBp^Yh>hO31d6ybFjAY3RXt0FyzyHwPvrVn`W7t(=3ZiXc)`qt`4n8x;-*h=IZ!7(_%iPdC855yUKabRi5fC|T8A@s3+VmR=O%%Nyv zehJV54rPsogyAy2G7ZJR3e6P}8xI8Cgb^3vE3JYqXNVu>IjlR9pMDzI+}jIzCk$ zpJpz-ax`x)sKL`)JA^U8As)^;@4$ScloK$gPM3p;mShKk%xmAhHsZkD(}79vn!e)Hx{nIW3m z%fwMfr%tk$06=ZZk1uD(aAocBWOi)d6WH^d-1VHyPj0vFj1@hXRL>;{R9&sFyul~z zBeu|S>BZQ~p`!P?>b+j@UN3o%=6ywP=ccRVJ+j}{k-Pne@D{7K{VY3Kg4V4+^yQb; zwlmqueGi~3n>kM@Fj5M%=7#cpTfcryTYdijC4j(6Kp4VzJb$9-J-O+INz0VO)%nWX zSqik}F6Axxv8@X`ju&TMTP=7H5I6=Pw%VXg_nTKu9e@K}u+b_3`S-m1u9q(bKHh15 z!M?oxjv0>T$)fkfCWrsZ4j2lj_WC0@upciHH2b^Bznv^GBPAx7bLAC~nzn(|v<;*N z!Vp|r?$a#@$A=5lu+#;b_kI775!&)6x@Tm7{!^F)`e(9jyo364NB;zC|4%0gbl!A{ zFaX0dZ2Erq9#j@Gn6~wE?a3RZh0L-!g)l6twn-6$tg;7{M<8%AdhB*ODk$Ka7H;kcRl)Vx3 zth)pT;FSwt*bM+So_w7=wXuRja2}A%t9S)=8M`$C4GEm$xksQ6aWN5UGNSG)Sdw30f?YWJ0+VZ)=z+QEZP5(wyMf6qAQ}z01d_$hnuMCGA`l3c zO2{aLd1o;%SAC%8b8(n2VKr~?&PGG`(P#pX#N;=a^)VP72;$8gE>zYwCLx*Y4{Y9k z9L}?&LpiYlE_grO9@^=C(f)F|NX@F$Y=N5n(ME9Xxih(Y zMYcy}d-Uw^9@n|cb(Z|?C3jQF?cAf< zcd7Q=PRv}IdVM_cY}(b1(kx-z!1EkHu{-y{Pr@8&YVBcFDqy9M2?~#2?u^ z7h>K{hId`=qt?dR!$QN+iX;M;!Nso)1CaU-WQQbFOPzraY z)T_ECNVqL?aKen5nrzZ&Y+g^!$kQn*C-fN_HT6_dUZANNsu@P)J8U){c3F}N)lEI2 zSrW=jDye0>2KEt|Gg8S|rGNIQj^)8PylxPUz?=xq2j=4XfJ_Q14^mVGkZv61WSi;+ zDXDEBJ*ouKt9n5CR4+)s>H`^2{V?)?C9;uc4qm{a8+C(bqmlKb9F@;ZoEw*?^|%&^ zIa7zR;G*+zKUiY}GeK6Ki<%{h(>hCXRcM{IsN8`23nvF! zh}9RBOpsgLc-U>ZQidfgM9o=Cm`mwN%R6ylrt3&xiF9w-~+XDv6W0n*5C`YW>yx~rt@j`fDfGG}_K^$3}B z7Cg%}_84=iV1H^8s2srZ6d5Fju;_Z9e}^n`9C?RuaH*V5bPl-a8n;BMEu@UFo8p%! zn!DxJ-q#HikW4LTNk9^;!x9j@TP_;J3fy!BE!v5V0w6NJ3IJHI3o?v#5U~2bz}kzK zhc69p_FWGbe0@bI5FGT{P!xsk2z-u0zi=hf_bw#-8g%dUy< zvDObLYI&`+hYugN%D&&xy}4_3IWIk7;F$l}HL{1lyGIy#vhD7E4rFWCy+8(ly^az! zP#ehLSX>V2vMm9~PK`$BRPMH`(H1#U4|;t|m2a%c2;!@Ktmaaol|!*CrwX^l8!)af zoUs7~S`r2=c?ZfsN!UGpABItU$M-!T>iGmN!iZkf97f>_tMhlL2#k`vYra380GmE{?Og8U zpGON_kz!ZmPFG$UE=a>gX&98PKqWY_k$@AJ2I}$FV8txHuCAG`0fcpCHPEOq}YkYCsT_QySDhW8M8dPk%GN65jDdc zGj{{&FEdlO1O+N*ycJsqeIJw=G6(=B*u9qh^!4jwgk9wOER(vN;HWtqp7g9`XHj2(_*GFuPs>~Y@`_R^o-@k%`k0PzG_`1 zYFeIn)G!reDH+3dN+Py)21WcX^e**MT448^uh_JL-(m7#ZmESSNzTJdDIZ-pG(_dIS>CwNmXe^O(((hRg%dz*qvYLhg4yk#9c) zzs)84xys+~Jy_~FP}(h*diQ$#f~1!=G-4d$!A?My@#GA8~N zlx83+Nzd<6+*0n*ooQ(;*N|pmwBL|BOxw3!Qh{xHjj% zfNg&Vq9wE?Upu(uLz}-Y?s_`!!(Zta$Nn+;x6!Z8etmZJUN zx`|(^B(QNnzjk!_IXO{(%~-F!Vg1#k6u z9)=-R@w9luq!jUlxDF$>I;)kYrW9}qx@!4nW>HI~6H^+cN1>X5ZVj!uYR{dNxhkwy z8CRy;#(rDDi`YI2;-`Xt>a8Zw-z)l$eDmrZ>8m%sexu+&@qh5Iw>lg)!~ZrY8)OCo z=LS@{8hSs)T2GSq2A5s9|J2&4{5;zk=u=s)^&SEB>-Q6vTF0MhS0hh2vUWI&0k}K! z%6ZL9)1(}SkXueo%g@&N`F^%JFh3u*%{U%+95yR6BDOUTz{Dw@cWOviY};~QjKYn(r8WSPF(#1CMarxFIzeKA(MrySW(=>PVU#*#*74)WK;<9Mg`O;>OmK9s~qw>xDT*MQCkQuC7RAWJl>C<`) z9T!G8(Kywj>QYmrQ(I$-POQpX9;il-V@9(Nqv&Rt!?L_~AcX(oY;#`ifB`zLN(`0h za*(OZJqukKZ-QtIP92x~?}hqrPJDTw5E?3mhE}~>0Nk+;m)Dka-p||r)P8gM&Qw7j zDas@H;7DE?VJMD=+wngNJo_k$E$~GPYqj3Sq4MPX#(G`_$qAqKvUfl_Hw-2ns(6wjSqD!@~z#qaUT0 zZ48F#dC0?E#P|dRoB%nlMEdij|32}pu&YGcSJ+h|o)vbL$TRs?zx$p&tKCJ=ehIad%Nt1M^s^h{~rN!ptAq~ literal 0 HcmV?d00001 diff --git a/bb2gh/__pycache__/syncer.cpython-311.pyc b/bb2gh/__pycache__/syncer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f715e89d0145717e088bdbaba7e82c532c60eed GIT binary patch literal 6934 zcma(VU5pdQdG^m<+iQF8{|G->0te?DXNRMOvb%<0 z`KnVCwGMKXOGLy;Z6zn7geZBqhd%Jo2f|Yx*JvfIl}lKJaSGe6Jp3(Gt|J-EA(@itEHy`wFFi+- zFEhuGFFVJQFE_`@jLc>|Iq#g8LNrpSCEl4-@!g~l`T}0N%=u-X%)Lj=1!T{AXilKf z43fR?A(_8P10K9>yr4{&Nt3iI`INMv7IMP z{q@p(^jkbK{w-cALRs8-J*g`xcS30O!pp$>@KeBn4N_^Ug38daOzQxhQg?}UTJTq< zWv1Reo7-8Td$-uT5en&e-F8`W4jEPG3MI4f5ML=3b|Y|FH87U6}_b9B~6zLB|T~}&|eM! zYCe-sPfKH(MM)P?{B>}j_l5VlO@{VqCsYNAqQN|EVb(MDsN*gQPy-aQNj0sh_-cUIoNReiMJr`>4Ii8Y<-hx4i-nvL zD`|=vn|GLgauO(BEX~K}=cm$(F)S8}8ZBP2_yiD@fKGKQOaKCov^xM+(1UMXn7DHGQFc~uiW!)H@w?7_|ON!Rlyh_UI-<@P*<2tUWa z#iy&{BP@Gj#ZopKD~Dn^MNKP!^>#_o^s>KHltD@3QDu5kj(7;$s$o2E2mr`iF`Ln| znY^NbJmI>Bh(wS4^rp7Ez!aYf>yqlKKpuvOkf^ zD#?7p-W$xYpsC}~u^nFRp8!@+i)Fa*M&FjT>9u{g;zr-$M&Dtx?{L*q^*q`L44HxP z7NWRt2Q>K*?rbs8`smRkVB)@!`{K@f;?9P+%M^FDkdGT8pvpGI!R!8y{2vO}getcY z2vtk(PuHf`sH;b+N1DQ5Gc>UJN^P;qGy?;xTJ>o4sIeouzRTG0V|ZFL6CBzQM}aK% zR?65ibvtB;GYxUZ6lb6U&!dfj?dHG>Kw1zMbO5Zm$^v!4AxxbYJ$tYfa46Dtz@Fdj zA3`68$C=NlaptS(!DC_iZa92wCwEt(0Hy%LEsmT$JP1wKk%GWGK-e*UQEatVtn8TZF_*NB3PwNI-ks1 z-t)6F7mlAhZG{qxU`n$}qTS(vj#-$JwpdNcE~xupLJf}9&HRwm_rlL;BSuN_TV>ss22Qq;aa{fn|oer@Y5zg zZSd1zLr8P3e!(1jvBAG&@-G?Aqi%zlQ|`^cAG}v`*Ka`OI_mL@9QK5sPJ!E`Hs6Iz z!wtIzfSf23FUQU)DJ63Wn9n{IX-wWHX^SNtT+p;67cS=|V9u1X0Q&4oG#Wh-VN?MY zve-<1p`gZ~3N1gvE#$;YIg49RlQ~7h1UpJdoq|8*A?Fx-F5)nBY0m*zLD;kgt_F<3 z<99AK__)c(4L;r+-HuPP3P1Z~x_9iJj0aku@?=m)9+uow-h5iXjd*TS)aeSUyZ))u ziod<)jzWOv%0)LG%;uha*P1|cCs4qnJ8P~_KgBI?^1~@&&LeEOAA6DdqGHDn*+QWh zEssNZppr`k0qTh_#xr?cQ7+Q}SU?37jMXcoZI}<#7E#QxHFhUyV0}ZKklz5Vqm}C$FAdWB(X1NB1{`DN~p- zgsB$S8S-x)`NB4LjWzf^CcnpU9-FP)3j%7eQX&HPoXu8v2ko-tR5o3Hg|Y)HxI`CQ z&2@WePaH|N{h+xXY?q$ya0&#Q?!kHjHDyrMy)50U`U$gj=-m1=b5`1Pu3x~BYWEvq z%7e3*8Prly&&+3m@$mgei8B<(Uca6JM+hqm@M>_E#pE)1M{#&H1$a^h_eBBNqc>iL zA?2W>H^|VF(4_qy0I*D`zm|FbV(sGEOe3@lTnP9N3_|&){>NlAvL372Dr|-8cfR~N!K)XOZu^Y^NH??jc z?%?3B%#nHFj=RHd@c!+2Ar7hi-0{wPW^>(yp4mJ&9N!$olRS#IJc_wS0(G>>zlxR* z#0#6+q=iB$FGnS;H}{el(?QrPEo4+pkH$&lYbS1cb1BeZNOtzmNo zRvHS47Sx}>|FVd;0s*$G(a%Dw_74CMSc`+#{P)Gld*Wn6jG1Dr$~D7-SSt@~2>n+RhIHtDaQ|N(NvGI@kp^#Nqp*bWfCwori93 zy_0E(XH4;oA)e_Pw(4ndFbG0|E@N=?&hXd64gQSDpD~xZp=72Nttg#IcracqRU zJ4^xW@yT5A_JeTsdLA7E@Pzys>i7U1i(jGs02#Up4Uqm5+benyNCQPq>6-uXqPr>3 z*0y(xYW?wo2|2jlHNl%;&m+ChSrvSKR|q$$DpnupV5LFGKZWcUWclD^Itflt4>-ha zm(9rl=3yPjv+!WgFBUy#6oPtF|u)#)gc5mbug!^QylrQME ze!_{mc!U858&3uq05y}&Jjq+*#+vA<^LEXHw-s4>w_YS_wx z7<%j|w)K&E8so(QSU!@$Aju0z6@oKrd6jIks44RIw9pLTm-~9O5I)81%8a(zpKKWy@c z4gRnj2Tk01>vrVM&l>z`lRs@Z5AikP5kEw`i9Cc_iG<}(Byt701kr865)z5GOUZ1z z$D2sVg;XM;{us`U#9_nVLvFJ9|MEYv5Aukb zf;Q}^X67M1d9Zdw@ob-sgeund*q? vduHv$hHtm&+r8pxc?4=_3-y8>A_X}u7wm~s6h5XWf4;N({`CVWWT*cJ8c*Qq literal 0 HcmV?d00001 diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py new file mode 100644 index 0000000..2d11577 --- /dev/null +++ b/bb2gh/bitbucket_client.py @@ -0,0 +1,82 @@ +"""Bitbucket Server REST API client.""" + +import logging +import requests + +logger = logging.getLogger(__name__) + + +class BitbucketClient: + """Client for Bitbucket Server (Data Center) REST API v1.0.""" + + def __init__(self, base_url, token): + self.base_url = base_url.rstrip("/") + self.api_url = f"{self.base_url}/rest/api/1.0" + self.session = requests.Session() + if token: + self.session.headers["Authorization"] = f"Bearer {token}" + + def _paginate(self, url, params=None): + """Iterate through all pages of a Bitbucket paginated endpoint.""" + params = dict(params or {}) + params.setdefault("limit", 25) + + while True: + resp = self.session.get(url, params=params) + resp.raise_for_status() + data = resp.json() + + yield from data.get("values", []) + + if data.get("isLastPage", True): + break + params["start"] = data["nextPageStart"] + + def list_projects(self): + """List all projects on the Bitbucket instance.""" + url = f"{self.api_url}/projects" + return list(self._paginate(url)) + + def list_repos(self, project_key): + """List all repositories in a project.""" + url = f"{self.api_url}/projects/{project_key}/repos" + return list(self._paginate(url)) + + def list_pull_requests(self, project_key, repo_slug, state="OPEN"): + """List pull requests for a repository. + + Args: + state: OPEN, DECLINED, MERGED, or ALL + """ + url = f"{self.api_url}/projects/{project_key}/repos/{repo_slug}/pull-requests" + return list(self._paginate(url, params={"state": state})) + + def get_pr_activities(self, project_key, repo_slug, pr_id): + """Get activities (comments, approvals, etc.) for a pull request.""" + url = ( + f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" + f"/pull-requests/{pr_id}/activities" + ) + return list(self._paginate(url)) + + def get_pr_diff(self, project_key, repo_slug, pr_id): + """Get the diff for a pull request.""" + url = ( + f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" + f"/pull-requests/{pr_id}/diff" + ) + resp = self.session.get(url) + resp.raise_for_status() + return resp.json() + + def get_repo_clone_url(self, repo, protocol="ssh"): + """Extract clone URL from a repo object. + + Args: + repo: Repo dict from the Bitbucket API. + protocol: 'ssh' or 'http'. + """ + for link in repo.get("links", {}).get("clone", []): + if link.get("name") == protocol: + return link["href"] + return None diff --git a/bb2gh/cli.py b/bb2gh/cli.py new file mode 100644 index 0000000..1ea61e2 --- /dev/null +++ b/bb2gh/cli.py @@ -0,0 +1,83 @@ +"""CLI entry point for bb2gh migration tool.""" + +import logging +import sys + +import click + +from .config import Config +from .migrator import migrate_repos +from .pr_migrator import migrate_pull_requests +from .syncer import Syncer + + +def _setup_logging(verbose): + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + +@click.group() +@click.option("--config", "config_path", default="config.yaml", help="Path to config file.") +@click.option("-v", "--verbose", is_flag=True, help="Enable debug logging.") +@click.pass_context +def cli(ctx, config_path, verbose): + """bb2gh - Bitbucket Server to GitHub Enterprise migration tool.""" + _setup_logging(verbose) + ctx.ensure_object(dict) + try: + ctx.obj["config"] = Config(config_path) + except Exception as e: + click.echo(f"Error loading config: {e}", err=True) + sys.exit(1) + + +@cli.command() +@click.pass_context +def migrate(ctx): + """Bulk migrate all repositories from Bitbucket to GitHub. + + Clones repos via SSH, creates them on GitHub, and pushes all + branches, tags, and history. + """ + config = ctx.obj["config"] + migrated, skipped, failed = migrate_repos(config) + click.echo(f"\nMigration complete: {migrated} migrated, {skipped} skipped, {failed} failed") + if failed > 0: + sys.exit(1) + + +@cli.command() +@click.pass_context +def sync(ctx): + """Continuously sync repos from Bitbucket to GitHub. + + Fetches changes from Bitbucket and pushes to GitHub every N seconds + (configured via sync.interval_seconds). Runs until interrupted. + """ + config = ctx.obj["config"] + syncer = Syncer(config) + syncer.run() + + +@cli.command("migrate-prs") +@click.option("--dry-run", is_flag=True, help="Log what would be done without making changes.") +@click.pass_context +def migrate_prs(ctx, dry_run): + """Migrate open pull requests from Bitbucket to GitHub. + + Creates matching PRs on GitHub with title, description, comments, + and reviewer assignments. + """ + config = ctx.obj["config"] + migrated, skipped, failed = migrate_pull_requests(config, dry_run=dry_run) + click.echo(f"\nPR migration complete: {migrated} migrated, {skipped} skipped, {failed} failed") + if failed > 0: + sys.exit(1) + + +if __name__ == "__main__": + cli() diff --git a/bb2gh/config.py b/bb2gh/config.py new file mode 100644 index 0000000..8388141 --- /dev/null +++ b/bb2gh/config.py @@ -0,0 +1,52 @@ +"""Configuration loading and validation.""" + +import os +import yaml + + +class Config: + """Loads and validates migration configuration from a YAML file.""" + + def __init__(self, path="config.yaml"): + with open(path) as f: + raw = yaml.safe_load(f) + + self._validate(raw) + self._raw = raw + + # Bitbucket settings + bb = raw["bitbucket"] + self.bb_base_url = bb["base_url"].rstrip("/") + self.bb_token = bb.get("token") or os.environ.get("BB_TOKEN", "") + self.bb_ssh_url = bb["ssh_url"].rstrip("/") + self.bb_projects = bb.get("projects") # None means all projects + + # GitHub settings + gh = raw["github"] + self.gh_base_url = gh["base_url"].rstrip("/") + self.gh_token = gh.get("token") or os.environ.get("GH_TOKEN", "") + self.gh_org = gh["org"] + + # Sync settings + sync = raw.get("sync", {}) + self.sync_interval = sync.get("interval_seconds", 60) + self.work_dir = sync.get("work_dir", "/data/mirror") + + # User mapping (Bitbucket username -> GitHub username) + self.user_mapping = raw.get("user_mapping", {}) + + @staticmethod + def _validate(raw): + for section in ("bitbucket", "github"): + if section not in raw: + raise ValueError(f"Missing required config section: {section}") + + bb = raw["bitbucket"] + for key in ("base_url", "ssh_url"): + if key not in bb: + raise ValueError(f"Missing required bitbucket config: {key}") + + gh = raw["github"] + for key in ("base_url", "org"): + if key not in gh: + raise ValueError(f"Missing required github config: {key}") diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py new file mode 100644 index 0000000..2018d8b --- /dev/null +++ b/bb2gh/github_client.py @@ -0,0 +1,86 @@ +"""GitHub Enterprise API client wrapper.""" + +import logging +from github import Github, GithubException + +logger = logging.getLogger(__name__) + + +class GithubClient: + """Wrapper around PyGithub for GitHub Enterprise operations.""" + + def __init__(self, base_url, token, org_name): + self.gh = Github(base_url=base_url, login_or_token=token) + self.org = self.gh.get_organization(org_name) + self.org_name = org_name + + def create_repo(self, name, description="", private=True): + """Create a repository in the organization. + + Returns the repo object. If the repo already exists, returns the existing one. + """ + try: + repo = self.org.create_repo( + name=name, + description=description, + private=private, + auto_init=False, + ) + logger.info("Created GitHub repo: %s/%s", self.org_name, name) + return repo + except GithubException as e: + if e.status == 422: # Already exists + logger.info("GitHub repo already exists: %s/%s", self.org_name, name) + return self.org.get_repo(name) + raise + + def get_repo(self, name): + """Get an existing repository.""" + return self.org.get_repo(name) + + def create_pull_request(self, repo_name, title, body, head, base): + """Create a pull request on a GitHub repository. + + Args: + repo_name: Repository name. + title: PR title. + body: PR body/description (markdown). + head: Source branch name. + base: Target branch name. + + Returns the created PR object. + """ + repo = self.org.get_repo(repo_name) + pr = repo.create_pull(title=title, body=body, head=head, base=base) + logger.info("Created PR #%d on %s: %s", pr.number, repo_name, title) + return pr + + def add_pr_comment(self, repo_name, pr_number, body): + """Add a comment to a pull request.""" + repo = self.org.get_repo(repo_name) + pr = repo.get_pull(pr_number) + comment = pr.create_issue_comment(body) + return comment + + def add_pr_reviewers(self, repo_name, pr_number, reviewers): + """Request reviewers on a pull request. + + Args: + reviewers: List of GitHub usernames. + """ + if not reviewers: + return + repo = self.org.get_repo(repo_name) + pr = repo.get_pull(pr_number) + try: + pr.create_review_request(reviewers=reviewers) + logger.info("Added reviewers to PR #%d: %s", pr_number, reviewers) + except GithubException as e: + logger.warning( + "Failed to add reviewers to PR #%d: %s", pr_number, e + ) + + def get_clone_url(self, repo_name): + """Get the HTTPS clone URL for a repo.""" + repo = self.org.get_repo(repo_name) + return repo.clone_url diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py new file mode 100644 index 0000000..3409bad --- /dev/null +++ b/bb2gh/migrator.py @@ -0,0 +1,141 @@ +"""Bulk migration of repositories from Bitbucket Server to GitHub Enterprise.""" + +import logging +import os +import subprocess + +from .bitbucket_client import BitbucketClient +from .github_client import GithubClient +from .state import State + +logger = logging.getLogger(__name__) + + +def _run_git(args, cwd=None): + """Run a git command and return stdout.""" + cmd = ["git"] + args + logger.debug("Running: %s", " ".join(cmd)) + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + logger.error("git %s failed: %s", args[0], result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def _clean_hidden_refs(bare_repo_path): + """Remove hidden refs (refs/pull/*, refs/merge-requests/*) that can't be pushed.""" + try: + output = _run_git(["show-ref"], cwd=bare_repo_path) + except subprocess.CalledProcessError: + return # No refs to clean + + for line in output.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + ref = parts[1] + if "/pull/" in ref or "/merge-request" in ref: + try: + _run_git(["update-ref", "-d", ref], cwd=bare_repo_path) + logger.debug("Deleted hidden ref: %s", ref) + except subprocess.CalledProcessError: + logger.warning("Failed to delete ref: %s", ref) + + +def migrate_repos(config): + """Run the full bulk migration. + + For each repo in Bitbucket: + 1. Create the repo on GitHub + 2. Bare-clone from Bitbucket via SSH + 3. Clean hidden refs + 4. Push --mirror to GitHub + 5. Record in state + """ + bb = BitbucketClient(config.bb_base_url, config.bb_token) + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + state = State(config.work_dir) + + os.makedirs(config.work_dir, exist_ok=True) + + projects = config.bb_projects or [p["key"] for p in bb.list_projects()] + + total_migrated = 0 + total_skipped = 0 + total_failed = 0 + + for project_key in projects: + logger.info("Processing project: %s", project_key) + repos = bb.list_repos(project_key) + + for repo in repos: + repo_slug = repo["slug"] + repo_name = repo.get("name", repo_slug) + + if state.is_migrated(project_key, repo_slug): + logger.info("Skipping already migrated: %s/%s", project_key, repo_slug) + total_skipped += 1 + continue + + try: + _migrate_single_repo( + config, bb, gh, state, project_key, repo_slug, repo_name, repo + ) + total_migrated += 1 + except Exception: + logger.exception("Failed to migrate %s/%s", project_key, repo_slug) + total_failed += 1 + + logger.info( + "Migration complete: %d migrated, %d skipped, %d failed", + total_migrated, total_skipped, total_failed, + ) + return total_migrated, total_skipped, total_failed + + +def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_name, repo): + """Migrate a single repository.""" + logger.info("Migrating %s/%s ...", project_key, repo_slug) + + # 1. Create repo on GitHub + description = repo.get("description", "") or f"Migrated from Bitbucket: {project_key}/{repo_slug}" + gh.create_repo(repo_slug, description=description, private=True) + + # 2. Bare clone from Bitbucket + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + + if os.path.exists(bare_path): + # Already cloned, fetch latest + logger.info("Bare clone exists, fetching latest: %s", bare_path) + _run_git(["fetch", "origin", "--prune"], cwd=bare_path) + else: + clone_url = bb.get_repo_clone_url(repo, protocol="ssh") + if not clone_url: + # Fallback: construct SSH URL from config + clone_url = f"{config.bb_ssh_url}/{project_key.lower()}/{repo_slug}.git" + + logger.info("Cloning %s -> %s", clone_url, bare_path) + _run_git(["clone", "--bare", clone_url, bare_path]) + + # 3. Clean hidden refs + _clean_hidden_refs(bare_path) + + # 4. Add GitHub remote and push + gh_clone_url = gh.get_clone_url(repo_slug) + + # Remove existing github remote if present, then add + try: + _run_git(["remote", "remove", "github"], cwd=bare_path) + except subprocess.CalledProcessError: + pass # Remote didn't exist + + _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) + _run_git(["push", "--mirror", "github"], cwd=bare_path) + + # 5. Record in state + state.mark_migrated(project_key, repo_slug) + logger.info("Successfully migrated %s/%s", project_key, repo_slug) diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py new file mode 100644 index 0000000..3387a7e --- /dev/null +++ b/bb2gh/pr_migrator.py @@ -0,0 +1,169 @@ +"""Migrate pull requests from Bitbucket Server to GitHub Enterprise.""" + +import logging + +from .bitbucket_client import BitbucketClient +from .github_client import GithubClient +from .state import State + +logger = logging.getLogger(__name__) + + +def _format_pr_body(pr, config): + """Format the GitHub PR body with migration metadata.""" + bb_url = config.bb_base_url + project = pr["toRef"]["repository"]["project"]["key"] + repo = pr["toRef"]["repository"]["slug"] + pr_id = pr["id"] + author = pr["author"]["user"].get("displayName", pr["author"]["user"].get("name", "Unknown")) + created = pr.get("createdDate", "") + + # Build metadata header + header = ( + f"> **Migrated from Bitbucket**\n" + f"> Source: [{project}/{repo} PR #{pr_id}]" + f"({bb_url}/projects/{project}/repos/{repo}/pull-requests/{pr_id})\n" + f"> Original author: **{author}**\n" + ) + if created: + header += f"> Created: {created}\n" + + description = pr.get("description", "") or "" + return f"{header}\n---\n\n{description}" + + +def _format_comment(activity, config): + """Format a Bitbucket comment as a GitHub PR comment.""" + comment = activity.get("comment", {}) + user = comment.get("author", {}) + display_name = user.get("displayName", user.get("name", "Unknown")) + text = comment.get("text", "") + created = comment.get("createdDate", "") + + header = f"**{display_name}** commented" + if created: + header += f" (originally at {created})" + header += ":" + + return f"{header}\n\n{text}" + + +def _map_reviewers(pr, config): + """Map Bitbucket reviewer usernames to GitHub usernames.""" + reviewers = [] + for reviewer in pr.get("reviewers", []): + bb_username = reviewer["user"].get("name", "") + gh_username = config.user_mapping.get(bb_username, bb_username) + if gh_username: + reviewers.append(gh_username) + return reviewers + + +def migrate_pull_requests(config, dry_run=False): + """Migrate open pull requests from Bitbucket to GitHub. + + Args: + config: Config object. + dry_run: If True, log what would be done without making changes. + """ + bb = BitbucketClient(config.bb_base_url, config.bb_token) + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + state = State(config.work_dir) + + migrated_repos = state.get_migrated_repos() + if not migrated_repos: + logger.warning("No migrated repos found. Run 'bb2gh migrate' first.") + return + + total_migrated = 0 + total_skipped = 0 + total_failed = 0 + + for project_key, repo_slug in migrated_repos: + logger.info("Processing PRs for %s/%s", project_key, repo_slug) + + try: + open_prs = bb.list_pull_requests(project_key, repo_slug, state="OPEN") + except Exception: + logger.exception("Failed to list PRs for %s/%s", project_key, repo_slug) + continue + + for pr in open_prs: + pr_id = pr["id"] + title = pr["title"] + + if state.is_pr_migrated(project_key, repo_slug, pr_id): + logger.info("Skipping already migrated PR #%d: %s", pr_id, title) + total_skipped += 1 + continue + + head_branch = pr["fromRef"]["displayId"] + base_branch = pr["toRef"]["displayId"] + + if dry_run: + logger.info( + "[DRY RUN] Would migrate PR #%d: %s (%s -> %s)", + pr_id, title, head_branch, base_branch, + ) + total_migrated += 1 + continue + + try: + _migrate_single_pr( + config, bb, gh, state, project_key, repo_slug, pr + ) + total_migrated += 1 + except Exception: + logger.exception( + "Failed to migrate PR #%d in %s/%s", pr_id, project_key, repo_slug + ) + total_failed += 1 + + logger.info( + "PR migration complete: %d migrated, %d skipped, %d failed", + total_migrated, total_skipped, total_failed, + ) + return total_migrated, total_skipped, total_failed + + +def _migrate_single_pr(config, bb, gh, state, project_key, repo_slug, pr): + """Migrate a single pull request.""" + pr_id = pr["id"] + title = pr["title"] + head_branch = pr["fromRef"]["displayId"] + base_branch = pr["toRef"]["displayId"] + + logger.info("Migrating PR #%d: %s (%s -> %s)", pr_id, title, head_branch, base_branch) + + # Create PR on GitHub + body = _format_pr_body(pr, config) + gh_pr = gh.create_pull_request( + repo_name=repo_slug, + title=title, + body=body, + head=head_branch, + base=base_branch, + ) + + # Migrate comments + activities = bb.get_pr_activities(project_key, repo_slug, pr_id) + comment_count = 0 + for activity in activities: + action = activity.get("action", "") + if action == "COMMENTED" and "comment" in activity: + comment_body = _format_comment(activity, config) + gh.add_pr_comment(repo_slug, gh_pr.number, comment_body) + comment_count += 1 + + # Assign reviewers (best effort) + reviewers = _map_reviewers(pr, config) + if reviewers: + gh.add_pr_reviewers(repo_slug, gh_pr.number, reviewers) + + # Record mapping + state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) + + logger.info( + "Migrated PR #%d -> GitHub PR #%d (%d comments, %d reviewers)", + pr_id, gh_pr.number, comment_count, len(reviewers), + ) diff --git a/bb2gh/state.py b/bb2gh/state.py new file mode 100644 index 0000000..c647114 --- /dev/null +++ b/bb2gh/state.py @@ -0,0 +1,79 @@ +"""State tracking for migration progress.""" + +import json +import logging +import os +from datetime import datetime, timezone + +logger = logging.getLogger(__name__) + +STATE_FILE = "state.json" + + +class State: + """Tracks migration state in a JSON file.""" + + def __init__(self, work_dir): + self.path = os.path.join(work_dir, STATE_FILE) + self._data = self._load() + + def _load(self): + if os.path.exists(self.path): + with open(self.path) as f: + return json.load(f) + return {"repos": {}} + + def _save(self): + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, "w") as f: + json.dump(self._data, f, indent=2) + + def _now(self): + return datetime.now(timezone.utc).isoformat() + + def mark_migrated(self, project_key, repo_slug): + """Record that a repo has been migrated.""" + key = f"{project_key}/{repo_slug}" + self._data["repos"][key] = { + "project_key": project_key, + "repo_slug": repo_slug, + "status": "migrated", + "migrated_at": self._now(), + "last_sync": self._now(), + "pr_mappings": {}, + } + self._save() + logger.info("Marked %s as migrated", key) + + def update_sync_time(self, project_key, repo_slug): + """Update the last sync timestamp for a repo.""" + key = f"{project_key}/{repo_slug}" + if key in self._data["repos"]: + self._data["repos"][key]["last_sync"] = self._now() + self._save() + + def record_pr_mapping(self, project_key, repo_slug, bb_pr_id, gh_pr_number): + """Record the mapping between a Bitbucket PR and GitHub PR.""" + key = f"{project_key}/{repo_slug}" + if key in self._data["repos"]: + self._data["repos"][key]["pr_mappings"][str(bb_pr_id)] = gh_pr_number + self._save() + + def get_migrated_repos(self): + """Return list of (project_key, repo_slug) for all migrated repos.""" + result = [] + for entry in self._data["repos"].values(): + if entry["status"] == "migrated": + result.append((entry["project_key"], entry["repo_slug"])) + return result + + def is_migrated(self, project_key, repo_slug): + """Check if a repo has been migrated.""" + key = f"{project_key}/{repo_slug}" + return key in self._data["repos"] + + def is_pr_migrated(self, project_key, repo_slug, bb_pr_id): + """Check if a specific PR has already been migrated.""" + key = f"{project_key}/{repo_slug}" + repo_state = self._data["repos"].get(key, {}) + return str(bb_pr_id) in repo_state.get("pr_mappings", {}) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py new file mode 100644 index 0000000..de4e90a --- /dev/null +++ b/bb2gh/syncer.py @@ -0,0 +1,126 @@ +"""Continuous sync from Bitbucket to GitHub.""" + +import logging +import os +import signal +import subprocess +import time + +from .state import State + +logger = logging.getLogger(__name__) + + +def _run_git(args, cwd=None): + """Run a git command and return stdout.""" + cmd = ["git"] + args + logger.debug("Running: %s", " ".join(cmd)) + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + logger.error("git %s failed: %s", args[0], result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def _clean_hidden_refs(bare_repo_path): + """Remove hidden refs that can't be pushed to GitHub.""" + try: + output = _run_git(["show-ref"], cwd=bare_repo_path) + except subprocess.CalledProcessError: + return + + for line in output.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + ref = parts[1] + if "/pull/" in ref or "/merge-request" in ref: + try: + _run_git(["update-ref", "-d", ref], cwd=bare_repo_path) + except subprocess.CalledProcessError: + pass + + +class Syncer: + """Continuously syncs migrated repos from Bitbucket to GitHub.""" + + def __init__(self, config): + self.config = config + self.state = State(config.work_dir) + self._running = True + + signal.signal(signal.SIGTERM, self._handle_signal) + signal.signal(signal.SIGINT, self._handle_signal) + + def _handle_signal(self, signum, frame): + logger.info("Received signal %d, shutting down gracefully...", signum) + self._running = False + + def run(self): + """Run the sync loop.""" + logger.info( + "Starting continuous sync (interval: %ds)", self.config.sync_interval + ) + + while self._running: + self._sync_all() + self._sleep(self.config.sync_interval) + + logger.info("Syncer stopped.") + + def _sleep(self, seconds): + """Interruptible sleep.""" + end = time.time() + seconds + while self._running and time.time() < end: + time.sleep(min(1, end - time.time())) + + def _sync_all(self): + """Sync all migrated repos.""" + repos = self.state.get_migrated_repos() + if not repos: + logger.warning("No migrated repos found. Run 'bb2gh migrate' first.") + return + + synced = 0 + failed = 0 + + for project_key, repo_slug in repos: + if not self._running: + break + try: + self._sync_repo(project_key, repo_slug) + synced += 1 + except Exception: + logger.exception("Failed to sync %s/%s", project_key, repo_slug) + failed += 1 + + logger.info("Sync cycle complete: %d synced, %d failed", synced, failed) + + def _sync_repo(self, project_key, repo_slug): + """Sync a single repo: fetch from Bitbucket, push to GitHub.""" + bare_path = os.path.join( + self.config.work_dir, f"{project_key}__{repo_slug}.git" + ) + + if not os.path.exists(bare_path): + logger.error("Bare repo not found: %s", bare_path) + return + + start = time.time() + + # Fetch from Bitbucket (origin) + _run_git(["fetch", "origin", "--prune"], cwd=bare_path) + + # Clean hidden refs before pushing + _clean_hidden_refs(bare_path) + + # Push to GitHub + _run_git(["push", "github", "--mirror"], cwd=bare_path) + + elapsed = time.time() - start + self.state.update_sync_time(project_key, repo_slug) + logger.info("Synced %s/%s in %.1fs", project_key, repo_slug, elapsed) diff --git a/config.yaml.example b/config.yaml.example new file mode 100644 index 0000000..4eccded --- /dev/null +++ b/config.yaml.example @@ -0,0 +1,31 @@ +bitbucket: + # Bitbucket Server base URL (no trailing slash) + base_url: "https://bitbucket.mycompany.com" + # Personal access token for REST API calls + token: "YOUR_BITBUCKET_TOKEN" + # SSH base URL for git clone operations + ssh_url: "ssh://git@bitbucket.mycompany.com:7999" + # Optional: limit migration to specific projects (omit to migrate all) + # projects: + # - PROJ1 + # - PROJ2 + +github: + # GitHub Enterprise API base URL + base_url: "https://github.mycompany.com/api/v3" + # Personal access token with repo + admin:org scopes + token: "YOUR_GITHUB_TOKEN" + # Target organization on GitHub + org: "my-org" + +sync: + # Sync interval in seconds + interval_seconds: 60 + # Local directory for bare repo clones + work_dir: "/data/mirror" + +# Optional: map Bitbucket usernames to GitHub usernames +# Used for PR reviewer assignments +# user_mapping: +# bb_user1: gh_user1 +# bb_user2: gh_user2 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..28d22e7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +services: + # One-time bulk migration + migrate: + build: . + command: ["migrate", "--config", "/config/config.yaml"] + volumes: + - ./config:/config:ro + - mirror-data:/data/mirror + - ${SSH_KEY_PATH:-~/.ssh}:/root/.ssh:ro + profiles: + - migrate + + # Continuous sync (runs as a long-lived service) + sync: + build: . + command: ["sync", "--config", "/config/config.yaml"] + volumes: + - ./config:/config:ro + - mirror-data:/data/mirror + - ${SSH_KEY_PATH:-~/.ssh}:/root/.ssh:ro + restart: unless-stopped + + # PR migration (one-time) + migrate-prs: + build: . + command: ["migrate-prs", "--config", "/config/config.yaml"] + volumes: + - ./config:/config:ro + - mirror-data:/data/mirror + - ${SSH_KEY_PATH:-~/.ssh}:/root/.ssh:ro + profiles: + - migrate-prs + +volumes: + mirror-data: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..860eb81 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +click>=8.0 +requests>=2.28 +PyGithub>=1.59 +pyyaml>=6.0 +gitpython>=3.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7081e10 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +from setuptools import setup, find_packages + +setup( + name="bb2gh", + version="1.0.0", + description="Bitbucket Server to GitHub Enterprise migration tool", + packages=find_packages(), + install_requires=[ + "click>=8.0", + "requests>=2.28", + "PyGithub>=1.59", + "pyyaml>=6.0", + "gitpython>=3.1", + ], + entry_points={ + "console_scripts": [ + "bb2gh=bb2gh.cli:cli", + ], + }, + python_requires=">=3.9", +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-311.pyc b/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab41a2f51a138b0c4938f687109485a1015449e9 GIT binary patch literal 155 zcmZ3^%ge<81efIwWrFC(AOZ#$p^VRLK*n^26oz01O-8?!3`I;p{%4TnFFpN?{M=Oi z(&E%2{iMv2q|)T<)Dm6Y^vsfs(j@(o)Z&t2{rLFIyv&mLc)fzkUmP~M`6;D2sdh!I ZKqMrM(z3cTlw)WxLvb|Dv{SIt5wi`Q5V8?klZPhk3C2}S0>eV6T zhbjS!!iPmpZBf8`NCBe(E^(^=4yQ`@Jx~HacBFRDk^qzh`%1=C z&)Zx)RGWE6(+xegSSB$er^V9M!g8#b&l1G|O|h(23tHkcZ0$OVM4|3KxajKBzAurgaBIHCn=ZBo%&PkAy ztej!no8#)7y9a=ya|E$@4ss!@R-Ff%25P%65KS?(m;wgOEXOj`K(TUqNy``sFqu!U zXqnmzIm4*vFC~-dypgVEmNg@x-BOAbFnFe1tVI@;Wo<5p8$?^Tx}Y z;$J#(>eMMyxcb%`ug%w{tfm2;tEQdpCY4G)dE-d8`E0J+yjZ+FS0>qbn4l>MTFeZk z({ftTHMvR(rUV(<6Eg@Y=nT~&q(aL0V7eK~<{Ws)FuVvUXxjAZ1~H|ZWwI=*d18jE zx<=%pQmN!iIC~_cSdn2;aZm5XiPSZ8eC%EdcxbatM_ z?2gW^skI4KFmc)i*uEQZm;W&~v3B4W#a|U^rCp$F`^Gz=k(JAj;YM5rg>_gn^znP3 zxyxx>iiiEfi)5cvuNbc1sP9>TP)Ip1OCL z>KeYfPyfAPuQrZb>a9{JTjW&%(Tkq@p#QK}CK~qYf;#rvb?<;TIDNaTZTmj;KHCyE z&R*~RxDv05t`>X9zUSTo<#k&}^?%^4LGK%aDyf0GkmqSmbnY{*mh2s-F1lNC=Y9sW z=oa7bz|RNv=I=SIWsMrgWnusS4eXmAh1_oe9J4KJgg39DaxlXpS8A7(bY_0xNDTM& zWKL65Jy}%prJ$-U9zAxPsVmh&AvsS=gV|jE#K}_*{Q}c(M3cDFCGn+^fZ6yN&ansl zhUd(g2YhW|-+{{Qg}5I4X*Ur&6T{StaXpby3I$l=)shix#Lh7NvAAwZbnQb^54@%q zpePnl9NUO0I&32bZQ=w#L3twX-3V4IDs0$u@GKjiIkh%SAIi~t*tETwn`+!k4npt9 zArMXwdz5(C3=_?$l9GHwDO5Frv(OCox@iVquH42AOICS(NjJqp84$X;Q`U3kYC*-`b zhBtaqF=XvlrJ$4)14_&x_Vlt+x=nBXzz?q?t*}UDu%jWoJ44JnGe@17BP_!jKkm$& z)bVj+>>%UV;uw8aq7}QWj7OnBBoirW$OkMHVAIWPz-WUvFor~fOMWv%BnIr!$1d}wVZfJcckG@q}az&tU2}F z?)iq?lwvLEdRw~Ql&*gz05EWp2{7vBoBkic(%)?QU>)>I*Zty7f=Bfl6!Kc!$|Fdm??Fw`eUx|yR zB-59HESo`DW)2&q!?OI7s#3t$ge()JQ{>wy&VzWs(aLm>CX=W|f%8GnfLO8rIzn*e zS~E1;685!)ee0gEpz!M)%Fn%=_^qC9Hz$T%NKrVj&VlIREDcq&`{P~lhW25xyT=mU zhMvTcDIe}*VpP}1yQt3R=1i*4&zs$B@=3YWrcC^4do$>}Kk;6I>IbeRnZvj|E>ovm zdkcN*+V(yw|F!}2r8e28y+pmg3x&K^GHQGJ3eQ_!Co3nrVHh&j2;h+5iuc0raR;W) zU9Rp~LM=TSB19{I;j2DXP(|Rjo-Ci|Rp|pC_}k(V?ke=?c@A~2qqZLyu)pUkI8Z(Z zN>YP$X$gDK*OnS``6K}g0iL?x9_b%J?M9#xY=qQsBU}&E!(DGAQjh3&H}@=5C<5 zh&o!2cKwVY0|{C8aJ3Y+wDlo(OYU6P&xka_>Nc=s@R=+L=mAS^e@;sV-Tm9rl6^R^ zeKj(ZsC893B2h_XN= zq8A&5C@Xrh<5xEP5c>irs=VR9d^?8wYb|sxhBO77)k@hEGKF$UgYcD}tIar~aA_KH zOJewIKG|&o9axBE`KWv5%Yhpi_koX9sAkC(=#v>zH9bS}73z)&g?wpQugyInSjVDr z&ekuQ961L~QPu_y@@Ur^wjAIi)B$Gi3iW}9?8DoFn|s9N`+}QH_G2p;oH70Sa=ud0 zR8w43@&!#LIF#`rO8m@VArB5KT0%D?SWiGni&HQHG?u^Mel(`_mFC^pTw z^X&W|4z*^_wr9__q^VWu=lj==KbqX#ly#@E+SL1XNU-yvG}*S&OoISO zwX6b$u(MFaG^;ml-GIUdtxee=)+R{j38f5c_k-b`wQPK6VpmhzwR&T13P|C1&KN#Lw%f@%4v9>h(iFEp7 z>GYqz_vd^|y4semHm#F|AM9^3F#T<&`5t4Myn_V_jAhkzye5k2rmmJIuC1a$xo1hXYGS|7$FXd~fIqc?~8xc^yRDL#|>Pso6V}3b9z&JE){2;ayC9 z9|dJglz7?10X~+}(AW>L{KqJsg#*Q-jN4qtiVBJ}3j9_{GAQt4Wgk@`i>O12Lb50* zH9@{>a}t_h#AK32+K}4mvQmIA(&}xSan%m@hZGn)*&K}q(0|D&ieyualjr_+s+qio zXX0Ipg7nzNaf@wy`VGsl-}B$Fj2?|)oWf=eka#AR!75r|P;vp+!*@`;f&z#HM{+2T zOOT5wCQx7;guH4A8l!GOZXoD6)e@%L!gMzTdY~o5+d{mXJJb>mw}r#~;n4}xapT=wVZ*l zuw;E*q_KN*LT2A16U_gDZ-fb=-6VMfzAf4+)8Exm@6PmqALv`s;d7#7XvNARIE-eb zTFM)^-6ZhSEBhA`905~g?VDj1eC+-|2C^tK>W)XUy%V{T>0iv>g0H9=+Y4!cnhuzI z5I3aLEZTjR+=5ydbEKaJ*1bH>cer?yi$8LXYj1>UDzHNTI$UUl{&l#a75dlV4mLMC zA90Zt`uB+2+jO5D?sRjr6TCQ&bY~9)&iPS@oAaYEcAjBNv1gozAo-y99Ccsx3XUdt R82Lf*IqJUHq<{|nzX6%7zMlX9 literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_pr_migrator.cpython-311-pytest-9.0.2.pyc b/tests/__pycache__/test_pr_migrator.cpython-311-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f788d9c96e6599d1c3d9fba0348ad1f344255b68 GIT binary patch literal 18766 zcmeG^TWlOha10P&)}3-Xgsf-M>%bCM#+MZ*#jJnQ&uC{RFc|>8I}^-?CPR$p$%M0! z$w)Rj8O^p$wq@HV+Z}=&nCmc4DtT`RcYyfNWC)X;wsK!fd6!eTC`jTBLGs^mLQVM5 zHo1+J1fV3?RMO2#LQoQJD%s9TB2W@-D(R7&uL<$CV*eFc)zsK@UWsY5a_rLOST;SQ zBsHM3d8v?*hrYz~PsE+Ne>^#pPL1bN^SWmtsikJ)4&8e$pPNq4#NGNfswXEF3Ykno zk$+r3#dizQC?BUp z$)9!rM>IJl0s1g70sY`s7vxm&saZ{1P)`mIPo=e~LTX;thUDwX>;m*6Tx~Hhot&2s zYWaCNSL`=MXVThiVah5uoLoo`Up-O`Gx3>OqlP!TcrdTb6#c_mc41h9-hCj>UXUJ~no3M1RXI^mGP(!^JGCC1nYFNiAhs7U(65Wv^2&TdN-KJ( zpvnr~orQF625)wX-E0;6IriY#@N7OS52H-udJi6?>tXFv8N@qpK2t*ri+V5%Pf&tX zDe#&KZSYes0azA(6Yl=E3zhIdH9SxY4=i7(_kM5rLe1S>@7}$9;l``=-ctrS$jGjp za2T(5_cLn8;O)gq-yxLPUJte}kKYHKLJEPo1p#5t(DF|5TmQxDDFVe!X%Dbhg)JAn@oKm~gaZ7|}qI2TQXWKJHQM(Tw5Jg_d>#liuQFmXu{OU{a*1FY9VMg~YDET=#rQvIA zsCZG*11PSh;bjN;Y%-mD5SUBmm5vOEn!c|7>}%dH1fik#GBX$-(!K^XjgFAF>|%2A-7f}e_S{<5&{6{79+_E>%UskKnY%H+L3rRyB*>n>+!U|kgY;`M<8 z_Xa=dzV~#c??u?xJ+7|6x_}@c^u=0BJFpZ$K!|j$%-!p&Y#YNpbiN+$T*=%!RoV6e z?jh9Fji=}tT;JOj@~#U6JYLpt+v8;pK==Eg(eDegFzEn80|w@XFzJ$<0Ns)c9uK$P zhQsa!HZopP&ce8u!X7p^9CFxW`YB-cVXx#UIqgIFl1p+dxl8Vn>#DHiDS6cQHH)sf z#vte{d8Wazq2#pl4h(=F@(dD9L?`Lt-GMYgjx25F!Dtaj>WaEGo#|1EhWajO@x zFQ2d%{ad~G4fJAQs~5k4UJP#a;y2KXp{-v026_=gh493K?c*%GlBF0rNwKqLpm5g# zZdLBCqu4RHZ(;FJT#bRxi0F~Hs=G0A0w2%?B?yi@h$L0;5;Z1q6eNIoR6&1989>kv zKzEKFR(1e#*RP;Mp$DE{Sj4a)kyP~dL@LR`v;_qslZB)r>;8nAoR+~O1jipF6H|qB zMoZ^Z-J8jSu&VbY)Y*I?BSEYNF8xd2e8u&`sC~2tmcyF z9o`ke6_qp7%(G*Dzj7K|6~uGV)r?q)!$eT@IBF%1sc73{#zvlFH@mQOE|n>ON1DjW zT2e}C$>QE-KWk`X!Q-G&{U!KA%fc7YT`P+p?5{?T)uPAB?zQN?ia1hY@!(Snd z*k3NLT&YC+0i%6Y?5~+PahQbIUu6YUTrLs?0>t1X=d4`eJQA@mNH0JT;W#10ff`ha zWS~J@nN_|X-BuBwDqmgcD*qG^+E>M=Y9>w`6acKs3aGezl_+2ngOi-I(#3fsVquV8 z(#df`pb{WZgDQ~>G>9v+%GaYk6){#`T!F_Z1{m$DVytH3#93}Bi6tvf}}>2-mFF1P2`PzihL{YzGtf!@qvEAnIjxQqZ7LS+^gf`=O?b5AJg49 z8?E3>y0QyEvHg{NeqN2uOv{T;Axo1Swn+UHMv`S=t-F71Ab#&a zrSBr_^`8Cprw^?A+&vK(k^p#oI1vt@EdI7b=;*)Q^TD50+mF}UkC(k`QE;CI%cYg8 zmFOT~w6BVTH4`TelMn~1tbmHkC89up7@Xvs74WAjmsJHOT{M*e$p620k=@j^IrMGJ+=SgzHxd0(?AGjQOqBDeO78F0=wN zM_6lPbJPbVt>&nC)nYukaG5XGx}b`=ahvojW|M5TXKU4BYt;e1Qd@;q3W+k>UD*=n}YGOW@zm`kj@~}!~!ul$b zm1vg3mtFl=W+uMq+I{=P2lLggleMmsW&aw^M-0Mj!;8OT=tl{2@TCzM=Ob+6IwwKU z80lZ+ycPx@zCncJgb)X6P$giJ8m`PLUq|D$4@}b{%+3Ku`>MFFX5z$Q5+Y2*VF4AR z@iN4S!AYn>oE5l8n}~(MPX;0!Cxkelg0ujW)No}@@<6g-+2-A&AA%$6{f`arenv<; zez!M2BY+$~$TQoyy~n+|ov=9RG$(2NN(v>FWZ&G9^}^Ws=;Y;$fCy8}ogl?m2) zdu_V;da!0zfTMl2Rk7EA^<3IEuR&|SkJujjdNOYhte8J+?BT6wmVwNbEFQw-|}5`;5ho1B`JvuEqmPvC~w1Jg%})R(C==2zuEN(Ze#w zi1HlZh#f{Sir{$Bx*l?Z);nmA%5j#-JL)}@K>(b5U| z>W4-)b^2*eho{992hr(o9nG7>egutDb8IlLM|xKdSHio>&M%@*txVm9<%*$Nbf_YZ zehmKbXeB!I@$S!Xx)b-RIJzp1))-D4Q-lhsghW7=3litupK@LcgS2f{b7iWjriJnF zns7a4J4m^O0X2yS7fDh?ehj{RqwRpxDL5n!$Qrd-zreLxYjI&KV)*5rX1_cQw?ny( zpvgI(K^}s?fM3hhk=ql;xgrCEoSviWLMtF&9hKO8b#qL+e;FQwhOh1vWXJ-tk8+IH zH1aWC)A%8|o0CxQz*!sPyq%hKKhe27k99836P>Fi`PP0Vc4`v*a~QiseLH&O7Z8LxO9uzqOn%`;th?3q5XbK0ui9 z{}mYC_l1&c&Md*?C&!GVG(?qFE0a)Hg{z_5y&NCxd1CBHFo7{-jyQrJj2i)d(!jLoJA zb2iylir6*TDI?`9v;Pe7p>B;aD_N3q5zD;|wNe+54Y`gwOpMN(!8F;Lo z1nHKZj67ja1|G2|o8+J;BY!MCS#mbUi#d#k*>*c(V%DSmU4?HVXbShPvyp{z;RO zrZch>kLn>s)(T24aW$DK$a=?&tQjfiGRs!Ut9m4@HWJaHZbv2!nJ~6*6%-t|EHV8< zy+yr~GMQ~E^ma;QN#ybxOR$lpyHPextb6ghiL+duOjwA)=c|5 zKLLdHRk5#T;>1A#z^Y~eNpKR1h=aK{M%@+$7g}lOLZlO*b%#=t4V9UP>GRq#B zP)hI?UAdqv!GRfLX3UjkB$<8s8Db54{g+6h&5DPWzd`(O5&Rv3GJ?mkxXM334Ruq~ zwu6~AbPd!*G6P>3NsHVEC?0vJrQ4(up6x=#9>MSxDtlh9?s>fiaLe$a<+55L zGEx>h*TVRA3?Es*hggRn{z0Q}^D9J1;DH1+h&)|V0C0#9(~cP@MC{NS+AI;fW!%04 zg&&$C5T{GwlGyl;72bwZXS=6nqpCiS*>zBT4|PpNULKL z$&fvElRAy)KO%KW+e#55`j2W)H?GyDsmTY+L zb*)FKN9vXO;JbWye;c1$8jGFzspN zIHhHkePgbBbA{{_e5(BSSpRV>x!z%f$@mQlMCf^pbt6eh!m}88PF8^de?sgg0NuwvK8FP*)-ULpG@@zcJtTjC zfMsmq3*8h&$LMPlO3*dtD%iXNHp6VXT0HhpcVp9f@4yMw_ns)!hHz-dGi!Sfee!H& z*EsC;oxAG;BajWYvjY|#0C;?1$O}WzCPcb#7Av9M>z#pP0gTl#fd6fe(7E%y{lC~> z_SJiLlzl&s)StUh_ELleY4Q}y|4F(ue^R0F+0VqYRdK8)j#b35Mv(mM=i=$l#M4#r zxtjP~MSKoRwnJLP=i>99iO+x1{p;S!l{YGHPFBUYYT{cJb2pa2e9cQ>=#?n{jNlgt z{vAQnVptJ*2xR!FoeuP;&A+-Qu>5)@xToqFta%0vYJb&ppyoMXP(xMEaLqHk?u>fI z9P0wYuUvw61bBEn2$Fbw2I{YSnapm%8+>G~LC?;00l?-uQKIj0R1SFFhK3^CTxmmw zBx$t|dnw+l`w|H$pMp$v(fkCDC86rQ3Hp9-h=xJ3h1n^%T($`_qRkn+O%j1Mw6k;(uTH4-kQ;K~UcXxbAW|9CcxDMcBJ$eL~D9g^m^B*$wwK zA-v4~)`X`k_I=&yaX8n7-w(72PVvUBb*J9}qbC5kq0Tmw(PI>DbHM+E0Kf~8F%osa z7Xhu{E98CMDA*HmK&oad_zHPnHwt$99k5K?3cf<#*NuXnmmLm#{5C&)MP*+F~f z&aTd9A%i&b;2wo*p^y}M#gvk^+!gxXm$tNU?UNJ)5eo-~au4^W!|h9+`pv9X|EzG= zj#l5!eBU?!zCXX2{cSWFLQsswzZ(AuBJ?lT@QS}zIsFw>o**4%kS^$AQOJn=?KAzF zC?HeH$bu^gWC9@Z6|ou21kF$;WQH^0o-C4yfXrWvnp2r6&XbBUb2>9^#xrpNk;v|h zH?b~1M$e$}p~pa(b0Yda(gTl?j-QDjfxlZa=XG%#C4;rqhs3fiC0C}3T_B29F4;z@ zTCQ4(bx_g>Rm`$pEt1q{tY13mbAtD>c|*Hb*7h8!lC`yhd2bCwt9+KJpr6Tl^@C+m$_N9k{EwS!y zsiRS8T~OJ$yyzmdJcppV09*E@VX1zrLbT7=fJuomZ7KtCe3nfeykDk!s%}sx%1odd zC7aNY=EDg5ts6iZ=+o%j2X~v%*>DnAQ}rP z|D=(0|9G#AG)ZsbjM8}Du^DAu2EPTI5DQf|vT8FsJDd4Js%Oq^J8P5L>-b-IMk97a91v}bjpt>|V$4Am=_QO72J(m1G zk0tN+_jim3s~FVuz&QQ@k_Ex3q1r{}RE{-9DX(M|3tmN{P*N$UYVrFI-g;A2&4V@G z25&+V`Wf?OpvQC|qa7(n7!b*2YUGVlEwHv$0q+w?bf31bgMzwBsvBatBYn!1K@XdF||myDb?V|aP!ultXOXXy7DPhXv#K4LA|4|rw2KuSuf%!A!h zz_*GQXvzVxQ|Y9iDlia7KjUf5H{TJymAJ+gZ1EKOg^)Uz^?3#a@0Q!@r5HJ-TyoJ;$z_eAm8!*D%w)NKGX+pG%-Cd%q{v~< zk+{l8Ycs9&&oWpN^A^jX(?>wyCMyhI(CTBCiMJ8GSVy{;0A07}zCAt~joCv9qWeeB zrgsIM?NUzD4d_N$54r%P7EKeIUBwi_p_+{gi{;Q7{N6hnsn|xj#Lj~(&0F*mhD8KBvEJvG$z4L0uAfizGjHuu5u zQ|%YmetE4KTY~4<{@r(fv;W7{=c`-I)vZtF7LQ}CxsCSRMss?jb8hyhn>+#BL*^!1 zzK~90x(KzQy#5Aj_LG~gu;y2pa>z~dRmK?PIzI>`b`acj_-oTme?APyfDm-(odEE1 zjgjC$Flp1Zl`mtO*I-QR4?qA8Fp4Ld7jLwtZyZS-9DP6hUiin+pG1$Mo$2|BeDAzC ziCpePE_Ws0t&j&J-H0zZ*F~cwgp}*-D7o|nAj6K53&ZdqXyN;9e81^E-27>{Ibu~U zWlf^0P6%!j{DTJS5mo(BHCuEglm$Cw*Ke9)~k6Z_`bCj~vk(2XKvHk?4(f@U% zP~(Tq@M23++LF?1Ew!ZOwzS+8dHkE)GNlaPJ@(=d8=nt{yKaQDTwl$><}s;ZYYFbLkM8b$d0vz*9xDhF!6l~5}O zEPzt({RPr4Mae!jY~lp_EGOXcLeQDVTb4Rv*>a-Ql40}y?0VyVZgFd~lfjW)Whdg^ z-g_TJ3@kmCd$n Date: Tue, 24 Mar 2026 05:23:33 +0000 Subject: [PATCH 02/74] Add .gitignore and remove cached Python bytecode files https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- .gitignore | 9 +++++++++ bb2gh/__pycache__/__init__.cpython-311.pyc | Bin 222 -> 0 bytes .../bitbucket_client.cpython-311.pyc | Bin 4962 -> 0 bytes bb2gh/__pycache__/config.cpython-311.pyc | Bin 3192 -> 0 bytes bb2gh/__pycache__/github_client.cpython-311.pyc | Bin 4956 -> 0 bytes bb2gh/__pycache__/migrator.cpython-311.pyc | Bin 7250 -> 0 bytes bb2gh/__pycache__/pr_migrator.cpython-311.pyc | Bin 7635 -> 0 bytes bb2gh/__pycache__/state.cpython-311.pyc | Bin 5515 -> 0 bytes bb2gh/__pycache__/syncer.cpython-311.pyc | Bin 6934 -> 0 bytes tests/__pycache__/__init__.cpython-311.pyc | Bin 155 -> 0 bytes .../test_migrator.cpython-311-pytest-9.0.2.pyc | Bin 9929 -> 0 bytes ...est_pr_migrator.cpython-311-pytest-9.0.2.pyc | Bin 18766 -> 0 bytes .../test_syncer.cpython-311-pytest-9.0.2.pyc | Bin 4245 -> 0 bytes 13 files changed, 9 insertions(+) create mode 100644 .gitignore delete mode 100644 bb2gh/__pycache__/__init__.cpython-311.pyc delete mode 100644 bb2gh/__pycache__/bitbucket_client.cpython-311.pyc delete mode 100644 bb2gh/__pycache__/config.cpython-311.pyc delete mode 100644 bb2gh/__pycache__/github_client.cpython-311.pyc delete mode 100644 bb2gh/__pycache__/migrator.cpython-311.pyc delete mode 100644 bb2gh/__pycache__/pr_migrator.cpython-311.pyc delete mode 100644 bb2gh/__pycache__/state.cpython-311.pyc delete mode 100644 bb2gh/__pycache__/syncer.cpython-311.pyc delete mode 100644 tests/__pycache__/__init__.cpython-311.pyc delete mode 100644 tests/__pycache__/test_migrator.cpython-311-pytest-9.0.2.pyc delete mode 100644 tests/__pycache__/test_pr_migrator.cpython-311-pytest-9.0.2.pyc delete mode 100644 tests/__pycache__/test_syncer.cpython-311-pytest-9.0.2.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..043ae1b --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.pytest_cache/ +config.yaml +state.json +*.egg diff --git a/bb2gh/__pycache__/__init__.cpython-311.pyc b/bb2gh/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index c8a524037c766ec60f7a749bb1b32f1190f6248c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 222 zcmZ3^%ge<81efIwW!eDg#~=<2FhUuhK}x1Gq%cG=q%a0EXfjoqI%Sq5l_qDWmM8?L z7L}zIDU{?ZxM!AllqM;-=9Q!t6%=I_rz+%TrWYlaWaj4qrSo(2{4^PFvB$@!1oEL$HVWxhMe0rm&j16)_TCvs z;0zKEQN^y1%0`hu-A46OnrQQohd$(`s?=X#BTHB#trV%M)Kyub=$%^qrbs=qPYu!FD8udGVxq%9P6eP zS2IQ&zA@*!p{p1S=ClrHt=Tj-a;jydQ!$?-V$;G##*wB}Q&+M$>G)s~MI@|$l?5)Q zjp4U?*QJ^f#yI10~=>nEc1k??W%R z;<3KcvA$J)t+jmO^dI^Qkt^;CBf%dcU8P8OF%m09Vx(VgX|Ev0e~T$!9(|)1E|0>y zIzNz7>v;xP&{f8TfX+vk>z|;kWZ882o350spo<(<5ZF8OA`3j>FO0iEYkhtnYrd4zrp#0rZ-5+!MrRH9v9gwa2kQZs;xq zuWISCHH*{PnORv)CgqGeqnmPiT2||V9tMpRP-?QC(lTiSP9{f48ViQ?BjNqvCF6)0 z6{`!TBR*7<;N#;?z%a*D)4B)zoYs`SNU;e^#g-$Oq=|_RuNeu;37ER2>C7~D9yhD_$U0Wpnr56U7uc0=GkILwA2w@70XiFwsd?;I=3oIv`!AO}BTt96JEL2j(frNVVxcoy?7Uj)yt+D44z_Ox&uj(H6ob)HFj@#kfp+8i zlk4Ap{Pg41k++eP8;egC^U{m6#YlfC(q9N3@V*Uo{%-M`#m!(|D~5VYp9hGyOQ$cb-dex<_2-pN5C)SENz{h|(FSqxok;*2#Py+dA2>&uXY4X#)rEkK z@?L1v=tyhtJDVE=5RwO*>l-sn%k>|ep3@T+xKIlGO0RPwBL#L$CG>%uKc1;lz2q=S z!498<_4q?TV=T2VJ`1=>#Hrb`m5?a4g}GDv?b$FLC9C@Ytf03YN1y(r(AML=tK1Kv z&h?Rv@h9W?OT}<^DcoHQ^^`(A1*ykXV7&@F1^xzDH7n3&XgcDT+4oW-+o-t49;lJa zMLjiQJ(D&K3la-NqmhDWtbX~xZot!CJSRd{0z%vk^2IneGFqyX%Dlehs*%{(i-`bL z6kV1(ai9|ETo0@PJrrGVjxzv zeZ4;nP@Cb8KMtvw1>aBzK54jN1@8961l@1@(2Gb!TNU|dR>#Caka1 z56L9^?nujmEgW11HI-FDzFt0%%6%H?s0+x%1xV`AjOpoj8yWu;xa5R5p@hPMq#{~@ z;4Xs61e^g76WyFdLX#RA6T?t5<0xPzxjN7@qgr>gR4qyP=1%U9$-2o;$m%5k4F;xp zzSlLdi@OGP@v4E{4l!_Vg&4R(47_GFY;pj|$QW8omx^nzK!nIE`VVU~m8?4{h_Kk$N05=HBHt-%JD}8oR%$ z9YMeTkXF?P({qinZL{+A+< zc^an|WR;%qa(arCCht7r$+6tRU8ZjsmD=tA(go40V;jyRL18dJCc6a^&RZt*3UOJ?g&ThwXh(w60pzt6~iK zxB=J@nnBV8$xW1eyP`M&MOjE|*(AY(it<%fO;%@oilU_xih_^88d?p{<4dMz;NmrdTS(6j79E2D z9pe$w7s)Ji$O=qsyQhU+X~@;>CwL4d$eqBP2LOqIValkxfV$tJwiWtUMxhn@_tt-6 jHB$1Q*t}NspDX##t%w!zF2jJp2g85U@&6n$ONIF#s``m5 diff --git a/bb2gh/__pycache__/config.cpython-311.pyc b/bb2gh/__pycache__/config.cpython-311.pyc deleted file mode 100644 index d7d17299e5019b206bdf0f0b27c73e7ee6241b22..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3192 zcmb_eO-vg{6rTODSp&uf8ppArF_1ZP5hW%l>F2Sn-KigJdBFo}DbvdniGXNv)-* z2QYf1&kUh7oglL1HmBpH=z7UCBS4x3RG&-=#ajB%X(Y2t^w-P`TB0a4gDALCXtM^l zf{##&smBFLCWZLQ2ZfP99nrE}=>Uh(jvuZlR9!$U^IZ#`Q-E6A#Yv1r&)MaAf zXu?|M&zUOy2HFO}Sf2^7ss64+4}5NV3!hsmHXHiP)%y%KTXh{qFjZ})4JpH0<;|?) z85%_X-}2UcK;GJ_HiedI3=o|~=P(VtX_fZ;k!fSf*g)1SwC($5`ZsT=Wf83VwA%mG zulBuuK{Oo|EdfCt=VzP_90BVJ zw-m?oyddF~Ru2?7zb7hT9LGv)QWh~E^)D?CUm}LK~$dmG@nD}yK z6cv`o{w2kT2tbdo*c29c3 z@u=t_;d=D!4-RUn^aKKDLSaw2t}ZMq91kX?1kbNn#?)%;u1t`%E^k2vSw+vy?Q6%r z_1*TB5M?@+J^j4HcGtbxp6}=@bo6D$N({`ZES0<>Lazy_&K=Zy^XAP#(1%?HHH_ok}+dPqX4i=n)Dx_xEe*60WVAp2P9=a=mr4~T7cHUXKwY0XF#d%9_ z!P1*yi`_jN*VeD)jt_73=DRNzx-SwB97TKA2Di>_Iv#$Nw~rL;BP0d@=yq+4ua9qf zpIpj!oiB8qC&EW^>MfJXxpwF@^BX(Vv_%ay1@%o47-XAKx6nVJ1Lq6sBXr%KrczW2 zEmF9nc1soL6!ks&Vb6B?4R!lS3eF$Za+anARz+c8!=v}17*b3_6k`nvY%MwpyAOqH zgrb%lleeG7WHn$?SXX+cBv~d$9xUESN>~(J+Mn-&l0l9{7hI|^=x{`bYz-@Y`*ieG zRMrM9f?m-grEI? z^^D!n!ww)T*2)5{RH)-b@=qYE=*3}o3DKrcD2-;uixyipn6vc2{oLlt^?0|x00MX3 zHeIky=d9Bu23dRSR#fK+qU}{Nyi{haXz9rIX409oiZHn?=XxgL&fBI6wyB(TiXe2? zE{)FP1mLO)aCINR#P(ROX9DiL?MlIRC1<^&0WejS9S8<_`zcJ~hr$QdDdvxgJg+qK zd^9d3Bc$KT^EZ7&-6`uVgxe_VTq-6crvNy40i%~)acHCG(nud1#uL`auqlnO8L2}m;W&S9$ z%Sa+9M+jViHG&u@0?0?5To=y4hZHc-i*5AO6Dbf7v48;sJ@rPzE)e9@_hy$XQkI(3 zL-OtXyqS6Pe!h9jzXSpu1j?&JPt$+wB;;S%Xcp0_y!-=H)`>=DiN{_jnkApEKgq_-PXL6o=2J%?pL3 zayDn`bdjbFU5Q`0sH8G!J!dLQG_km-)7Zap)>M?YJitC*NLoSrd3HIaFPiCmF6FGt z9?e}|ehkfZq7#st%<>vHD`@;JGAn9+O}fR+N}2~+9?c8XtNDQXG#RL@bpY+qIzhsK z<+ryw!z3z>|K1i!Nzi;Drzuxf>|y14o+?n?ah5ztpFjbPSgNU3lo9I!{9fJ!vQA7_ zWrUQt68}BduC5YMU5*l2aO)e*qrxiNr6jbQ0cPV~tds<3xsqrEj>w~8eaVEOs|A{| zx-$8>bWY7v)y&_}b5V)nWQ(7hw?cEe2?KM9T)N0kWeL!-d_dG(BCAJ5OEmP%b?U?i%qeyykVHg{-!f{dVr?Ql==pApToD&3T($=6P8iCZGe!e?ZOi+YQH zIz#n@sVfPE>Wg_JZRY8UlFljSysosmRji`}odwHP-7L_Y!B}vTl20z^DKn;AyuRI+ z$iTYVilQ&44bylRR^Oh)`qH^MC7;t{?y~35RncXwPE9vbG|f=N@&W>V2@+azqG0Bk z(-jA7p|xgoqZAX$urW4lr1AX4x16wN;ek8rfC$d#I-9Ym*Whpd-qy0dNeQlA_vhO|^ zHU17{jWjv}au}oI)K=u!X5?7Cr@wyiSiSFXec*6ICfy@1h}7Nn3Q76X|M*EDa_92x z%ay(l9%?^iAJ0_%mumh?75^nzqI~(sXTd`aBFf=yQV)i{S-yMX_E+Vvo`v?WcmBDv za_~2gyZ>at^UG0t4dSjK)&Trkn??)&N8P-`ecQ>_Sz(S*O3zXG!Ui{iQ6!vQV1 z;(_zJsU&i?x1ndD9c*WgMR{guItUv;-8GwmVknM%aBv5bHBF7@GWvk5k>|n4H_KZ= zWizO3bXS8%Yr&%x-%%Qd!IT^D&~RbuAvE6PP^+GjXOD{nz^o*eB)E>iGh5)yYqm(R zfnNJs(fE>gJ2OCFY+nNa%?s)j67NF{xwx=c$YemDUler1gop*CXbCPy$J^i#r*p=H z8`Bs9XbN@}O#pHXF{3I7idaXk~EP^&F``)Ua=F(OoD>7+qv8`w8EVovyft3ZGxW52IwWRBUmEJ zXo)BQ)M0~SpRG(b@5r!*S{ybIhyX?qgqUx-NQ4Tf1K524h~>!@vPqo+UZLw+va3(} zHu4-nGK2&jl^z0O@r%^J`ySPbg<+5taS#XB2M}WhEut|Ydvk8@T{xn!j&ThLU{fCn zb-Mx3Uk#4df}>@z9@w`P7~BjDZk&B6KKkIXRtXGN1D9)o%VqwVuj@|d?aoSf>>>Yy z{Am2KUiD4Ze3KR5WIedAEZecu>a8vyd<%QNQS68%aj)2EhYq}`cfa^|rfV3F3}}?N zra=Ux^4TmV!)6{DtpO5?iY*pp*98t+-Hv9{hEdQ}XL{6YqtOVQ)RGseYFpYiDH@WZ z@-45!Zs3j2?Af$vS3=WNnCCD647M7xeQ*LM8=nAahp|>G-ue9Y=Xb}ef#F(UxXjnX z-CN=4W;l9(s2U!tg~!U?dT@Wa>vfa4a5n+C0VWd+NbINI0LxeZ57R;Jx2N$S*K5my zIAUTrIAz0qIcoOc6LLL()}0w1o&z#G3{s~+hIee$)3)BOnV2TRdY+x@K{m#%dTx4f zUAxwjjY-AqRR?FO{$*NU(y3u1Zkvn!Z?JXdO(>I)cq#eoO|-`J9zoaGhRLZYhj8M| zC=P;e>l)y7b8GN!06G9HgsS53xkNex`Y!0q>Tz7(p3nrIZGf{g zEbmf+=HTXTjR_iq`SdsvHyY?+Xuci^A;;NWsfvTU#ezE=$5s9b1W?USLj4WG$&tE0 zTs~KCx#RZ^jt1|MyXyX~E&tG_f9U>7#XnT_kJtR;7614%f7s5+K77>w_+r&RRr60( znqNC56XdQ2X%gh0x;<47A%FaZzy~0|1Jd|;!V)k67QzG!Gtj>sk4MRqXvgUR;mL`E zr+b8_JrdM+Pq{vY32%h3e2HIhlQ@QpM=$}ycZLfyGgqeJzXC|@m1|cg**kcG+?q9C)gfmeWgYXvY5bD4_+ZDws7n}%ryy2oA!2bR5A@eMKngBNTX z0p6KtC%(&KWT>jup{m)uR>)x8uc}`Z5*cU2tEyT)rK%Kjb(YTCF8&TS-$!y13BK|E zm9QP`#2>MdB9Y-|l!2_dpSpOaBA#i8g7g-6Lz8rbCCJy?$p|kU19xZ=Ul3Vb^0p-P zHi%2QLK66nONImqUIJbxwvZ@VRHh%n!}8+W55%D5nPc||`SVM+MxT?eHTJ2KcPhJmpUVT~-kLmc e_f%CLs>wrZo`&bQ90!NkoxWiGul5*avi%oI2zxsK diff --git a/bb2gh/__pycache__/migrator.cpython-311.pyc b/bb2gh/__pycache__/migrator.cpython-311.pyc deleted file mode 100644 index fe4c4c1ba7d4a2ba8f9b49ea335e173aee2c63d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7250 zcma)BU2NM{mcEooi4^ri+wxx$Cv@T@RuW5Xv(2V<;-vX;>?X}*>r6ZGU{))Ow5(W` zG$fTDRUQaIfNBZ@>j}Duv7NzcicYFQ^5BPkFc$leot+H&NE!qzaA3f|q7Qvjku4Bp zU-n#5q9i9Bl%@_Z?>+b2AKr7$caFdE`8)`&UHkr&{J&0w{s+G_E@!>)yam?6xkW5!`Voik3sDUd00k(!|h1Z67aT6E92&9W;+FETR>fgDIAZknTExgz_B zK2Ph{vrcub7fA3r_wZJbX6H-h`X3~6Sj+ex2 zMouajDJjbQoRnGQ&n1=FTzo-P_-RqPB}%-K;V&hX$=ob|F|CMFR!YiZ_**>a)o=0W zA;JjQozAC{Vp@SJFZ7wu&D!tOv=URqxWk&PN!>v47Crw0#K86t38H}VP_g7tOa_aT z?IpHUBEcaz%dOJv#)6}LV;%w5W1Eve3Ldw9-6R{Of}=nPB>YiNT#qQ|0YB?nau%F7 z?Ki=_V18C9+BQd5gS+)9&;=y8`|T=_V~<^??dq1oBPc(%-ygw9j~j;w?U*^!U~8Qn ztistmBCX>+f|WkDS9_0W<@+&nfdjni2@c7VhvX+lSqh?K?&k)E@6;2Gs(2!k#nFlZsIaCI&j_MH ziIS9&3>QobN(Ln>QZj3Fo{yzK>NTs^MVkd?12ZIJQbLxnj=^LX1;ZtYaxSIFc;k4u zUWOy{nME;@lSL^q+t8Ge5tuO?_Q>q)TZ#FIrC=E;oV{z%QJ7W~nxz2{;fJ666i5+m zvs~%$%7tHti|4mU&nH*zU0E3@JHMj8petR!_kHQp$goa^RWiKg>s|iGD$TBRm9MXn z>lbTIhu`%b0`k9@6ensfMEiAj-|D*??g7<3utod-+54$i?H;ZSRet&8@Y;!gKfQT; zV&nLPc6?GlKB>`DIz6S*Q(OK%-G8*|?bN*kt8#hb%l9$scTPvZ1EK?-8geimknZhV zKE28A-C*}>EU&Y?O7hQP4Jh8Km{l0Y}W7EYtv=#KLSx=j>_vaJkw{KnO;G#Ah>K3{Sa9A*2kH~8?gop;Hpu+7$j zv)ynAe=&F>_1g=aYgUra`&q zWPTXaNH&*BMUEUZt0Ie{ln_TG@s~MKR^-T$5UUKGN4TAt6` z2EMsJ5~gBZ{T6>@34h7F%MzLQw@b@=a#;bq1@4u1j0pLj3t~!CM4@F)*evAxCQK87 zU6f#Q;HjAggE93@>cB(w0s&KzO(m67GA+tr3UJwQro^;_RZ$Za+fsjlFnU% z!WP%_(EGrv4!pf4YuuR5jj87SVykOF?|K8K4BcBP?%;pmSNFZMeq7_u=-e5#al>+e z8Lx*y`%cu@<{IR^uvzyyyV1V{hMiv#!_NO2jhs8;_|uUi=X`JtW8&g z3v8W@?&E7FwNZrv{Iv~~b*mMHkS;0JaKETyqTE4!Ng z%UBy;xdf^}x9=E%XV9|dW*6Xu9dDx2C8of@IDc636g+~f#e%X0Pnm1>YnH&GvhAZS zc?-@m*5p^N0=<)0+W#ZgL!15Jw6ub&f=giXJ^-Jt<)48yKiC|s0IS8o3im2_3!X>t zeIMK381{my_oS9{V5JW=dx94m0^8iW!@*{2vji3&uVm+p+;zEbu+v}IcKWYv@?QVN z*aB5>72NHh56=|W&RKT;uNcfJ^CCY7W}TmH3k|{^4>oEO8HpET@p&+-*$kgdH-m#w zv;EC5e_j&7Qk!y2CI~AonbCsD@K%^V2c~o+p30UIHu)GUBW$Yu0NIiJjy4T(;$12p9^LB-VCuM=)o;pGW zhr|QKy(z{O?AQ!aPGR_)jxCCLetIF9&0>yN3O<8yw;tPBpgD3-w!Ca_(@WN;34)+C zf8u>R6vWHGE*8cRY$?a^i;Tw*1+#1+@=&*gchO*HXQQ(*S&ZhSltIIbl34(ciY=zW zCgxk(;Kgh+Tu{nL2?<+LgT9@S7NSB@G8`G%pci8cBD~0^-1?Gb!-sb;+IThHwLvD+ za~Z>9wwM7dgdCz`ut_;uU!Y(F8^x#$8w)#x?v(Mv0{wG6lnDz1KI#@LI3d{upe%TNF-@L0K~-@xx#+jhdg( zD&p@bgVljk)pA*2!q;}Fr0c--Cj8`06Gwa>M4tZD9=-o)<-;dYjXA9|r;F#dnC_K1 zy=S;`=t*S##DAXFm?@o^f_iVq%AwC9ns>kM-Cw-;4e7dndBycvpfaSA5si%KWCRM^ zfo?r8s0QlibKS9da&qJ3q;~Q>{p5RaO0|QR^@Ep-liS@r#jDj|ck%sdSFm_xyLVp= z1*!ceDX};JVLxTn;IYbrMvvjXR=qM@rOou)l^l208&LU8wf$ zS^e<0-qI8l{*C@uy4+Lg*ZPj@eaA~v+kxO_fZqu4<=vH}7C5N~PS%jq*N@=^`+!yV zj6Y?+zM*j+>D)&u_feG_D0gVw{uN)9>#LD&-~KIbs2be0damp!U#yI+Gf&_7T7WpG zqZ8tsj!uYkfB>x8SH_TP#lPYQM6(B~eK5NQfx$L8euLxH!O`{0U-xU=w9ZYd+;o-O zQzkTSaK(%NY7}(9ux0?3_5Bc{*iSCsyIiHuR_UHqU%Bud<)k2T0|H9`LL=|4)zg(@ zDtkgDPsor$LVq6GeG!q*(9w%w_p=cKXx#QT&}^qoo?3VZsv1|>Mmvof{1k`+S)g9^ zF0W6Y;KWM1|INAuvOs>000&T?qu-HBG(=Z0_Ny287g}&Vaqk>UX_S@fu$R_Q$p4`4MXh6!5BcAdOaB zDsVyv=1Gw4I|evrcXK5>_Y<&%uV3!Ir~Nj}EUiAsDPV~P~;$|mRtIStCJ zhCo`6UP$E*mOoyb(#Ub099PM4 z@OfRm4~HKNg9F;Hb&lwrBWmZI`sUo%KL>*Ep9@x@c)3ctKDm1D>dHHnqw7lXszzSW z$qOoZ;d}6#U7xveUI6@ebZxg9{t4Vq8S4pD>r*=fFd+mlwpV8JX2z-fj*qE{3)@$dr;qfOzpc4cje>F$k~m^SuJu-kDRmWp5CyYKuc5h_=hUY z+B+J3TBlE|^l6yE%Hh?M&&i(qph%s(0nCMh@!aph^zH4#I4@4s3KC zC{Jl!VZAF{yi#*HDc|2*LQA5@hZ8*(snXsOiPJnjzz=vNUuk}oTD?=*qxw#)PpRaD zMo#GD1QfPNuj)H$-D~|SIi`_gIynXfm=WdKBzrf=-c`1I!-jS^$!s2hk;=#Ga0Kdt zKo;C4?0B}j>tX~wiv%VJ=U)&3G(=ntbr|kwRLI1mQNxW>@SxSE2AL<_;OrbYoJ;(l zA1gBOS<2j~!x4dX-c8)kfXZC4viWp0iV<)$YH2J^lbLE|f?ewZ@ z86lSv$E92FioJ^b3E>FbjM6D47O}hJfMjKI}Jzxq*;R z_o`cJNp`Z#m8GYu>Q_~-s$aeLd(Z#iaybbo!^gjm{YNuF{1R?YK5Mq^|~!?Td~jDNy0*iY2rWd#N(Cpdt*1ed`6mYj799JD-wbTSQS=u2XcfgIao{g_#p%D`r9-6VF$ayrV`4&v zE)K>lrRK~weM5!?=Pib$I`sj?=kde8!Zc3^h5XulxNcG4Tw2YxV5zHH>*`dkzHa?1 zW9T|Q3ixbwZ5rD4T3cXh^}KnurYKMm9RKeTPJw+2dw*u`zp=G$Q;7Gh#R;whymrB@ zSbt9l9)$wxRcN3-#RAl?_c!VN0lmLju|a?ETi-gP*pWkFgcijPJx9K^rd-J$%gGsWL31K;H7Uj9B)YHJS5fk7abDJ}%i?{FlH#dF%@PwdTQnsvC6Pv@ zBoS$>5R+Ep(fiZU6;Y!SnA(4tSWYC@5*j;?L{Kqd6o^KoLl^np-in?J2UNSa*Ljh@ zkxZd^@jQR$EjgH`c@X+!+H&`F+BsMeMH)=o2Xzq#)1EH*txmLTvQSxDHPYN zi=rHIYL-=`+2)gph1jBIgXi!FVSA!cSQL$3MUj|*+_=YH*|BC@5~G5MPz&@*n3*@r z+Ybkqk}KjMKEA=Z>fH{97xn8oI5&4@aS1jYF~<-T9U6nH}a|zIxH44qe$Bn%W(jdbv>eGEyA+ zl{)mRqARMpqWW477alBZ-pyYsx=+D^nTaxEb%aa7qh%}nmI>V8yXqQ!AJiKAoWD$0 z+4hfh*EI&t)HOyf*BWITapF|jULB>cP__~@TjmJHQzl5dzr?r;{=U2&i18^heOcSS zKk%gIQBUsZmZj+LQ}MG8w@_xXM74TDhRN|v`R;$uI z05{8E(C-=Y!Vx0ut&ksxM#5OW}exwLCfd z=xlCk>u3Lb_8-q~kMCS5_KmB34wP_-AwL0cr4i(u>mF@T<~AGo%5H@+wz=oe)$f2-ghG(JUUFl#GO&g}eerZVIE_-63%$#ogO z1&T*_4-Geih7SE^l%Vp2SR^n^$V2^b5y-^@w_}vJdkr9hwH`|34-bSM48#s-D+S=h zgaYp#rBKSjQM5v%X3%rf>DMo((2d}YBsQ)q7@pDTp{eL3U>nELPKyRo6LtSI&`0nbf2I3CKnK14%#~6g_+OI-6F*RjiWdrJx(@v}&?kg!zEy%yh}t`b zy7|@)=z>MY7P?Y^kE*L<4b=r0GEdg<`(GZw#E_bY@+`pA$QdHyTIa+M8o=ZXIA4A?C)Ct zHnAbB5#LyE6KkZx!oAnlIfbi-_3JKZJ8Ce0-3@1rt^A>a=T}^f@}kZL$=-c3FiLSZ zjw1=|w+t_}F$ofIH3W@ai9G4SU0 zs&8(S+?e&zqSMbU@e)73;dkO3(1722S&-1 z%&$b3!4mWHOVPxlC>ax_KblTff+kUb*c>7xej%Ak2m|~~D#4%95qp*46u%Hdk~}bt zu(e7bxrUPSq9ox-ug&0bi0_sLyCseK;@a3Wgypn^!|-&+cr+G=>0!I^m?SqcBYbxu zMvKXDG2L}zIi~vyJ{pIJ2BJD6Qh>oYV(J#o!wLvnH_eXo2RN>CQ~=Bs0b%2o4&ND_ zxy{e~a{4ZROFxIo6m>K3r@JLSd=cgfrBBp^Yh>hO31d6ybFjAY3RXt0FyzyHwPvrVn`W7t(=3ZiXc)`qt`4n8x;-*h=IZ!7(_%iPdC855yUKabRi5fC|T8A@s3+VmR=O%%Nyv zehJV54rPsogyAy2G7ZJR3e6P}8xI8Cgb^3vE3JYqXNVu>IjlR9pMDzI+}jIzCk$ zpJpz-ax`x)sKL`)JA^U8As)^;@4$ScloK$gPM3p;mShKk%xmAhHsZkD(}79vn!e)Hx{nIW3m z%fwMfr%tk$06=ZZk1uD(aAocBWOi)d6WH^d-1VHyPj0vFj1@hXRL>;{R9&sFyul~z zBeu|S>BZQ~p`!P?>b+j@UN3o%=6ywP=ccRVJ+j}{k-Pne@D{7K{VY3Kg4V4+^yQb; zwlmqueGi~3n>kM@Fj5M%=7#cpTfcryTYdijC4j(6Kp4VzJb$9-J-O+INz0VO)%nWX zSqik}F6Axxv8@X`ju&TMTP=7H5I6=Pw%VXg_nTKu9e@K}u+b_3`S-m1u9q(bKHh15 z!M?oxjv0>T$)fkfCWrsZ4j2lj_WC0@upciHH2b^Bznv^GBPAx7bLAC~nzn(|v<;*N z!Vp|r?$a#@$A=5lu+#;b_kI775!&)6x@Tm7{!^F)`e(9jyo364NB;zC|4%0gbl!A{ zFaX0dZ2Erq9#j@Gn6~wE?a3RZh0L-!g)l6twn-6$tg;7{M<8%AdhB*ODk$Ka7H;kcRl)Vx3 zth)pT;FSwt*bM+So_w7=wXuRja2}A%t9S)=8M`$C4GEm$xksQ6aWN5UGNSG)Sdw30f?YWJ0+VZ)=z+QEZP5(wyMf6qAQ}z01d_$hnuMCGA`l3c zO2{aLd1o;%SAC%8b8(n2VKr~?&PGG`(P#pX#N;=a^)VP72;$8gE>zYwCLx*Y4{Y9k z9L}?&LpiYlE_grO9@^=C(f)F|NX@F$Y=N5n(ME9Xxih(Y zMYcy}d-Uw^9@n|cb(Z|?C3jQF?cAf< zcd7Q=PRv}IdVM_cY}(b1(kx-z!1EkHu{-y{Pr@8&YVBcFDqy9M2?~#2?u^ z7h>K{hId`=qt?dR!$QN+iX;M;!Nso)1CaU-WQQbFOPzraY z)T_ECNVqL?aKen5nrzZ&Y+g^!$kQn*C-fN_HT6_dUZANNsu@P)J8U){c3F}N)lEI2 zSrW=jDye0>2KEt|Gg8S|rGNIQj^)8PylxPUz?=xq2j=4XfJ_Q14^mVGkZv61WSi;+ zDXDEBJ*ouKt9n5CR4+)s>H`^2{V?)?C9;uc4qm{a8+C(bqmlKb9F@;ZoEw*?^|%&^ zIa7zR;G*+zKUiY}GeK6Ki<%{h(>hCXRcM{IsN8`23nvF! zh}9RBOpsgLc-U>ZQidfgM9o=Cm`mwN%R6ylrt3&xiF9w-~+XDv6W0n*5C`YW>yx~rt@j`fDfGG}_K^$3}B z7Cg%}_84=iV1H^8s2srZ6d5Fju;_Z9e}^n`9C?RuaH*V5bPl-a8n;BMEu@UFo8p%! zn!DxJ-q#HikW4LTNk9^;!x9j@TP_;J3fy!BE!v5V0w6NJ3IJHI3o?v#5U~2bz}kzK zhc69p_FWGbe0@bI5FGT{P!xsk2z-u0zi=hf_bw#-8g%dUy< zvDObLYI&`+hYugN%D&&xy}4_3IWIk7;F$l}HL{1lyGIy#vhD7E4rFWCy+8(ly^az! zP#ehLSX>V2vMm9~PK`$BRPMH`(H1#U4|;t|m2a%c2;!@Ktmaaol|!*CrwX^l8!)af zoUs7~S`r2=c?ZfsN!UGpABItU$M-!T>iGmN!iZkf97f>_tMhlL2#k`vYra380GmE{?Og8U zpGON_kz!ZmPFG$UE=a>gX&98PKqWY_k$@AJ2I}$FV8txHuCAG`0fcpCHPEOq}YkYCsT_QySDhW8M8dPk%GN65jDdc zGj{{&FEdlO1O+N*ycJsqeIJw=G6(=B*u9qh^!4jwgk9wOER(vN;HWtqp7g9`XHj2(_*GFuPs>~Y@`_R^o-@k%`k0PzG_`1 zYFeIn)G!reDH+3dN+Py)21WcX^e**MT448^uh_JL-(m7#ZmESSNzTJdDIZ-pG(_dIS>CwNmXe^O(((hRg%dz*qvYLhg4yk#9c) zzs)84xys+~Jy_~FP}(h*diQ$#f~1!=G-4d$!A?My@#GA8~N zlx83+Nzd<6+*0n*ooQ(;*N|pmwBL|BOxw3!Qh{xHjj% zfNg&Vq9wE?Upu(uLz}-Y?s_`!!(Zta$Nn+;x6!Z8etmZJUN zx`|(^B(QNnzjk!_IXO{(%~-F!Vg1#k6u z9)=-R@w9luq!jUlxDF$>I;)kYrW9}qx@!4nW>HI~6H^+cN1>X5ZVj!uYR{dNxhkwy z8CRy;#(rDDi`YI2;-`Xt>a8Zw-z)l$eDmrZ>8m%sexu+&@qh5Iw>lg)!~ZrY8)OCo z=LS@{8hSs)T2GSq2A5s9|J2&4{5;zk=u=s)^&SEB>-Q6vTF0MhS0hh2vUWI&0k}K! z%6ZL9)1(}SkXueo%g@&N`F^%JFh3u*%{U%+95yR6BDOUTz{Dw@cWOviY};~QjKYn(r8WSPF(#1CMarxFIzeKA(MrySW(=>PVU#*#*74)WK;<9Mg`O;>OmK9s~qw>xDT*MQCkQuC7RAWJl>C<`) z9T!G8(Kywj>QYmrQ(I$-POQpX9;il-V@9(Nqv&Rt!?L_~AcX(oY;#`ifB`zLN(`0h za*(OZJqukKZ-QtIP92x~?}hqrPJDTw5E?3mhE}~>0Nk+;m)Dka-p||r)P8gM&Qw7j zDas@H;7DE?VJMD=+wngNJo_k$E$~GPYqj3Sq4MPX#(G`_$qAqKvUfl_Hw-2ns(6wjSqD!@~z#qaUT0 zZ48F#dC0?E#P|dRoB%nlMEdij|32}pu&YGcSJ+h|o)vbL$TRs?zx$p&tKCJ=ehIad%Nt1M^s^h{~rN!ptAq~ diff --git a/bb2gh/__pycache__/syncer.cpython-311.pyc b/bb2gh/__pycache__/syncer.cpython-311.pyc deleted file mode 100644 index 8f715e89d0145717e088bdbaba7e82c532c60eed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6934 zcma(VU5pdQdG^m<+iQF8{|G->0te?DXNRMOvb%<0 z`KnVCwGMKXOGLy;Z6zn7geZBqhd%Jo2f|Yx*JvfIl}lKJaSGe6Jp3(Gt|J-EA(@itEHy`wFFi+- zFEhuGFFVJQFE_`@jLc>|Iq#g8LNrpSCEl4-@!g~l`T}0N%=u-X%)Lj=1!T{AXilKf z43fR?A(_8P10K9>yr4{&Nt3iI`INMv7IMP z{q@p(^jkbK{w-cALRs8-J*g`xcS30O!pp$>@KeBn4N_^Ug38daOzQxhQg?}UTJTq< zWv1Reo7-8Td$-uT5en&e-F8`W4jEPG3MI4f5ML=3b|Y|FH87U6}_b9B~6zLB|T~}&|eM! zYCe-sPfKH(MM)P?{B>}j_l5VlO@{VqCsYNAqQN|EVb(MDsN*gQPy-aQNj0sh_-cUIoNReiMJr`>4Ii8Y<-hx4i-nvL zD`|=vn|GLgauO(BEX~K}=cm$(F)S8}8ZBP2_yiD@fKGKQOaKCov^xM+(1UMXn7DHGQFc~uiW!)H@w?7_|ON!Rlyh_UI-<@P*<2tUWa z#iy&{BP@Gj#ZopKD~Dn^MNKP!^>#_o^s>KHltD@3QDu5kj(7;$s$o2E2mr`iF`Ln| znY^NbJmI>Bh(wS4^rp7Ez!aYf>yqlKKpuvOkf^ zD#?7p-W$xYpsC}~u^nFRp8!@+i)Fa*M&FjT>9u{g;zr-$M&Dtx?{L*q^*q`L44HxP z7NWRt2Q>K*?rbs8`smRkVB)@!`{K@f;?9P+%M^FDkdGT8pvpGI!R!8y{2vO}getcY z2vtk(PuHf`sH;b+N1DQ5Gc>UJN^P;qGy?;xTJ>o4sIeouzRTG0V|ZFL6CBzQM}aK% zR?65ibvtB;GYxUZ6lb6U&!dfj?dHG>Kw1zMbO5Zm$^v!4AxxbYJ$tYfa46Dtz@Fdj zA3`68$C=NlaptS(!DC_iZa92wCwEt(0Hy%LEsmT$JP1wKk%GWGK-e*UQEatVtn8TZF_*NB3PwNI-ks1 z-t)6F7mlAhZG{qxU`n$}qTS(vj#-$JwpdNcE~xupLJf}9&HRwm_rlL;BSuN_TV>ss22Qq;aa{fn|oer@Y5zg zZSd1zLr8P3e!(1jvBAG&@-G?Aqi%zlQ|`^cAG}v`*Ka`OI_mL@9QK5sPJ!E`Hs6Iz z!wtIzfSf23FUQU)DJ63Wn9n{IX-wWHX^SNtT+p;67cS=|V9u1X0Q&4oG#Wh-VN?MY zve-<1p`gZ~3N1gvE#$;YIg49RlQ~7h1UpJdoq|8*A?Fx-F5)nBY0m*zLD;kgt_F<3 z<99AK__)c(4L;r+-HuPP3P1Z~x_9iJj0aku@?=m)9+uow-h5iXjd*TS)aeSUyZ))u ziod<)jzWOv%0)LG%;uha*P1|cCs4qnJ8P~_KgBI?^1~@&&LeEOAA6DdqGHDn*+QWh zEssNZppr`k0qTh_#xr?cQ7+Q}SU?37jMXcoZI}<#7E#QxHFhUyV0}ZKklz5Vqm}C$FAdWB(X1NB1{`DN~p- zgsB$S8S-x)`NB4LjWzf^CcnpU9-FP)3j%7eQX&HPoXu8v2ko-tR5o3Hg|Y)HxI`CQ z&2@WePaH|N{h+xXY?q$ya0&#Q?!kHjHDyrMy)50U`U$gj=-m1=b5`1Pu3x~BYWEvq z%7e3*8Prly&&+3m@$mgei8B<(Uca6JM+hqm@M>_E#pE)1M{#&H1$a^h_eBBNqc>iL zA?2W>H^|VF(4_qy0I*D`zm|FbV(sGEOe3@lTnP9N3_|&){>NlAvL372Dr|-8cfR~N!K)XOZu^Y^NH??jc z?%?3B%#nHFj=RHd@c!+2Ar7hi-0{wPW^>(yp4mJ&9N!$olRS#IJc_wS0(G>>zlxR* z#0#6+q=iB$FGnS;H}{el(?QrPEo4+pkH$&lYbS1cb1BeZNOtzmNo zRvHS47Sx}>|FVd;0s*$G(a%Dw_74CMSc`+#{P)Gld*Wn6jG1Dr$~D7-SSt@~2>n+RhIHtDaQ|N(NvGI@kp^#Nqp*bWfCwori93 zy_0E(XH4;oA)e_Pw(4ndFbG0|E@N=?&hXd64gQSDpD~xZp=72Nttg#IcracqRU zJ4^xW@yT5A_JeTsdLA7E@Pzys>i7U1i(jGs02#Up4Uqm5+benyNCQPq>6-uXqPr>3 z*0y(xYW?wo2|2jlHNl%;&m+ChSrvSKR|q$$DpnupV5LFGKZWcUWclD^Itflt4>-ha zm(9rl=3yPjv+!WgFBUy#6oPtF|u)#)gc5mbug!^QylrQME ze!_{mc!U858&3uq05y}&Jjq+*#+vA<^LEXHw-s4>w_YS_wx z7<%j|w)K&E8so(QSU!@$Aju0z6@oKrd6jIks44RIw9pLTm-~9O5I)81%8a(zpKKWy@c z4gRnj2Tk01>vrVM&l>z`lRs@Z5AikP5kEw`i9Cc_iG<}(Byt701kr865)z5GOUZ1z z$D2sVg;XM;{us`U#9_nVLvFJ9|MEYv5Aukb zf;Q}^X67M1d9Zdw@ob-sgeund*q? vduHv$hHtm&+r8pxc?4=_3-y8>A_X}u7wm~s6h5XWf4;N({`CVWWT*cJ8c*Qq diff --git a/tests/__pycache__/__init__.cpython-311.pyc b/tests/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index ab41a2f51a138b0c4938f687109485a1015449e9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 155 zcmZ3^%ge<81efIwWrFC(AOZ#$p^VRLK*n^26oz01O-8?!3`I;p{%4TnFFpN?{M=Oi z(&E%2{iMv2q|)T<)Dm6Y^vsfs(j@(o)Z&t2{rLFIyv&mLc)fzkUmP~M`6;D2sdh!I ZKqMrM(z3cTlw)WxLvb|Dv{SIt5wi`Q5V8?klZPhk3C2}S0>eV6T zhbjS!!iPmpZBf8`NCBe(E^(^=4yQ`@Jx~HacBFRDk^qzh`%1=C z&)Zx)RGWE6(+xegSSB$er^V9M!g8#b&l1G|O|h(23tHkcZ0$OVM4|3KxajKBzAurgaBIHCn=ZBo%&PkAy ztej!no8#)7y9a=ya|E$@4ss!@R-Ff%25P%65KS?(m;wgOEXOj`K(TUqNy``sFqu!U zXqnmzIm4*vFC~-dypgVEmNg@x-BOAbFnFe1tVI@;Wo<5p8$?^Tx}Y z;$J#(>eMMyxcb%`ug%w{tfm2;tEQdpCY4G)dE-d8`E0J+yjZ+FS0>qbn4l>MTFeZk z({ftTHMvR(rUV(<6Eg@Y=nT~&q(aL0V7eK~<{Ws)FuVvUXxjAZ1~H|ZWwI=*d18jE zx<=%pQmN!iIC~_cSdn2;aZm5XiPSZ8eC%EdcxbatM_ z?2gW^skI4KFmc)i*uEQZm;W&~v3B4W#a|U^rCp$F`^Gz=k(JAj;YM5rg>_gn^znP3 zxyxx>iiiEfi)5cvuNbc1sP9>TP)Ip1OCL z>KeYfPyfAPuQrZb>a9{JTjW&%(Tkq@p#QK}CK~qYf;#rvb?<;TIDNaTZTmj;KHCyE z&R*~RxDv05t`>X9zUSTo<#k&}^?%^4LGK%aDyf0GkmqSmbnY{*mh2s-F1lNC=Y9sW z=oa7bz|RNv=I=SIWsMrgWnusS4eXmAh1_oe9J4KJgg39DaxlXpS8A7(bY_0xNDTM& zWKL65Jy}%prJ$-U9zAxPsVmh&AvsS=gV|jE#K}_*{Q}c(M3cDFCGn+^fZ6yN&ansl zhUd(g2YhW|-+{{Qg}5I4X*Ur&6T{StaXpby3I$l=)shix#Lh7NvAAwZbnQb^54@%q zpePnl9NUO0I&32bZQ=w#L3twX-3V4IDs0$u@GKjiIkh%SAIi~t*tETwn`+!k4npt9 zArMXwdz5(C3=_?$l9GHwDO5Frv(OCox@iVquH42AOICS(NjJqp84$X;Q`U3kYC*-`b zhBtaqF=XvlrJ$4)14_&x_Vlt+x=nBXzz?q?t*}UDu%jWoJ44JnGe@17BP_!jKkm$& z)bVj+>>%UV;uw8aq7}QWj7OnBBoirW$OkMHVAIWPz-WUvFor~fOMWv%BnIr!$1d}wVZfJcckG@q}az&tU2}F z?)iq?lwvLEdRw~Ql&*gz05EWp2{7vBoBkic(%)?QU>)>I*Zty7f=Bfl6!Kc!$|Fdm??Fw`eUx|yR zB-59HESo`DW)2&q!?OI7s#3t$ge()JQ{>wy&VzWs(aLm>CX=W|f%8GnfLO8rIzn*e zS~E1;685!)ee0gEpz!M)%Fn%=_^qC9Hz$T%NKrVj&VlIREDcq&`{P~lhW25xyT=mU zhMvTcDIe}*VpP}1yQt3R=1i*4&zs$B@=3YWrcC^4do$>}Kk;6I>IbeRnZvj|E>ovm zdkcN*+V(yw|F!}2r8e28y+pmg3x&K^GHQGJ3eQ_!Co3nrVHh&j2;h+5iuc0raR;W) zU9Rp~LM=TSB19{I;j2DXP(|Rjo-Ci|Rp|pC_}k(V?ke=?c@A~2qqZLyu)pUkI8Z(Z zN>YP$X$gDK*OnS``6K}g0iL?x9_b%J?M9#xY=qQsBU}&E!(DGAQjh3&H}@=5C<5 zh&o!2cKwVY0|{C8aJ3Y+wDlo(OYU6P&xka_>Nc=s@R=+L=mAS^e@;sV-Tm9rl6^R^ zeKj(ZsC893B2h_XN= zq8A&5C@Xrh<5xEP5c>irs=VR9d^?8wYb|sxhBO77)k@hEGKF$UgYcD}tIar~aA_KH zOJewIKG|&o9axBE`KWv5%Yhpi_koX9sAkC(=#v>zH9bS}73z)&g?wpQugyInSjVDr z&ekuQ961L~QPu_y@@Ur^wjAIi)B$Gi3iW}9?8DoFn|s9N`+}QH_G2p;oH70Sa=ud0 zR8w43@&!#LIF#`rO8m@VArB5KT0%D?SWiGni&HQHG?u^Mel(`_mFC^pTw z^X&W|4z*^_wr9__q^VWu=lj==KbqX#ly#@E+SL1XNU-yvG}*S&OoISO zwX6b$u(MFaG^;ml-GIUdtxee=)+R{j38f5c_k-b`wQPK6VpmhzwR&T13P|C1&KN#Lw%f@%4v9>h(iFEp7 z>GYqz_vd^|y4semHm#F|AM9^3F#T<&`5t4Myn_V_jAhkzye5k2rmmJIuC1a$xo1hXYGS|7$FXd~fIqc?~8xc^yRDL#|>Pso6V}3b9z&JE){2;ayC9 z9|dJglz7?10X~+}(AW>L{KqJsg#*Q-jN4qtiVBJ}3j9_{GAQt4Wgk@`i>O12Lb50* zH9@{>a}t_h#AK32+K}4mvQmIA(&}xSan%m@hZGn)*&K}q(0|D&ieyualjr_+s+qio zXX0Ipg7nzNaf@wy`VGsl-}B$Fj2?|)oWf=eka#AR!75r|P;vp+!*@`;f&z#HM{+2T zOOT5wCQx7;guH4A8l!GOZXoD6)e@%L!gMzTdY~o5+d{mXJJb>mw}r#~;n4}xapT=wVZ*l zuw;E*q_KN*LT2A16U_gDZ-fb=-6VMfzAf4+)8Exm@6PmqALv`s;d7#7XvNARIE-eb zTFM)^-6ZhSEBhA`905~g?VDj1eC+-|2C^tK>W)XUy%V{T>0iv>g0H9=+Y4!cnhuzI z5I3aLEZTjR+=5ydbEKaJ*1bH>cer?yi$8LXYj1>UDzHNTI$UUl{&l#a75dlV4mLMC zA90Zt`uB+2+jO5D?sRjr6TCQ&bY~9)&iPS@oAaYEcAjBNv1gozAo-y99Ccsx3XUdt R82Lf*IqJUHq<{|nzX6%7zMlX9 diff --git a/tests/__pycache__/test_pr_migrator.cpython-311-pytest-9.0.2.pyc b/tests/__pycache__/test_pr_migrator.cpython-311-pytest-9.0.2.pyc deleted file mode 100644 index f788d9c96e6599d1c3d9fba0348ad1f344255b68..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18766 zcmeG^TWlOha10P&)}3-Xgsf-M>%bCM#+MZ*#jJnQ&uC{RFc|>8I}^-?CPR$p$%M0! z$w)Rj8O^p$wq@HV+Z}=&nCmc4DtT`RcYyfNWC)X;wsK!fd6!eTC`jTBLGs^mLQVM5 zHo1+J1fV3?RMO2#LQoQJD%s9TB2W@-D(R7&uL<$CV*eFc)zsK@UWsY5a_rLOST;SQ zBsHM3d8v?*hrYz~PsE+Ne>^#pPL1bN^SWmtsikJ)4&8e$pPNq4#NGNfswXEF3Ykno zk$+r3#dizQC?BUp z$)9!rM>IJl0s1g70sY`s7vxm&saZ{1P)`mIPo=e~LTX;thUDwX>;m*6Tx~Hhot&2s zYWaCNSL`=MXVThiVah5uoLoo`Up-O`Gx3>OqlP!TcrdTb6#c_mc41h9-hCj>UXUJ~no3M1RXI^mGP(!^JGCC1nYFNiAhs7U(65Wv^2&TdN-KJ( zpvnr~orQF625)wX-E0;6IriY#@N7OS52H-udJi6?>tXFv8N@qpK2t*ri+V5%Pf&tX zDe#&KZSYes0azA(6Yl=E3zhIdH9SxY4=i7(_kM5rLe1S>@7}$9;l``=-ctrS$jGjp za2T(5_cLn8;O)gq-yxLPUJte}kKYHKLJEPo1p#5t(DF|5TmQxDDFVe!X%Dbhg)JAn@oKm~gaZ7|}qI2TQXWKJHQM(Tw5Jg_d>#liuQFmXu{OU{a*1FY9VMg~YDET=#rQvIA zsCZG*11PSh;bjN;Y%-mD5SUBmm5vOEn!c|7>}%dH1fik#GBX$-(!K^XjgFAF>|%2A-7f}e_S{<5&{6{79+_E>%UskKnY%H+L3rRyB*>n>+!U|kgY;`M<8 z_Xa=dzV~#c??u?xJ+7|6x_}@c^u=0BJFpZ$K!|j$%-!p&Y#YNpbiN+$T*=%!RoV6e z?jh9Fji=}tT;JOj@~#U6JYLpt+v8;pK==Eg(eDegFzEn80|w@XFzJ$<0Ns)c9uK$P zhQsa!HZopP&ce8u!X7p^9CFxW`YB-cVXx#UIqgIFl1p+dxl8Vn>#DHiDS6cQHH)sf z#vte{d8Wazq2#pl4h(=F@(dD9L?`Lt-GMYgjx25F!Dtaj>WaEGo#|1EhWajO@x zFQ2d%{ad~G4fJAQs~5k4UJP#a;y2KXp{-v026_=gh493K?c*%GlBF0rNwKqLpm5g# zZdLBCqu4RHZ(;FJT#bRxi0F~Hs=G0A0w2%?B?yi@h$L0;5;Z1q6eNIoR6&1989>kv zKzEKFR(1e#*RP;Mp$DE{Sj4a)kyP~dL@LR`v;_qslZB)r>;8nAoR+~O1jipF6H|qB zMoZ^Z-J8jSu&VbY)Y*I?BSEYNF8xd2e8u&`sC~2tmcyF z9o`ke6_qp7%(G*Dzj7K|6~uGV)r?q)!$eT@IBF%1sc73{#zvlFH@mQOE|n>ON1DjW zT2e}C$>QE-KWk`X!Q-G&{U!KA%fc7YT`P+p?5{?T)uPAB?zQN?ia1hY@!(Snd z*k3NLT&YC+0i%6Y?5~+PahQbIUu6YUTrLs?0>t1X=d4`eJQA@mNH0JT;W#10ff`ha zWS~J@nN_|X-BuBwDqmgcD*qG^+E>M=Y9>w`6acKs3aGezl_+2ngOi-I(#3fsVquV8 z(#df`pb{WZgDQ~>G>9v+%GaYk6){#`T!F_Z1{m$DVytH3#93}Bi6tvf}}>2-mFF1P2`PzihL{YzGtf!@qvEAnIjxQqZ7LS+^gf`=O?b5AJg49 z8?E3>y0QyEvHg{NeqN2uOv{T;Axo1Swn+UHMv`S=t-F71Ab#&a zrSBr_^`8Cprw^?A+&vK(k^p#oI1vt@EdI7b=;*)Q^TD50+mF}UkC(k`QE;CI%cYg8 zmFOT~w6BVTH4`TelMn~1tbmHkC89up7@Xvs74WAjmsJHOT{M*e$p620k=@j^IrMGJ+=SgzHxd0(?AGjQOqBDeO78F0=wN zM_6lPbJPbVt>&nC)nYukaG5XGx}b`=ahvojW|M5TXKU4BYt;e1Qd@;q3W+k>UD*=n}YGOW@zm`kj@~}!~!ul$b zm1vg3mtFl=W+uMq+I{=P2lLggleMmsW&aw^M-0Mj!;8OT=tl{2@TCzM=Ob+6IwwKU z80lZ+ycPx@zCncJgb)X6P$giJ8m`PLUq|D$4@}b{%+3Ku`>MFFX5z$Q5+Y2*VF4AR z@iN4S!AYn>oE5l8n}~(MPX;0!Cxkelg0ujW)No}@@<6g-+2-A&AA%$6{f`arenv<; zez!M2BY+$~$TQoyy~n+|ov=9RG$(2NN(v>FWZ&G9^}^Ws=;Y;$fCy8}ogl?m2) zdu_V;da!0zfTMl2Rk7EA^<3IEuR&|SkJujjdNOYhte8J+?BT6wmVwNbEFQw-|}5`;5ho1B`JvuEqmPvC~w1Jg%})R(C==2zuEN(Ze#w zi1HlZh#f{Sir{$Bx*l?Z);nmA%5j#-JL)}@K>(b5U| z>W4-)b^2*eho{992hr(o9nG7>egutDb8IlLM|xKdSHio>&M%@*txVm9<%*$Nbf_YZ zehmKbXeB!I@$S!Xx)b-RIJzp1))-D4Q-lhsghW7=3litupK@LcgS2f{b7iWjriJnF zns7a4J4m^O0X2yS7fDh?ehj{RqwRpxDL5n!$Qrd-zreLxYjI&KV)*5rX1_cQw?ny( zpvgI(K^}s?fM3hhk=ql;xgrCEoSviWLMtF&9hKO8b#qL+e;FQwhOh1vWXJ-tk8+IH zH1aWC)A%8|o0CxQz*!sPyq%hKKhe27k99836P>Fi`PP0Vc4`v*a~QiseLH&O7Z8LxO9uzqOn%`;th?3q5XbK0ui9 z{}mYC_l1&c&Md*?C&!GVG(?qFE0a)Hg{z_5y&NCxd1CBHFo7{-jyQrJj2i)d(!jLoJA zb2iylir6*TDI?`9v;Pe7p>B;aD_N3q5zD;|wNe+54Y`gwOpMN(!8F;Lo z1nHKZj67ja1|G2|o8+J;BY!MCS#mbUi#d#k*>*c(V%DSmU4?HVXbShPvyp{z;RO zrZch>kLn>s)(T24aW$DK$a=?&tQjfiGRs!Ut9m4@HWJaHZbv2!nJ~6*6%-t|EHV8< zy+yr~GMQ~E^ma;QN#ybxOR$lpyHPextb6ghiL+duOjwA)=c|5 zKLLdHRk5#T;>1A#z^Y~eNpKR1h=aK{M%@+$7g}lOLZlO*b%#=t4V9UP>GRq#B zP)hI?UAdqv!GRfLX3UjkB$<8s8Db54{g+6h&5DPWzd`(O5&Rv3GJ?mkxXM334Ruq~ zwu6~AbPd!*G6P>3NsHVEC?0vJrQ4(up6x=#9>MSxDtlh9?s>fiaLe$a<+55L zGEx>h*TVRA3?Es*hggRn{z0Q}^D9J1;DH1+h&)|V0C0#9(~cP@MC{NS+AI;fW!%04 zg&&$C5T{GwlGyl;72bwZXS=6nqpCiS*>zBT4|PpNULKL z$&fvElRAy)KO%KW+e#55`j2W)H?GyDsmTY+L zb*)FKN9vXO;JbWye;c1$8jGFzspN zIHhHkePgbBbA{{_e5(BSSpRV>x!z%f$@mQlMCf^pbt6eh!m}88PF8^de?sgg0NuwvK8FP*)-ULpG@@zcJtTjC zfMsmq3*8h&$LMPlO3*dtD%iXNHp6VXT0HhpcVp9f@4yMw_ns)!hHz-dGi!Sfee!H& z*EsC;oxAG;BajWYvjY|#0C;?1$O}WzCPcb#7Av9M>z#pP0gTl#fd6fe(7E%y{lC~> z_SJiLlzl&s)StUh_ELleY4Q}y|4F(ue^R0F+0VqYRdK8)j#b35Mv(mM=i=$l#M4#r zxtjP~MSKoRwnJLP=i>99iO+x1{p;S!l{YGHPFBUYYT{cJb2pa2e9cQ>=#?n{jNlgt z{vAQnVptJ*2xR!FoeuP;&A+-Qu>5)@xToqFta%0vYJb&ppyoMXP(xMEaLqHk?u>fI z9P0wYuUvw61bBEn2$Fbw2I{YSnapm%8+>G~LC?;00l?-uQKIj0R1SFFhK3^CTxmmw zBx$t|dnw+l`w|H$pMp$v(fkCDC86rQ3Hp9-h=xJ3h1n^%T($`_qRkn+O%j1Mw6k;(uTH4-kQ;K~UcXxbAW|9CcxDMcBJ$eL~D9g^m^B*$wwK zA-v4~)`X`k_I=&yaX8n7-w(72PVvUBb*J9}qbC5kq0Tmw(PI>DbHM+E0Kf~8F%osa z7Xhu{E98CMDA*HmK&oad_zHPnHwt$99k5K?3cf<#*NuXnmmLm#{5C&)MP*+F~f z&aTd9A%i&b;2wo*p^y}M#gvk^+!gxXm$tNU?UNJ)5eo-~au4^W!|h9+`pv9X|EzG= zj#l5!eBU?!zCXX2{cSWFLQsswzZ(AuBJ?lT@QS}zIsFw>o**4%kS^$AQOJn=?KAzF zC?HeH$bu^gWC9@Z6|ou21kF$;WQH^0o-C4yfXrWvnp2r6&XbBUb2>9^#xrpNk;v|h zH?b~1M$e$}p~pa(b0Yda(gTl?j-QDjfxlZa=XG%#C4;rqhs3fiC0C}3T_B29F4;z@ zTCQ4(bx_g>Rm`$pEt1q{tY13mbAtD>c|*Hb*7h8!lC`yhd2bCwt9+KJpr6Tl^@C+m$_N9k{EwS!y zsiRS8T~OJ$yyzmdJcppV09*E@VX1zrLbT7=fJuomZ7KtCe3nfeykDk!s%}sx%1odd zC7aNY=EDg5ts6iZ=+o%j2X~v%*>DnAQ}rP z|D=(0|9G#AG)ZsbjM8}Du^DAu2EPTI5DQf|vT8FsJDd4Js%Oq^J8P5L>-b-IMk97a91v}bjpt>|V$4Am=_QO72J(m1G zk0tN+_jim3s~FVuz&QQ@k_Ex3q1r{}RE{-9DX(M|3tmN{P*N$UYVrFI-g;A2&4V@G z25&+V`Wf?OpvQC|qa7(n7!b*2YUGVlEwHv$0q+w?bf31bgMzwBsvBatBYn!1K@XdF||myDb?V|aP!ultXOXXy7DPhXv#K4LA|4|rw2KuSuf%!A!h zz_*GQXvzVxQ|Y9iDlia7KjUf5H{TJymAJ+gZ1EKOg^)Uz^?3#a@0Q!@r5HJ-TyoJ;$z_eAm8!*D%w)NKGX+pG%-Cd%q{v~< zk+{l8Ycs9&&oWpN^A^jX(?>wyCMyhI(CTBCiMJ8GSVy{;0A07}zCAt~joCv9qWeeB zrgsIM?NUzD4d_N$54r%P7EKeIUBwi_p_+{gi{;Q7{N6hnsn|xj#Lj~(&0F*mhD8KBvEJvG$z4L0uAfizGjHuu5u zQ|%YmetE4KTY~4<{@r(fv;W7{=c`-I)vZtF7LQ}CxsCSRMss?jb8hyhn>+#BL*^!1 zzK~90x(KzQy#5Aj_LG~gu;y2pa>z~dRmK?PIzI>`b`acj_-oTme?APyfDm-(odEE1 zjgjC$Flp1Zl`mtO*I-QR4?qA8Fp4Ld7jLwtZyZS-9DP6hUiin+pG1$Mo$2|BeDAzC ziCpePE_Ws0t&j&J-H0zZ*F~cwgp}*-D7o|nAj6K53&ZdqXyN;9e81^E-27>{Ibu~U zWlf^0P6%!j{DTJS5mo(BHCuEglm$Cw*Ke9)~k6Z_`bCj~vk(2XKvHk?4(f@U% zP~(Tq@M23++LF?1Ew!ZOwzS+8dHkE)GNlaPJ@(=d8=nt{yKaQDTwl$><}s;ZYYFbLkM8b$d0vz*9xDhF!6l~5}O zEPzt({RPr4Mae!jY~lp_EGOXcLeQDVTb4Rv*>a-Ql40}y?0VyVZgFd~lfjW)Whdg^ z-g_TJ3@kmCd$n Date: Sat, 28 Mar 2026 11:30:07 +0000 Subject: [PATCH 03/74] Add flexible Bitbucket-to-GitHub project/repo mapping Support mapping Bitbucket projects to different GitHub orgs and customizing repo names via templates and per-repo overrides in config. - Config.resolve_target() resolves (bb_project, bb_slug) -> (gh_org, gh_repo_name) - GithubClient now supports multi-org operations via per-method org_name param - State tracks the resolved GitHub org/repo for each migrated repo - PR migrator uses stored mapping with fallback to config resolution https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 42 ++++++++++ bb2gh/github_client.py | 53 ++++++++----- bb2gh/migrator.py | 30 ++++--- bb2gh/pr_migrator.py | 37 ++++++--- bb2gh/state.py | 25 +++++- bb2gh/syncer.py | 9 ++- config.yaml.example | 31 +++++++- tests/test_config.py | 162 ++++++++++++++++++++++++++++++++++++++ tests/test_migrator.py | 50 +++++++++++- tests/test_pr_migrator.py | 66 +++++++++++++++- tests/test_syncer.py | 20 +++++ 11 files changed, 471 insertions(+), 54 deletions(-) create mode 100644 tests/test_config.py diff --git a/bb2gh/config.py b/bb2gh/config.py index 8388141..7b6e619 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -35,6 +35,48 @@ def __init__(self, path="config.yaml"): # User mapping (Bitbucket username -> GitHub username) self.user_mapping = raw.get("user_mapping", {}) + # Repository mapping (Bitbucket project/repo -> GitHub org/repo) + rm = raw.get("repo_mapping", {}) + self._repo_mapping = rm + self._name_template = rm.get("name_template", "{slug}") + self._project_mappings = rm.get("projects", {}) + + def resolve_target(self, project_key, repo_slug): + """Resolve a Bitbucket project/repo to a GitHub org and repo name. + + Lookup order: + 1. Explicit per-repo override in repo_mapping.projects..repos..github_name + 2. Per-project name_template override in repo_mapping.projects..name_template + 3. Global name_template from repo_mapping.name_template (default: "{slug}") + + For the org: + 1. Per-project github_org in repo_mapping.projects..github_org + 2. Global github.org + + Returns: + (github_org, github_repo_name) tuple + """ + project_conf = self._project_mappings.get(project_key, {}) + + # Resolve org + gh_org = project_conf.get("github_org", self.gh_org) + + # Resolve repo name: check explicit per-repo override first + repos_conf = project_conf.get("repos", {}) + if repo_slug in repos_conf: + repo_conf = repos_conf[repo_slug] + gh_repo = repo_conf.get("github_name", repo_slug) + else: + # Use per-project template, falling back to global template + template = project_conf.get("name_template", self._name_template) + gh_repo = template.format( + project=project_key, + project_lower=project_key.lower(), + slug=repo_slug, + ) + + return gh_org, gh_repo + @staticmethod def _validate(raw): for section in ("bitbucket", "github"): diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py index 2018d8b..c73f77e 100644 --- a/bb2gh/github_client.py +++ b/bb2gh/github_client.py @@ -7,38 +7,52 @@ class GithubClient: - """Wrapper around PyGithub for GitHub Enterprise operations.""" + """Wrapper around PyGithub for GitHub Enterprise operations. - def __init__(self, base_url, token, org_name): + Supports multiple GitHub organizations. Each method accepts an + org_name parameter to target the correct org. + """ + + def __init__(self, base_url, token, default_org): self.gh = Github(base_url=base_url, login_or_token=token) - self.org = self.gh.get_organization(org_name) - self.org_name = org_name + self.default_org = default_org + self._org_cache = {} + + def _get_org(self, org_name=None): + """Get a GitHub organization object, with caching.""" + org_name = org_name or self.default_org + if org_name not in self._org_cache: + self._org_cache[org_name] = self.gh.get_organization(org_name) + return self._org_cache[org_name] - def create_repo(self, name, description="", private=True): + def create_repo(self, name, description="", private=True, org_name=None): """Create a repository in the organization. Returns the repo object. If the repo already exists, returns the existing one. """ + org = self._get_org(org_name) + actual_org = org_name or self.default_org try: - repo = self.org.create_repo( + repo = org.create_repo( name=name, description=description, private=private, auto_init=False, ) - logger.info("Created GitHub repo: %s/%s", self.org_name, name) + logger.info("Created GitHub repo: %s/%s", actual_org, name) return repo except GithubException as e: if e.status == 422: # Already exists - logger.info("GitHub repo already exists: %s/%s", self.org_name, name) - return self.org.get_repo(name) + logger.info("GitHub repo already exists: %s/%s", actual_org, name) + return org.get_repo(name) raise - def get_repo(self, name): + def get_repo(self, name, org_name=None): """Get an existing repository.""" - return self.org.get_repo(name) + org = self._get_org(org_name) + return org.get_repo(name) - def create_pull_request(self, repo_name, title, body, head, base): + def create_pull_request(self, repo_name, title, body, head, base, org_name=None): """Create a pull request on a GitHub repository. Args: @@ -47,22 +61,23 @@ def create_pull_request(self, repo_name, title, body, head, base): body: PR body/description (markdown). head: Source branch name. base: Target branch name. + org_name: Target GitHub org (defaults to default_org). Returns the created PR object. """ - repo = self.org.get_repo(repo_name) + repo = self.get_repo(repo_name, org_name) pr = repo.create_pull(title=title, body=body, head=head, base=base) logger.info("Created PR #%d on %s: %s", pr.number, repo_name, title) return pr - def add_pr_comment(self, repo_name, pr_number, body): + def add_pr_comment(self, repo_name, pr_number, body, org_name=None): """Add a comment to a pull request.""" - repo = self.org.get_repo(repo_name) + repo = self.get_repo(repo_name, org_name) pr = repo.get_pull(pr_number) comment = pr.create_issue_comment(body) return comment - def add_pr_reviewers(self, repo_name, pr_number, reviewers): + def add_pr_reviewers(self, repo_name, pr_number, reviewers, org_name=None): """Request reviewers on a pull request. Args: @@ -70,7 +85,7 @@ def add_pr_reviewers(self, repo_name, pr_number, reviewers): """ if not reviewers: return - repo = self.org.get_repo(repo_name) + repo = self.get_repo(repo_name, org_name) pr = repo.get_pull(pr_number) try: pr.create_review_request(reviewers=reviewers) @@ -80,7 +95,7 @@ def add_pr_reviewers(self, repo_name, pr_number, reviewers): "Failed to add reviewers to PR #%d: %s", pr_number, e ) - def get_clone_url(self, repo_name): + def get_clone_url(self, repo_name, org_name=None): """Get the HTTPS clone URL for a repo.""" - repo = self.org.get_repo(repo_name) + repo = self.get_repo(repo_name, org_name) return repo.clone_url diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 3409bad..05ef2c0 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -50,11 +50,12 @@ def migrate_repos(config): """Run the full bulk migration. For each repo in Bitbucket: - 1. Create the repo on GitHub - 2. Bare-clone from Bitbucket via SSH - 3. Clean hidden refs - 4. Push --mirror to GitHub - 5. Record in state + 1. Resolve the target GitHub org and repo name via config mapping + 2. Create the repo on GitHub + 3. Bare-clone from Bitbucket via SSH + 4. Clean hidden refs + 5. Push --mirror to GitHub + 6. Record in state """ bb = BitbucketClient(config.bb_base_url, config.bb_token) gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) @@ -99,11 +100,16 @@ def migrate_repos(config): def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_name, repo): """Migrate a single repository.""" - logger.info("Migrating %s/%s ...", project_key, repo_slug) + # Resolve target GitHub org and repo name + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + logger.info( + "Migrating %s/%s -> %s/%s ...", + project_key, repo_slug, gh_org, gh_repo_name, + ) - # 1. Create repo on GitHub + # 1. Create repo on GitHub (in the resolved org) description = repo.get("description", "") or f"Migrated from Bitbucket: {project_key}/{repo_slug}" - gh.create_repo(repo_slug, description=description, private=True) + gh.create_repo(gh_repo_name, description=description, private=True, org_name=gh_org) # 2. Bare clone from Bitbucket bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") @@ -125,7 +131,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _clean_hidden_refs(bare_path) # 4. Add GitHub remote and push - gh_clone_url = gh.get_clone_url(repo_slug) + gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org) # Remove existing github remote if present, then add try: @@ -136,6 +142,6 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) _run_git(["push", "--mirror", "github"], cwd=bare_path) - # 5. Record in state - state.mark_migrated(project_key, repo_slug) - logger.info("Successfully migrated %s/%s", project_key, repo_slug) + # 5. Record in state (includes the resolved GitHub org and repo name) + state.mark_migrated(project_key, repo_slug, gh_org=gh_org, gh_repo_name=gh_repo_name) + logger.info("Successfully migrated %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py index 3387a7e..870f990 100644 --- a/bb2gh/pr_migrator.py +++ b/bb2gh/pr_migrator.py @@ -73,14 +73,20 @@ def migrate_pull_requests(config, dry_run=False): migrated_repos = state.get_migrated_repos() if not migrated_repos: logger.warning("No migrated repos found. Run 'bb2gh migrate' first.") - return + return 0, 0, 0 total_migrated = 0 total_skipped = 0 total_failed = 0 for project_key, repo_slug in migrated_repos: - logger.info("Processing PRs for %s/%s", project_key, repo_slug) + # Look up the GitHub target for this repo + gh_org, gh_repo_name = state.get_github_target(project_key, repo_slug) + if not gh_org or not gh_repo_name: + # Fallback: resolve from config (for repos migrated before mapping was added) + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + + logger.info("Processing PRs for %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) try: open_prs = bb.list_pull_requests(project_key, repo_slug, state="OPEN") @@ -102,15 +108,16 @@ def migrate_pull_requests(config, dry_run=False): if dry_run: logger.info( - "[DRY RUN] Would migrate PR #%d: %s (%s -> %s)", - pr_id, title, head_branch, base_branch, + "[DRY RUN] Would migrate PR #%d: %s (%s -> %s) to %s/%s", + pr_id, title, head_branch, base_branch, gh_org, gh_repo_name, ) total_migrated += 1 continue try: _migrate_single_pr( - config, bb, gh, state, project_key, repo_slug, pr + config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, ) total_migrated += 1 except Exception: @@ -126,23 +133,27 @@ def migrate_pull_requests(config, dry_run=False): return total_migrated, total_skipped, total_failed -def _migrate_single_pr(config, bb, gh, state, project_key, repo_slug, pr): +def _migrate_single_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr): """Migrate a single pull request.""" pr_id = pr["id"] title = pr["title"] head_branch = pr["fromRef"]["displayId"] base_branch = pr["toRef"]["displayId"] - logger.info("Migrating PR #%d: %s (%s -> %s)", pr_id, title, head_branch, base_branch) + logger.info( + "Migrating PR #%d: %s (%s -> %s) to %s/%s", + pr_id, title, head_branch, base_branch, gh_org, gh_repo_name, + ) - # Create PR on GitHub + # Create PR on GitHub (in the correct org) body = _format_pr_body(pr, config) gh_pr = gh.create_pull_request( - repo_name=repo_slug, + repo_name=gh_repo_name, title=title, body=body, head=head_branch, base=base_branch, + org_name=gh_org, ) # Migrate comments @@ -152,18 +163,18 @@ def _migrate_single_pr(config, bb, gh, state, project_key, repo_slug, pr): action = activity.get("action", "") if action == "COMMENTED" and "comment" in activity: comment_body = _format_comment(activity, config) - gh.add_pr_comment(repo_slug, gh_pr.number, comment_body) + gh.add_pr_comment(gh_repo_name, gh_pr.number, comment_body, org_name=gh_org) comment_count += 1 # Assign reviewers (best effort) reviewers = _map_reviewers(pr, config) if reviewers: - gh.add_pr_reviewers(repo_slug, gh_pr.number, reviewers) + gh.add_pr_reviewers(gh_repo_name, gh_pr.number, reviewers, org_name=gh_org) # Record mapping state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) logger.info( - "Migrated PR #%d -> GitHub PR #%d (%d comments, %d reviewers)", - pr_id, gh_pr.number, comment_count, len(reviewers), + "Migrated PR #%d -> GitHub PR #%d on %s/%s (%d comments, %d reviewers)", + pr_id, gh_pr.number, gh_org, gh_repo_name, comment_count, len(reviewers), ) diff --git a/bb2gh/state.py b/bb2gh/state.py index c647114..799be8c 100644 --- a/bb2gh/state.py +++ b/bb2gh/state.py @@ -31,19 +31,28 @@ def _save(self): def _now(self): return datetime.now(timezone.utc).isoformat() - def mark_migrated(self, project_key, repo_slug): - """Record that a repo has been migrated.""" + def mark_migrated(self, project_key, repo_slug, gh_org=None, gh_repo_name=None): + """Record that a repo has been migrated. + + Args: + project_key: Bitbucket project key. + repo_slug: Bitbucket repo slug. + gh_org: GitHub organization the repo was migrated to. + gh_repo_name: GitHub repository name. + """ key = f"{project_key}/{repo_slug}" self._data["repos"][key] = { "project_key": project_key, "repo_slug": repo_slug, + "gh_org": gh_org, + "gh_repo_name": gh_repo_name or repo_slug, "status": "migrated", "migrated_at": self._now(), "last_sync": self._now(), "pr_mappings": {}, } self._save() - logger.info("Marked %s as migrated", key) + logger.info("Marked %s as migrated -> %s/%s", key, gh_org, gh_repo_name) def update_sync_time(self, project_key, repo_slug): """Update the last sync timestamp for a repo.""" @@ -67,6 +76,16 @@ def get_migrated_repos(self): result.append((entry["project_key"], entry["repo_slug"])) return result + def get_github_target(self, project_key, repo_slug): + """Get the GitHub org and repo name for a migrated repo. + + Returns: + (gh_org, gh_repo_name) tuple, or (None, None) if not found. + """ + key = f"{project_key}/{repo_slug}" + entry = self._data["repos"].get(key, {}) + return entry.get("gh_org"), entry.get("gh_repo_name") + def is_migrated(self, project_key, repo_slug): """Check if a repo has been migrated.""" key = f"{project_key}/{repo_slug}" diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index de4e90a..6f5c1c7 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -110,6 +110,10 @@ def _sync_repo(self, project_key, repo_slug): logger.error("Bare repo not found: %s", bare_path) return + # Look up the GitHub target from state (set during migration) + gh_org, gh_repo_name = self.state.get_github_target(project_key, repo_slug) + target_label = f"{gh_org}/{gh_repo_name}" if gh_org else "github" + start = time.time() # Fetch from Bitbucket (origin) @@ -123,4 +127,7 @@ def _sync_repo(self, project_key, repo_slug): elapsed = time.time() - start self.state.update_sync_time(project_key, repo_slug) - logger.info("Synced %s/%s in %.1fs", project_key, repo_slug, elapsed) + logger.info( + "Synced %s/%s -> %s in %.1fs", + project_key, repo_slug, target_label, elapsed, + ) diff --git a/config.yaml.example b/config.yaml.example index 4eccded..50a0ace 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,7 +1,7 @@ bitbucket: # Bitbucket Server base URL (no trailing slash) base_url: "https://bitbucket.mycompany.com" - # Personal access token for REST API calls + # Personal access token for REST API calls (or set BB_TOKEN env var) token: "YOUR_BITBUCKET_TOKEN" # SSH base URL for git clone operations ssh_url: "ssh://git@bitbucket.mycompany.com:7999" @@ -13,9 +13,9 @@ bitbucket: github: # GitHub Enterprise API base URL base_url: "https://github.mycompany.com/api/v3" - # Personal access token with repo + admin:org scopes + # Personal access token with repo + admin:org scopes (or set GH_TOKEN env var) token: "YOUR_GITHUB_TOKEN" - # Target organization on GitHub + # Default target organization on GitHub (used when no project-specific mapping exists) org: "my-org" sync: @@ -24,8 +24,31 @@ sync: # Local directory for bare repo clones work_dir: "/data/mirror" +# Optional: map Bitbucket projects/repos to specific GitHub orgs/repo names. +# Without this section, all repos go to github.org with their original slug as name. +repo_mapping: + # Default naming template for GitHub repos. + # Available variables: {project}, {project_lower}, {slug} + # Default: "{slug}" (just the Bitbucket repo slug) + name_template: "{project_lower}-{slug}" + + # Per-project overrides + projects: + INFRA: + # Send INFRA repos to a different GitHub org + github_org: "infra-team" + # Optional: override the name template for this project only + # name_template: "{slug}" + repos: + # Optional: explicit per-repo name overrides + legacy-monolith: + github_name: "infra-monolith" + PLATFORM: + github_org: "platform-eng" + # Projects not listed here use github.org and the global name_template + # Optional: map Bitbucket usernames to GitHub usernames -# Used for PR reviewer assignments +# Used for PR author attribution and reviewer assignments # user_mapping: # bb_user1: gh_user1 # bb_user2: gh_user2 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..16ab937 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,162 @@ +"""Tests for the config module, especially repo_mapping resolution.""" + +import os +import tempfile + +import pytest +import yaml + +from bb2gh.config import Config + + +def _write_config(tmp_path, data): + """Write a config dict to a YAML file and return the path.""" + path = tmp_path / "config.yaml" + with open(path, "w") as f: + yaml.dump(data, f) + return str(path) + + +@pytest.fixture +def base_config(): + """Minimal valid config dict.""" + return { + "bitbucket": { + "base_url": "https://bitbucket.example.com", + "ssh_url": "ssh://git@bitbucket.example.com:7999", + "token": "fake", + }, + "github": { + "base_url": "https://github.example.com/api/v3", + "org": "default-org", + "token": "fake", + }, + } + + +class TestResolveTarget: + def test_defaults_to_slug_and_default_org(self, tmp_path, base_config): + """Without repo_mapping, returns default org and slug as-is.""" + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("PROJ", "my-repo") + assert org == "default-org" + assert name == "my-repo" + + def test_global_name_template(self, tmp_path, base_config): + """Global name_template applies to all repos.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("INFRA", "my-service") + assert org == "default-org" + assert name == "infra-my-service" + + def test_per_project_org(self, tmp_path, base_config): + """Per-project github_org overrides the default org.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": {"github_org": "infra-team"}, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("INFRA", "my-service") + assert org == "infra-team" + assert name == "my-service" # default template is "{slug}" + + def test_per_project_name_template(self, tmp_path, base_config): + """Per-project name_template overrides the global template.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + "projects": { + "INFRA": { + "github_org": "infra-team", + "name_template": "infra-{slug}", + }, + }, + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # INFRA uses its own template + org, name = config.resolve_target("INFRA", "my-service") + assert org == "infra-team" + assert name == "infra-my-service" + + # Other projects use the global template + org, name = config.resolve_target("PLATFORM", "api") + assert org == "default-org" + assert name == "platform-api" + + def test_per_repo_override(self, tmp_path, base_config): + """Explicit per-repo github_name overrides all templates.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + "projects": { + "INFRA": { + "github_org": "infra-team", + "repos": { + "legacy-monolith": {"github_name": "the-monolith"}, + }, + }, + }, + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # Explicit override + org, name = config.resolve_target("INFRA", "legacy-monolith") + assert org == "infra-team" + assert name == "the-monolith" + + # Non-overridden repo in same project uses global template + # (no per-project template set, so falls back to global) + org, name = config.resolve_target("INFRA", "other-repo") + assert org == "infra-team" + assert name == "infra-other-repo" + + def test_unmapped_project_uses_defaults(self, tmp_path, base_config): + """Projects not listed in repo_mapping use the default org and global template.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + "projects": { + "INFRA": {"github_org": "infra-team"}, + }, + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("OTHER", "some-repo") + assert org == "default-org" + assert name == "other-some-repo" + + def test_template_with_project_variable(self, tmp_path, base_config): + """Template can use {project} (original case).""" + base_config["repo_mapping"] = { + "name_template": "{project}-{slug}", + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("MyProject", "api") + assert name == "MyProject-api" + + +class TestConfigValidation: + def test_missing_bitbucket_section(self, tmp_path): + data = {"github": {"base_url": "x", "org": "y"}} + path = _write_config(tmp_path, data) + with pytest.raises(ValueError, match="bitbucket"): + Config(path) + + def test_missing_github_section(self, tmp_path): + data = {"bitbucket": {"base_url": "x", "ssh_url": "y"}} + path = _write_config(tmp_path, data) + with pytest.raises(ValueError, match="github"): + Config(path) diff --git a/tests/test_migrator.py b/tests/test_migrator.py index e55af75..9ca29ac 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -24,6 +24,8 @@ def mock_config(tmp_path): config.gh_org = "my-org" config.work_dir = str(tmp_path) config.user_mapping = {} + # Default resolve_target returns the default org with slug as name + config.resolve_target = MagicMock(side_effect=lambda proj, slug: ("my-org", slug)) return config @@ -83,7 +85,53 @@ def test_migrates_new_repo(self, mock_git, MockBB, MockGH, MockState, mock_confi assert migrated == 1 assert skipped == 0 assert failed == 0 - state_instance.mark_migrated.assert_called_once_with("PROJ1", "my-repo") + # resolve_target should be called to determine the GitHub org and repo name + mock_config.resolve_target.assert_called_once_with("PROJ1", "my-repo") + # State should record with org and repo name + state_instance.mark_migrated.assert_called_once_with( + "PROJ1", "my-repo", gh_org="my-org", gh_repo_name="my-repo" + ) + + @patch("bb2gh.migrator.State") + @patch("bb2gh.migrator.GithubClient") + @patch("bb2gh.migrator.BitbucketClient") + @patch("bb2gh.migrator._run_git") + def test_migrates_to_mapped_org(self, mock_git, MockBB, MockGH, MockState, mock_config): + """Test that repos are migrated to the correct org when mapping is configured.""" + # Override resolve_target to return a different org + mock_config.resolve_target = MagicMock( + side_effect=lambda proj, slug: ("infra-team", f"infra-{slug}") + ) + + bb_instance = MockBB.return_value + bb_instance.list_repos.return_value = [ + { + "slug": "my-repo", + "name": "My Repo", + "description": "A test repo", + "links": {"clone": [{"name": "ssh", "href": "ssh://git@bb:7999/proj1/my-repo.git"}]}, + } + ] + + gh_instance = MockGH.return_value + gh_instance.get_clone_url.return_value = "https://github.example.com/infra-team/infra-my-repo.git" + + state_instance = MockState.return_value + state_instance.is_migrated.return_value = False + + mock_git.return_value = "" + + migrated, skipped, failed = migrate_repos(mock_config) + + assert migrated == 1 + # GitHub repo should be created with the mapped name and org + gh_instance.create_repo.assert_called_once_with( + "infra-my-repo", description="A test repo", private=True, org_name="infra-team" + ) + gh_instance.get_clone_url.assert_called_once_with("infra-my-repo", org_name="infra-team") + state_instance.mark_migrated.assert_called_once_with( + "PROJ1", "my-repo", gh_org="infra-team", gh_repo_name="infra-my-repo" + ) @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") diff --git a/tests/test_pr_migrator.py b/tests/test_pr_migrator.py index 94292fa..b339ad1 100644 --- a/tests/test_pr_migrator.py +++ b/tests/test_pr_migrator.py @@ -23,6 +23,7 @@ def mock_config(): config.gh_org = "my-org" config.work_dir = "/tmp/test" config.user_mapping = {"john.doe": "johndoe"} + config.resolve_target = MagicMock(return_value=("my-org", "my-repo")) return config @@ -105,6 +106,7 @@ class TestMigratePullRequests: def test_dry_run(self, MockBB, MockGH, MockState, mock_config, sample_pr): state_instance = MockState.return_value state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") state_instance.is_pr_migrated.return_value = False bb_instance = MockBB.return_value @@ -125,6 +127,7 @@ def test_dry_run(self, MockBB, MockGH, MockState, mock_config, sample_pr): def test_skips_already_migrated(self, MockBB, MockGH, MockState, mock_config, sample_pr): state_instance = MockState.return_value state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") state_instance.is_pr_migrated.return_value = True bb_instance = MockBB.return_value @@ -141,6 +144,7 @@ def test_skips_already_migrated(self, MockBB, MockGH, MockState, mock_config, sa def test_migrates_pr_with_comments(self, MockBB, MockGH, MockState, mock_config, sample_pr): state_instance = MockState.return_value state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") state_instance.is_pr_migrated.return_value = False bb_instance = MockBB.return_value @@ -166,11 +170,71 @@ def test_migrates_pr_with_comments(self, MockBB, MockGH, MockState, mock_config, assert migrated == 1 assert failed == 0 - # Verify PR was created + # Verify PR was created with correct org MockGH.return_value.create_pull_request.assert_called_once() + call_kwargs = MockGH.return_value.create_pull_request.call_args + assert call_kwargs[1]["org_name"] == "my-org" + assert call_kwargs[1]["repo_name"] == "my-repo" # Verify comment was added (only 1 — the APPROVED activity is skipped) MockGH.return_value.add_pr_comment.assert_called_once() # Verify state recorded state_instance.record_pr_mapping.assert_called_once_with("PROJ", "my-repo", 42, 99) + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_uses_mapped_org_for_pr(self, MockBB, MockGH, MockState, mock_config, sample_pr): + """Test that PRs are created in the correct mapped org.""" + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + # Simulate a repo migrated to a different org + state_instance.get_github_target.return_value = ("infra-team", "infra-my-repo") + state_instance.is_pr_migrated.return_value = False + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.return_value = [sample_pr] + bb_instance.get_pr_activities.return_value = [] + + mock_pr = MagicMock() + mock_pr.number = 5 + MockGH.return_value.create_pull_request.return_value = mock_pr + + migrated, skipped, failed = migrate_pull_requests(mock_config, dry_run=False) + + assert migrated == 1 + + # Verify PR was created in the mapped org with the mapped repo name + call_kwargs = MockGH.return_value.create_pull_request.call_args + assert call_kwargs[1]["org_name"] == "infra-team" + assert call_kwargs[1]["repo_name"] == "infra-my-repo" + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_fallback_to_config_resolve(self, MockBB, MockGH, MockState, mock_config, sample_pr): + """Test fallback to config.resolve_target when state has no GitHub target.""" + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + # Simulate old state without gh_org/gh_repo_name + state_instance.get_github_target.return_value = (None, None) + state_instance.is_pr_migrated.return_value = False + + mock_config.resolve_target.return_value = ("fallback-org", "fallback-repo") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.return_value = [sample_pr] + bb_instance.get_pr_activities.return_value = [] + + mock_pr = MagicMock() + mock_pr.number = 10 + MockGH.return_value.create_pull_request.return_value = mock_pr + + migrated, _, _ = migrate_pull_requests(mock_config, dry_run=False) + + assert migrated == 1 + mock_config.resolve_target.assert_called_once_with("PROJ", "my-repo") + call_kwargs = MockGH.return_value.create_pull_request.call_args + assert call_kwargs[1]["org_name"] == "fallback-org" + assert call_kwargs[1]["repo_name"] == "fallback-repo" diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 329c7fd..f8c2c91 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -28,6 +28,7 @@ def test_sync_repo(self, mock_git, MockState, mock_config, tmp_path): state_instance = MockState.return_value state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") mock_git.return_value = "" @@ -62,6 +63,7 @@ def test_sync_handles_failure(self, mock_git, MockState, mock_config, tmp_path): ("PROJ", "repo1"), ("PROJ", "repo2"), ] + state_instance.get_github_target.return_value = ("my-org", "repo1") # First repo fetch fails, second succeeds def side_effect(args, cwd=None): @@ -76,3 +78,21 @@ def side_effect(args, cwd=None): # repo2 should still be synced state_instance.update_sync_time.assert_called_once_with("PROJ", "repo2") + + @patch("bb2gh.syncer.State") + @patch("bb2gh.syncer._run_git") + def test_sync_logs_github_target(self, mock_git, MockState, mock_config, tmp_path): + """Test that sync uses the stored GitHub target for logging.""" + bare_path = tmp_path / "INFRA__my-service.git" + bare_path.mkdir() + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("INFRA", "my-service")] + state_instance.get_github_target.return_value = ("infra-team", "infra-my-service") + + mock_git.return_value = "" + + syncer = Syncer(mock_config) + syncer._sync_all() + + state_instance.get_github_target.assert_called_once_with("INFRA", "my-service") From be5d7e4eb07a4b9674aa9ffb64c38141706a02d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 28 Mar 2026 11:43:01 +0000 Subject: [PATCH 04/74] Rewrite README with step-by-step getting started guide Covers prerequisites, Bitbucket access requirements, SSH setup, config with repo mapping examples, migration workflow, Docker usage, CLI reference, full configuration table, and troubleshooting. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- README.md | 348 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 310 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index b977a76..02f0b6f 100644 --- a/README.md +++ b/README.md @@ -4,86 +4,348 @@ A Python CLI tool for migrating repositories and pull requests from self-hosted ## Features -- **Bulk migration** — Clone all repos from Bitbucket, push to GitHub (branches, tags, full history) -- **Continuous sync** — Fetch from Bitbucket and push to GitHub every 60 seconds +- **Bulk migration** — Clone all repos from Bitbucket and push to GitHub with full history, branches, and tags +- **Continuous sync** — Keep repos in lockstep by fetching from Bitbucket and pushing to GitHub on a configurable interval - **PR migration** — Recreate open Bitbucket PRs on GitHub with title, description, comments, and reviewers +- **Project-to-org mapping** — Route Bitbucket projects to different GitHub orgs with flexible repo naming - **Idempotent** — Safe to re-run; skips already-migrated repos and PRs -- **Docker support** — Run the sync as a long-lived service +- **Docker support** — Run as a one-shot command or a long-lived sync service -## Quick Start +## Prerequisites -### 1. Configure +| Requirement | Details | +|---|---| +| **Python** | 3.9+ | +| **Git** | Installed and on `PATH` | +| **Bitbucket Server** | A service account with **Project READ** access on each project you want to migrate | +| **GitHub Enterprise** | A personal access token with `repo` + `admin:org` scopes | +| **SSH key** | The machine running bb2gh needs SSH access to Bitbucket for `git clone` | + +### Bitbucket Server Access + +You do **not** need admin access to Bitbucket. Request a service account from your Bitbucket admin with: + +- **Project READ** on every project you want to migrate + +That single permission covers: +- Cloning repos via SSH +- Listing repos via REST API +- Reading pull requests, comments, and reviewer info + +### GitHub Enterprise Access + +You need a personal access token (PAT) with these scopes: + +- `repo` — full control of private repositories +- `admin:org` — needed to create repos in organizations + +If you are migrating to multiple GitHub orgs, the token must have access to all of them. + +--- + +## Getting Started + +### Step 1: Clone this repo ```bash -cp config.yaml.example config.yaml -# Edit config.yaml with your Bitbucket and GitHub credentials +git clone +cd bitbucket--github ``` -### 2. Install +### Step 2: Install dependencies ```bash pip install -r requirements.txt pip install -e . ``` -### 3. Migrate +Verify the install: + +```bash +bb2gh --help +``` + +### Step 3: Set up SSH access to Bitbucket + +The tool clones repos from Bitbucket via SSH. Make sure the machine running bb2gh can reach your Bitbucket Server over SSH: + +```bash +# Test connectivity (use your actual Bitbucket SSH host and port) +ssh -T git@bitbucket.mycompany.com -p 7999 +``` + +If you are using a non-default SSH key, configure it in `~/.ssh/config`: + +``` +Host bitbucket.mycompany.com + IdentityFile ~/.ssh/bb_migration_key + Port 7999 +``` + +### Step 4: Create your config file + +```bash +cp config.yaml.example config.yaml +``` + +Edit `config.yaml` with your actual values. At minimum, fill in: + +```yaml +bitbucket: + base_url: "https://bitbucket.mycompany.com" + token: "YOUR_BITBUCKET_TOKEN" + ssh_url: "ssh://git@bitbucket.mycompany.com:7999" + # Optional: limit to specific projects (omit to migrate all accessible projects) + projects: + - PROJ1 + - PROJ2 + +github: + base_url: "https://github.mycompany.com/api/v3" + token: "YOUR_GITHUB_TOKEN" + org: "my-org" +``` + +Alternatively, set tokens via environment variables instead of putting them in the file: ```bash -# Bulk migrate all repos +export BB_TOKEN="your-bitbucket-token" +export GH_TOKEN="your-github-token" +``` + +### Step 5: Configure project-to-org mapping (optional) + +If your Bitbucket projects should land in different GitHub organizations, or you want to control repo naming, add the `repo_mapping` section: + +```yaml +repo_mapping: + # Default naming template for GitHub repos + # Available variables: {project}, {project_lower}, {slug} + name_template: "{project_lower}-{slug}" + + # Per-project overrides + projects: + INFRA: + github_org: "infra-team" # INFRA repos → infra-team org + repos: + legacy-monolith: + github_name: "infra-monolith" # Explicit rename for one repo + PLATFORM: + github_org: "platform-eng" # PLATFORM repos → platform-eng org +``` + +**How mapping resolution works:** + +| What | Resolution order | +|---|---| +| **GitHub org** | Per-project `github_org` → default `github.org` | +| **Repo name** | Per-repo `github_name` → per-project `name_template` → global `name_template` → `{slug}` | + +**Examples** (given the config above): + +| Bitbucket | GitHub | +|---|---| +| `INFRA/my-service` | `infra-team/infra-my-service` | +| `INFRA/legacy-monolith` | `infra-team/infra-monolith` (explicit override) | +| `PLATFORM/api-gateway` | `platform-eng/platform-api-gateway` | +| `OTHER/some-tool` | `my-org/other-some-tool` (uses defaults) | + +If you omit `repo_mapping` entirely, all repos go to `github.org` with their original Bitbucket slug as the name. + +### Step 6: Configure user mapping (optional) + +Map Bitbucket usernames to GitHub usernames for PR reviewer assignments and author attribution: + +```yaml +user_mapping: + bb_jsmith: "gh-john-smith" + bb_jdoe: "gh-jane-doe" +``` + +Unmapped users fall through with their Bitbucket username as-is. + +### Step 7: Run the migration + +Run these commands in order: + +```bash +# 1. Bulk migrate all repos (creates GitHub repos, clones, pushes) bb2gh --config config.yaml migrate -# Start continuous sync (runs until interrupted) +# 2. Start continuous sync (keeps repos in lockstep during transition) bb2gh --config config.yaml sync -# Migrate open PRs (dry-run first) +# 3. In a separate terminal, migrate open PRs +# Always dry-run first to review what will be created: bb2gh --config config.yaml migrate-prs --dry-run bb2gh --config config.yaml migrate-prs ``` -### Docker +Use `-v` for debug logging on any command: ```bash -# Copy your config -mkdir config && cp config.yaml config/ +bb2gh --config config.yaml -v migrate +``` + +### Step 8: Cut over + +Once your team is ready to switch to GitHub: + +1. Stop the sync process (`Ctrl+C` or `docker compose down`) +2. Set Bitbucket repos to read-only (ask your Bitbucket admin) +3. Update CI/CD pipelines to point to GitHub +4. Notify your team to use GitHub going forward + +--- + +## Docker Usage + +### Build + +```bash +docker compose build +``` + +### Run bulk migration + +```bash +mkdir -p config && cp config.yaml config/ -# Run bulk migration docker compose --profile migrate run migrate +``` -# Start continuous sync +### Run continuous sync as a background service + +```bash docker compose up -d sync +``` + +Check logs: -# Migrate PRs +```bash +docker compose logs -f sync +``` + +### Run PR migration + +```bash docker compose --profile migrate-prs run migrate-prs ``` -## Configuration +### Environment variables + +Pass tokens via environment instead of the config file: + +```bash +BB_TOKEN=xxx GH_TOKEN=yyy docker compose up -d sync +``` + +### SSH keys -See `config.yaml.example` for all options. Key settings: +By default, Docker Compose mounts `~/.ssh` into the container. Override with: -| Setting | Description | -|---------|-------------| -| `bitbucket.base_url` | Bitbucket Server URL | -| `bitbucket.token` | Personal access token (or set `BB_TOKEN` env var) | -| `bitbucket.ssh_url` | SSH base URL for git clone | -| `bitbucket.projects` | Optional list of projects to migrate (omit for all) | -| `github.base_url` | GitHub Enterprise API URL | -| `github.token` | GitHub PAT with repo + admin:org (or set `GH_TOKEN` env var) | -| `github.org` | Target GitHub organization | -| `sync.interval_seconds` | Sync frequency (default: 60) | -| `user_mapping` | Bitbucket → GitHub username mapping for PR reviewers | +```bash +SSH_KEY_PATH=/path/to/keys docker compose --profile migrate run migrate +``` + +--- + +## CLI Reference + +``` +bb2gh [OPTIONS] COMMAND + +Options: + --config PATH Path to config file (default: config.yaml) + -v, --verbose Enable debug logging + +Commands: + migrate Bulk migrate repos from Bitbucket to GitHub + sync Continuously sync repos (runs until interrupted) + migrate-prs Migrate open pull requests from Bitbucket to GitHub +``` + +### migrate + +Clones each Bitbucket repo as a bare mirror and pushes to GitHub. Creates the target GitHub repo if it does not exist. Skips repos that have already been migrated (tracked in `state.json`). + +### sync + +Runs a loop that fetches from Bitbucket (`origin`) and pushes to GitHub (`github` remote) for every migrated repo. The interval is configured via `sync.interval_seconds` (default: 60s). Handles `SIGTERM`/`SIGINT` for graceful shutdown. + +### migrate-prs + +Reads open pull requests from Bitbucket and creates matching PRs on GitHub. Each migrated PR includes: +- A metadata header with the original author, creation date, and a link back to the Bitbucket PR +- All general comments (attributed to the original commenter) +- Reviewer assignments (mapped via `user_mapping`) + +Use `--dry-run` to preview without creating anything. + +--- ## How It Works -1. **`migrate`** — For each Bitbucket repo: creates a GitHub repo, bare-clones via SSH, cleans hidden refs, and pushes `--mirror` -2. **`sync`** — Loops every N seconds: `git fetch origin --prune` then `git push github --mirror` for each migrated repo -3. **`migrate-prs`** — For each open PR: creates a GitHub PR with metadata header, migrates comments, and assigns reviewers +``` +Bitbucket Server GitHub Enterprise +┌──────────────┐ bb2gh migrate ┌──────────────┐ +│ PROJ/repo-a ├──── git clone ──────►│ org/repo-a │ +│ PROJ/repo-b ├──── --bare ────────►│ org/repo-b │ +│ INFRA/svc ├──── + push mirror ──►│ infra/svc │ +└──────┬───────┘ └──────▲───────┘ + │ bb2gh sync │ + └──── fetch origin ── push github ─────┘ + (every 60s) +``` + +1. **`migrate`** — For each Bitbucket repo: resolves the target GitHub org/name from the mapping config, creates the GitHub repo, bare-clones via SSH, cleans hidden refs (`refs/pull/*`), and pushes `--mirror`. + +2. **`sync`** — Loops on a configurable interval: `git fetch origin --prune` then `git push github --mirror` for each migrated repo. The `--mirror` push ensures GitHub is an exact replica (all branches, tags, force-pushes). During the transition, Bitbucket is the source of truth. + +3. **`migrate-prs`** — For each open PR: looks up the correct GitHub org/repo from state, creates a GitHub PR with metadata header, migrates comments, and assigns reviewers. + +### State tracking + +Migration progress is stored in `state.json` (inside `sync.work_dir`). This file tracks: +- Which repos have been migrated and their GitHub org/repo mapping +- Last sync timestamp per repo +- Bitbucket PR ID → GitHub PR number mappings + +This makes every operation idempotent — re-running any command skips already-completed work. + +--- ## PR Migration Notes -- PRs are created under the service account (original author is noted in the PR body) -- Inline/file-level comments are migrated as regular PR comments -- Reviewer assignments use the `user_mapping` config (falls back to same username) -- Merged/closed PRs are not migrated (only open PRs) +- PRs are created under the service account — the original author is attributed in the PR body +- Inline/file-level Bitbucket comments are migrated as regular PR comments +- Only **open** PRs are migrated (merged/declined PRs are preserved in git history) +- Reviewer assignments use `user_mapping`; unmapped usernames pass through as-is + +--- + +## Configuration Reference + +| Setting | Required | Default | Description | +|---|---|---|---| +| `bitbucket.base_url` | Yes | — | Bitbucket Server URL (no trailing slash) | +| `bitbucket.token` | Yes* | `$BB_TOKEN` | Personal access token for REST API | +| `bitbucket.ssh_url` | Yes | — | SSH base URL for git clone (e.g. `ssh://git@host:7999`) | +| `bitbucket.projects` | No | all | List of project keys to migrate | +| `github.base_url` | Yes | — | GitHub Enterprise API URL | +| `github.token` | Yes* | `$GH_TOKEN` | PAT with `repo` + `admin:org` scopes | +| `github.org` | Yes | — | Default target GitHub organization | +| `sync.interval_seconds` | No | `60` | Seconds between sync cycles | +| `sync.work_dir` | No | `/data/mirror` | Directory for bare repo clones and state | +| `repo_mapping.name_template` | No | `{slug}` | Template for GitHub repo names | +| `repo_mapping.projects..github_org` | No | `github.org` | Override target org per project | +| `repo_mapping.projects..name_template` | No | global template | Override naming per project | +| `repo_mapping.projects..repos..github_name` | No | template | Explicit repo name override | +| `user_mapping` | No | `{}` | Bitbucket → GitHub username map | + +\* Can be set via environment variable instead. + +--- ## Testing @@ -91,3 +353,13 @@ See `config.yaml.example` for all options. Key settings: pip install pytest python -m pytest tests/ -v ``` + +## Troubleshooting + +| Problem | Solution | +|---|---| +| `git clone` fails with permission denied | Verify SSH key is configured and the Bitbucket service account has Project READ | +| `422` error when creating GitHub repo | Repo already exists (this is handled automatically) — or the token lacks `repo` scope | +| PR migration fails with `404` | The source or target branch was deleted; the PR cannot be recreated | +| Sync takes too long for many repos | Increase `sync.interval_seconds` or reduce the project list | +| `state.json` is corrupted | Delete it and re-run `migrate` (it will skip repos that already exist on GitHub) | From 1d5ae0c3b2d97895b77ca30fe04a832ea26ada0a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 10:05:35 +0000 Subject: [PATCH 05/74] Add include_repos and exclude_repos per-project filters Lets users migrate only a subset of repos from a Bitbucket project: - include_repos acts as an allowlist (exclusive) - exclude_repos acts as a denylist - include wins when both are set https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- README.md | 29 ++++++++++++++ bb2gh/config.py | 21 ++++++++++ bb2gh/migrator.py | 8 ++++ config.yaml.example | 11 ++++++ tests/test_config.py | 87 ++++++++++++++++++++++++++++++++++++++++++ tests/test_migrator.py | 40 +++++++++++++++++++ 6 files changed, 196 insertions(+) diff --git a/README.md b/README.md index 02f0b6f..47ae7f1 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,33 @@ repo_mapping: If you omit `repo_mapping` entirely, all repos go to `github.org` with their original Bitbucket slug as the name. +#### Migrate only a subset of repos from a project + +If you only want specific repos from a project, use `include_repos` (allowlist) or `exclude_repos` (denylist): + +```yaml +repo_mapping: + projects: + INFRA: + github_org: "infra-team" + # Only these repos from INFRA are migrated; everything else is skipped + include_repos: + - my-service + - my-api + PLATFORM: + github_org: "platform-eng" + # Migrate all repos EXCEPT these + exclude_repos: + - deprecated-tool + - archived-spike +``` + +**Resolution:** +- If `include_repos` is set, only those repos migrate (acts as an allowlist). +- Otherwise, `exclude_repos` skips the listed repos. +- If both are set, `include_repos` wins and `exclude_repos` is ignored. +- Projects with no filter migrate all repos (the default). + ### Step 6: Configure user mapping (optional) Map Bitbucket usernames to GitHub usernames for PR reviewer assignments and author attribution: @@ -340,6 +367,8 @@ This makes every operation idempotent — re-running any command skips already-c | `repo_mapping.name_template` | No | `{slug}` | Template for GitHub repo names | | `repo_mapping.projects..github_org` | No | `github.org` | Override target org per project | | `repo_mapping.projects..name_template` | No | global template | Override naming per project | +| `repo_mapping.projects..include_repos` | No | — | Allowlist: only listed repos migrate | +| `repo_mapping.projects..exclude_repos` | No | `[]` | Denylist: listed repos are skipped | | `repo_mapping.projects..repos..github_name` | No | template | Explicit repo name override | | `user_mapping` | No | `{}` | Bitbucket → GitHub username map | diff --git a/bb2gh/config.py b/bb2gh/config.py index 7b6e619..40c320e 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -77,6 +77,27 @@ def resolve_target(self, project_key, repo_slug): return gh_org, gh_repo + def should_migrate_repo(self, project_key, repo_slug): + """Check if a repo should be migrated based on include/exclude lists. + + Resolution: + - If `include_repos` is set for the project, the repo is migrated only + if it is in that list (allowlist). + - Otherwise, the repo is migrated unless it is in `exclude_repos` + (denylist). + - Projects with no repo filter config migrate all repos. + + Returns: + True if the repo should be migrated. + """ + project_conf = self._project_mappings.get(project_key, {}) + include = project_conf.get("include_repos") + exclude = project_conf.get("exclude_repos", []) + + if include is not None: + return repo_slug in include + return repo_slug not in exclude + @staticmethod def _validate(raw): for section in ("bitbucket", "github"): diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 05ef2c0..1b9254c 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -77,6 +77,14 @@ def migrate_repos(config): repo_slug = repo["slug"] repo_name = repo.get("name", repo_slug) + if not config.should_migrate_repo(project_key, repo_slug): + logger.info( + "Skipping %s/%s (filtered out by include/exclude_repos)", + project_key, repo_slug, + ) + total_skipped += 1 + continue + if state.is_migrated(project_key, repo_slug): logger.info("Skipping already migrated: %s/%s", project_key, repo_slug) total_skipped += 1 diff --git a/config.yaml.example b/config.yaml.example index 50a0ace..2a26188 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -39,12 +39,23 @@ repo_mapping: github_org: "infra-team" # Optional: override the name template for this project only # name_template: "{slug}" + # + # Optional: migrate ONLY these repos from INFRA (allowlist). + # If set, all other repos in this project are skipped. + # include_repos: + # - my-service + # - my-api repos: # Optional: explicit per-repo name overrides legacy-monolith: github_name: "infra-monolith" PLATFORM: github_org: "platform-eng" + # Optional: migrate all repos EXCEPT these (denylist). + # Ignored if include_repos is set. + # exclude_repos: + # - deprecated-tool + # - archived-spike # Projects not listed here use github.org and the global name_template # Optional: map Bitbucket usernames to GitHub usernames diff --git a/tests/test_config.py b/tests/test_config.py index 16ab937..8378668 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -148,6 +148,93 @@ def test_template_with_project_variable(self, tmp_path, base_config): assert name == "MyProject-api" +class TestShouldMigrateRepo: + def test_no_filters_migrates_everything(self, tmp_path, base_config): + """Without include/exclude, all repos migrate.""" + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("PROJ", "any-repo") is True + assert config.should_migrate_repo("OTHER", "other-repo") is True + + def test_include_repos_acts_as_allowlist(self, tmp_path, base_config): + """include_repos limits migration to listed repos only.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": { + "include_repos": ["my-service", "my-api"], + }, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("INFRA", "my-service") is True + assert config.should_migrate_repo("INFRA", "my-api") is True + assert config.should_migrate_repo("INFRA", "other-repo") is False + + def test_exclude_repos_acts_as_denylist(self, tmp_path, base_config): + """exclude_repos skips listed repos, migrates the rest.""" + base_config["repo_mapping"] = { + "projects": { + "PLATFORM": { + "exclude_repos": ["deprecated-tool", "archived-spike"], + }, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("PLATFORM", "api-gateway") is True + assert config.should_migrate_repo("PLATFORM", "deprecated-tool") is False + assert config.should_migrate_repo("PLATFORM", "archived-spike") is False + + def test_include_takes_precedence_over_exclude(self, tmp_path, base_config): + """When both are set, include_repos wins (exclude is ignored).""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": { + "include_repos": ["my-service"], + "exclude_repos": ["my-service"], # should be ignored + }, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # include wins + assert config.should_migrate_repo("INFRA", "my-service") is True + assert config.should_migrate_repo("INFRA", "other") is False + + def test_empty_include_skips_everything(self, tmp_path, base_config): + """An empty include_repos list means migrate nothing from that project.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": {"include_repos": []}, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("INFRA", "any-repo") is False + + def test_filters_scoped_to_project(self, tmp_path, base_config): + """Filters on one project don't affect other projects.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": {"include_repos": ["my-service"]}, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # INFRA is filtered + assert config.should_migrate_repo("INFRA", "my-service") is True + assert config.should_migrate_repo("INFRA", "other") is False + # OTHER project has no filter, migrates everything + assert config.should_migrate_repo("OTHER", "anything") is True + + class TestConfigValidation: def test_missing_bitbucket_section(self, tmp_path): data = {"github": {"base_url": "x", "org": "y"}} diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 9ca29ac..533a37a 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -26,6 +26,8 @@ def mock_config(tmp_path): config.user_mapping = {} # Default resolve_target returns the default org with slug as name config.resolve_target = MagicMock(side_effect=lambda proj, slug: ("my-org", slug)) + # Default: no filtering — all repos migrate + config.should_migrate_repo = MagicMock(return_value=True) return config @@ -148,3 +150,41 @@ def test_skips_already_migrated(self, MockBB, MockGH, MockState, mock_config): assert migrated == 0 assert skipped == 1 assert failed == 0 + + @patch("bb2gh.migrator.State") + @patch("bb2gh.migrator.GithubClient") + @patch("bb2gh.migrator.BitbucketClient") + @patch("bb2gh.migrator._run_git") + def test_respects_repo_filter(self, mock_git, MockBB, MockGH, MockState, mock_config): + """Test that repos filtered out by include/exclude_repos are skipped.""" + # Only allow "keep-me" through the filter + mock_config.should_migrate_repo = MagicMock( + side_effect=lambda proj, slug: slug == "keep-me" + ) + + bb_instance = MockBB.return_value + bb_instance.list_repos.return_value = [ + {"slug": "keep-me", "name": "Keep", + "links": {"clone": [{"name": "ssh", "href": "ssh://bb/p/keep-me.git"}]}}, + {"slug": "skip-me", "name": "Skip"}, + {"slug": "skip-also", "name": "Skip Also"}, + ] + + gh_instance = MockGH.return_value + gh_instance.get_clone_url.return_value = "https://github/my-org/keep-me.git" + + state_instance = MockState.return_value + state_instance.is_migrated.return_value = False + + mock_git.return_value = "" + + migrated, skipped, failed = migrate_repos(mock_config) + + assert migrated == 1 + assert skipped == 2 # two filtered out + assert failed == 0 + # Only the allowed repo gets created on GitHub + gh_instance.create_repo.assert_called_once() + state_instance.mark_migrated.assert_called_once_with( + "PROJ1", "keep-me", gh_org="my-org", gh_repo_name="keep-me" + ) From e98266c4f58fce8569f8b1ef612f35fee4ae9241 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 10:33:30 +0000 Subject: [PATCH 06/74] Add verify_ssl option for Bitbucket with self-signed certificates Fixes SSL handshake failure when Bitbucket Server uses an internal CA. Set bitbucket.verify_ssl: false in config to disable verification. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/bitbucket_client.py | 3 ++- bb2gh/config.py | 1 + bb2gh/migrator.py | 2 +- bb2gh/pr_migrator.py | 2 +- config.yaml.example | 2 ++ tests/test_migrator.py | 1 + tests/test_pr_migrator.py | 1 + 7 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py index 2d11577..2ef21fa 100644 --- a/bb2gh/bitbucket_client.py +++ b/bb2gh/bitbucket_client.py @@ -9,10 +9,11 @@ class BitbucketClient: """Client for Bitbucket Server (Data Center) REST API v1.0.""" - def __init__(self, base_url, token): + def __init__(self, base_url, token, verify_ssl=True): self.base_url = base_url.rstrip("/") self.api_url = f"{self.base_url}/rest/api/1.0" self.session = requests.Session() + self.session.verify = verify_ssl if token: self.session.headers["Authorization"] = f"Bearer {token}" diff --git a/bb2gh/config.py b/bb2gh/config.py index 40c320e..b7c3375 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -20,6 +20,7 @@ def __init__(self, path="config.yaml"): self.bb_token = bb.get("token") or os.environ.get("BB_TOKEN", "") self.bb_ssh_url = bb["ssh_url"].rstrip("/") self.bb_projects = bb.get("projects") # None means all projects + self.bb_verify_ssl = bb.get("verify_ssl", True) # GitHub settings gh = raw["github"] diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 1b9254c..aaec219 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -57,7 +57,7 @@ def migrate_repos(config): 5. Push --mirror to GitHub 6. Record in state """ - bb = BitbucketClient(config.bb_base_url, config.bb_token) + bb = BitbucketClient(config.bb_base_url, config.bb_token, verify_ssl=config.bb_verify_ssl) gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) state = State(config.work_dir) diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py index 870f990..42ff452 100644 --- a/bb2gh/pr_migrator.py +++ b/bb2gh/pr_migrator.py @@ -66,7 +66,7 @@ def migrate_pull_requests(config, dry_run=False): config: Config object. dry_run: If True, log what would be done without making changes. """ - bb = BitbucketClient(config.bb_base_url, config.bb_token) + bb = BitbucketClient(config.bb_base_url, config.bb_token, verify_ssl=config.bb_verify_ssl) gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) state = State(config.work_dir) diff --git a/config.yaml.example b/config.yaml.example index 2a26188..b68590b 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -5,6 +5,8 @@ bitbucket: token: "YOUR_BITBUCKET_TOKEN" # SSH base URL for git clone operations ssh_url: "ssh://git@bitbucket.mycompany.com:7999" + # Set to false if Bitbucket uses a self-signed or internal CA certificate + verify_ssl: true # Optional: limit migration to specific projects (omit to migrate all) # projects: # - PROJ1 diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 533a37a..cf8f459 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -19,6 +19,7 @@ def mock_config(tmp_path): config.bb_token = "fake-bb-token" config.bb_ssh_url = "ssh://git@bitbucket.example.com:7999" config.bb_projects = ["PROJ1"] + config.bb_verify_ssl = True config.gh_base_url = "https://github.example.com/api/v3" config.gh_token = "fake-gh-token" config.gh_org = "my-org" diff --git a/tests/test_pr_migrator.py b/tests/test_pr_migrator.py index b339ad1..f2eb067 100644 --- a/tests/test_pr_migrator.py +++ b/tests/test_pr_migrator.py @@ -18,6 +18,7 @@ def mock_config(): config = MagicMock(spec=Config) config.bb_base_url = "https://bitbucket.example.com" config.bb_token = "fake-token" + config.bb_verify_ssl = True config.gh_base_url = "https://github.example.com/api/v3" config.gh_token = "fake-gh-token" config.gh_org = "my-org" From 8fa38732477594de15f1cc4ab99e8d5417de0916 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 10:41:03 +0000 Subject: [PATCH 07/74] Embed token in GitHub push URL and suppress SSL warnings - Inject PAT into HTTPS clone URL so git push doesn't prompt for credentials interactively - Suppress urllib3 InsecureRequestWarning when verify_ssl is false https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/bitbucket_client.py | 3 +++ bb2gh/github_client.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py index 2ef21fa..4535368 100644 --- a/bb2gh/bitbucket_client.py +++ b/bb2gh/bitbucket_client.py @@ -14,6 +14,9 @@ def __init__(self, base_url, token, verify_ssl=True): self.api_url = f"{self.base_url}/rest/api/1.0" self.session = requests.Session() self.session.verify = verify_ssl + if not verify_ssl: + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) if token: self.session.headers["Authorization"] = f"Bearer {token}" diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py index c73f77e..23dc4b1 100644 --- a/bb2gh/github_client.py +++ b/bb2gh/github_client.py @@ -15,6 +15,7 @@ class GithubClient: def __init__(self, base_url, token, default_org): self.gh = Github(base_url=base_url, login_or_token=token) + self._token = token self.default_org = default_org self._org_cache = {} @@ -96,6 +97,9 @@ def add_pr_reviewers(self, repo_name, pr_number, reviewers, org_name=None): ) def get_clone_url(self, repo_name, org_name=None): - """Get the HTTPS clone URL for a repo.""" + """Get the HTTPS clone URL for a repo, with token embedded for auth.""" repo = self.get_repo(repo_name, org_name) - return repo.clone_url + url = repo.clone_url + # Embed token so git push doesn't prompt for credentials + url = url.replace("https://", f"https://x-access-token:{self._token}@", 1) + return url From b4408d234bdcccc6df839a0f2a43bd641ebbf582 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 10:49:39 +0000 Subject: [PATCH 08/74] Auto-remap submodule URLs from Bitbucket to GitHub during migrate and sync Rewrites .gitmodules on all branches using git plumbing (hash-object, mktree, commit-tree) in bare repos. Runs between fetch and push so GitHub always has the correct URLs. Uses fixed timestamps for deterministic commits to avoid unnecessary force-pushes. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 8 +- bb2gh/submodules.py | 143 +++++++++++++++++++++++++++++++++ bb2gh/syncer.py | 4 + tests/test_migrator.py | 9 ++- tests/test_submodules.py | 166 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 325 insertions(+), 5 deletions(-) create mode 100644 bb2gh/submodules.py create mode 100644 tests/test_submodules.py diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index aaec219..badc5e4 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -7,6 +7,7 @@ from .bitbucket_client import BitbucketClient from .github_client import GithubClient from .state import State +from .submodules import remap_submodules_in_bare_repo logger = logging.getLogger(__name__) @@ -138,7 +139,10 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # 3. Clean hidden refs _clean_hidden_refs(bare_path) - # 4. Add GitHub remote and push + # 4. Remap submodule URLs from Bitbucket to GitHub + remap_submodules_in_bare_repo(bare_path, config) + + # 5. Add GitHub remote and push gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org) # Remove existing github remote if present, then add @@ -150,6 +154,6 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) _run_git(["push", "--mirror", "github"], cwd=bare_path) - # 5. Record in state (includes the resolved GitHub org and repo name) + # 6. Record in state (includes the resolved GitHub org and repo name) state.mark_migrated(project_key, repo_slug, gh_org=gh_org, gh_repo_name=gh_repo_name) logger.info("Successfully migrated %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py new file mode 100644 index 0000000..4ef6685 --- /dev/null +++ b/bb2gh/submodules.py @@ -0,0 +1,143 @@ +"""Submodule URL remapping for Bitbucket-to-GitHub migration.""" + +import logging +import os +import re +import subprocess + +logger = logging.getLogger(__name__) + +_REMAP_ENV = { + "GIT_AUTHOR_NAME": "bb2gh", + "GIT_AUTHOR_EMAIL": "bb2gh@migration", + "GIT_AUTHOR_DATE": "2000-01-01T00:00:00+00:00", + "GIT_COMMITTER_NAME": "bb2gh", + "GIT_COMMITTER_EMAIL": "bb2gh@migration", + "GIT_COMMITTER_DATE": "2000-01-01T00:00:00+00:00", +} + +_COMMIT_MSG = "bb2gh: remap submodule URLs for GitHub migration" + + +def _git(args, cwd, stdin_data=None, env_extra=None): + cmd = ["git"] + args + env = None + if env_extra: + env = dict(os.environ) + env.update(env_extra) + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False, + input=stdin_data, env=env, + ) + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def remap_submodule_urls(content, config): + """Replace Bitbucket submodule URLs in .gitmodules with GitHub URLs. + + Only remaps URLs for repos that are in the migration scope + (matching project list and include/exclude filters). + """ + bb_ssh = config.bb_ssh_url.rstrip("/") + bb_http = config.bb_base_url.rstrip("/") + gh_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") + + def _replace(match): + project_key_raw = match.group(1) + slug = match.group(2) + for pk in [project_key_raw.upper(), project_key_raw]: + if config.bb_projects and pk not in config.bb_projects: + continue + if not config.should_migrate_repo(pk, slug): + continue + gh_org, gh_repo = config.resolve_target(pk, slug) + return f"{gh_base}/{gh_org}/{gh_repo}.git" + return match.group(0) + + ssh_pat = re.escape(bb_ssh) + r"/([^/]+)/([^/]+?)\.git" + content = re.sub(ssh_pat, _replace, content) + + http_pat = re.escape(bb_http) + r"/scm/([^/]+)/([^/]+?)\.git" + content = re.sub(http_pat, _replace, content) + + return content + + +def remap_submodules_in_bare_repo(bare_repo_path, config): + """Rewrite .gitmodules in all branches of a bare repo. + + Uses git plumbing to create deterministic commits (fixed timestamp) + so repeated runs produce identical hashes when nothing changed on + the source side — avoiding unnecessary force-pushes. + + Returns the number of branches remapped. + """ + try: + output = _git( + ["for-each-ref", "--format=%(refname)", "refs/heads/"], + cwd=bare_repo_path, + ) + except subprocess.CalledProcessError: + return 0 + + if not output.strip(): + return 0 + + remapped = 0 + for ref in output.strip().splitlines(): + if _remap_branch(bare_repo_path, ref, config): + remapped += 1 + + if remapped: + logger.info( + "Remapped submodule URLs on %d branch(es) in %s", + remapped, os.path.basename(bare_repo_path), + ) + + return remapped + + +def _remap_branch(bare_repo_path, ref, config): + """Remap .gitmodules on a single branch ref. Returns True if changed.""" + try: + content = _git(["show", f"{ref}:.gitmodules"], cwd=bare_repo_path) + except subprocess.CalledProcessError: + return False + + new_content = remap_submodule_urls(content, config) + if new_content == content: + return False + + blob_hash = _git( + ["hash-object", "-w", "--stdin"], + cwd=bare_repo_path, stdin_data=new_content, + ) + + tree_listing = _git(["ls-tree", ref], cwd=bare_repo_path) + new_lines = [] + for line in tree_listing.splitlines(): + if "\t.gitmodules" in line: + meta, _ = line.split("\t", 1) + parts = meta.split() + new_lines.append(f"{parts[0]} {parts[1]} {blob_hash}\t.gitmodules") + else: + new_lines.append(line) + + new_tree = _git( + ["mktree"], + cwd=bare_repo_path, stdin_data="\n".join(new_lines) + "\n", + ) + + parent = _git(["rev-parse", ref], cwd=bare_repo_path) + new_commit = _git( + ["commit-tree", new_tree, "-p", parent, "-m", _COMMIT_MSG], + cwd=bare_repo_path, env_extra=_REMAP_ENV, + ) + + _git(["update-ref", ref, new_commit], cwd=bare_repo_path) + logger.debug("Remapped submodules on %s", ref) + return True diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 6f5c1c7..1ef0c98 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -7,6 +7,7 @@ import time from .state import State +from .submodules import remap_submodules_in_bare_repo logger = logging.getLogger(__name__) @@ -122,6 +123,9 @@ def _sync_repo(self, project_key, repo_slug): # Clean hidden refs before pushing _clean_hidden_refs(bare_path) + # Remap submodule URLs from Bitbucket to GitHub + remap_submodules_in_bare_repo(bare_path, self.config) + # Push to GitHub _run_git(["push", "github", "--mirror"], cwd=bare_path) diff --git a/tests/test_migrator.py b/tests/test_migrator.py index cf8f459..30ac899 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -59,11 +59,12 @@ def test_no_hidden_refs(self, mock_git): class TestMigrateRepos: + @patch("bb2gh.migrator.remap_submodules_in_bare_repo") @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") @patch("bb2gh.migrator.BitbucketClient") @patch("bb2gh.migrator._run_git") - def test_migrates_new_repo(self, mock_git, MockBB, MockGH, MockState, mock_config): + def test_migrates_new_repo(self, mock_git, MockBB, MockGH, MockState, mock_remap, mock_config): # Setup mocks bb_instance = MockBB.return_value bb_instance.list_repos.return_value = [ @@ -95,11 +96,12 @@ def test_migrates_new_repo(self, mock_git, MockBB, MockGH, MockState, mock_confi "PROJ1", "my-repo", gh_org="my-org", gh_repo_name="my-repo" ) + @patch("bb2gh.migrator.remap_submodules_in_bare_repo") @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") @patch("bb2gh.migrator.BitbucketClient") @patch("bb2gh.migrator._run_git") - def test_migrates_to_mapped_org(self, mock_git, MockBB, MockGH, MockState, mock_config): + def test_migrates_to_mapped_org(self, mock_git, MockBB, MockGH, MockState, mock_remap, mock_config): """Test that repos are migrated to the correct org when mapping is configured.""" # Override resolve_target to return a different org mock_config.resolve_target = MagicMock( @@ -152,11 +154,12 @@ def test_skips_already_migrated(self, MockBB, MockGH, MockState, mock_config): assert skipped == 1 assert failed == 0 + @patch("bb2gh.migrator.remap_submodules_in_bare_repo") @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") @patch("bb2gh.migrator.BitbucketClient") @patch("bb2gh.migrator._run_git") - def test_respects_repo_filter(self, mock_git, MockBB, MockGH, MockState, mock_config): + def test_respects_repo_filter(self, mock_git, MockBB, MockGH, MockState, mock_remap, mock_config): """Test that repos filtered out by include/exclude_repos are skipped.""" # Only allow "keep-me" through the filter mock_config.should_migrate_repo = MagicMock( diff --git a/tests/test_submodules.py b/tests/test_submodules.py new file mode 100644 index 0000000..d6164f2 --- /dev/null +++ b/tests/test_submodules.py @@ -0,0 +1,166 @@ +"""Tests for submodule URL remapping.""" + +from unittest.mock import MagicMock + +import pytest + +from bb2gh.config import Config +from bb2gh.submodules import remap_submodule_urls + + +@pytest.fixture +def mock_config(): + config = MagicMock(spec=Config) + config.bb_ssh_url = "ssh://git@cph1-eud-rep001:7999" + config.bb_base_url = "https://cph1-eud-rep001" + config.gh_base_url = "https://gatehousesatcom.ghe.com/api/v3" + config.bb_projects = ["SYS_YAHSAT_NGSP", "DCKR", "NGRM"] + config.bb_verify_ssl = True + config.should_migrate_repo = MagicMock(return_value=True) + config.resolve_target = MagicMock( + side_effect=lambda proj, slug: ("networks-ngsp", slug) + ) + return config + + +SAMPLE_GITMODULES_SSH = """\ +[submodule "cai_lib"] +\tpath = cai_lib +\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_lib.git +[submodule "cai_def"] +\tpath = cai_def +\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_def.git +""" + +SAMPLE_GITMODULES_HTTP = """\ +[submodule "cai_lib"] +\tpath = cai_lib +\turl = https://cph1-eud-rep001/scm/sys_yahsat_ngsp/cai_lib.git +""" + + +class TestRemapSubmoduleUrls: + def test_remaps_ssh_urls(self, mock_config): + result = remap_submodule_urls(SAMPLE_GITMODULES_SSH, mock_config) + + assert "gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "gatehousesatcom.ghe.com/networks-ngsp/cai_def.git" in result + assert "cph1-eud-rep001" not in result + + def test_remaps_http_urls(self, mock_config): + result = remap_submodule_urls(SAMPLE_GITMODULES_HTTP, mock_config) + + assert "gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "cph1-eud-rep001" not in result + + def test_preserves_non_url_lines(self, mock_config): + result = remap_submodule_urls(SAMPLE_GITMODULES_SSH, mock_config) + + assert '[submodule "cai_lib"]' in result + assert "\tpath = cai_lib" in result + + def test_leaves_unknown_project_urls(self, mock_config): + mock_config.bb_projects = ["SYS_YAHSAT_NGSP"] + content = ( + "[submodule \"other\"]\n" + "\tpath = other\n" + "\turl = ssh://git@cph1-eud-rep001:7999/unknown_proj/other.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert "cph1-eud-rep001:7999/unknown_proj/other.git" in result + + def test_leaves_excluded_repo_urls(self, mock_config): + mock_config.should_migrate_repo = MagicMock( + side_effect=lambda proj, slug: slug != "excluded-repo" + ) + content = ( + "[submodule \"included\"]\n" + "\tpath = included\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/included.git\n" + "[submodule \"excluded\"]\n" + "\tpath = excluded\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/excluded-repo.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert "gatehousesatcom.ghe.com/networks-ngsp/included.git" in result + assert "cph1-eud-rep001:7999/sys_yahsat_ngsp/excluded-repo.git" in result + + def test_handles_uppercase_project_in_url(self, mock_config): + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = ssh://git@cph1-eud-rep001:7999/SYS_YAHSAT_NGSP/cai_lib.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert "gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + + def test_no_changes_returns_same_content(self, mock_config): + content = ( + "[submodule \"lib\"]\n" + "\tpath = lib\n" + "\turl = https://github.com/some/other.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert result == content + + def test_empty_content(self, mock_config): + assert remap_submodule_urls("", mock_config) == "" + + def test_mixed_ssh_and_http(self, mock_config): + content = ( + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + "[submodule \"b\"]\n" + "\tpath = b\n" + "\turl = https://cph1-eud-rep001/scm/sys_yahsat_ngsp/repo_b.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert "gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result + assert "cph1-eud-rep001" not in result + + def test_all_projects_when_bb_projects_is_none(self, mock_config): + """When bb_projects is None (migrate all), remap all URLs.""" + mock_config.bb_projects = None + content = ( + "[submodule \"x\"]\n" + "\tpath = x\n" + "\turl = ssh://git@cph1-eud-rep001:7999/any_project/some_repo.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert "gatehousesatcom.ghe.com/networks-ngsp/some_repo.git" in result + + def test_uses_correct_org_per_project(self, mock_config): + """Different projects should resolve to different GitHub orgs.""" + def resolve(proj, slug): + orgs = {"SYS_YAHSAT_NGSP": "networks-ngsp", "DCKR": "networks-docker"} + return orgs.get(proj, "default-org"), slug + + mock_config.resolve_target = MagicMock(side_effect=resolve) + + content = ( + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + "[submodule \"b\"]\n" + "\tpath = b\n" + "\turl = ssh://git@cph1-eud-rep001:7999/dckr/repo_b.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert "gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "gatehousesatcom.ghe.com/networks-docker/repo_b.git" in result From 85df82192cdc9ea4a96953e3e868ecd3bba022ca Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 10:51:07 +0000 Subject: [PATCH 09/74] Set default branch on GitHub to match Bitbucket's HEAD after push Reads the symbolic-ref HEAD from the bare clone (Bitbucket's default branch, typically 'master') and sets it via the GitHub API after mirror push, preventing GitHub from picking a random default. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/github_client.py | 6 ++++++ bb2gh/migrator.py | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py index 23dc4b1..1d3b3b2 100644 --- a/bb2gh/github_client.py +++ b/bb2gh/github_client.py @@ -96,6 +96,12 @@ def add_pr_reviewers(self, repo_name, pr_number, reviewers, org_name=None): "Failed to add reviewers to PR #%d: %s", pr_number, e ) + def set_default_branch(self, repo_name, branch, org_name=None): + """Set the default branch for a repository.""" + repo = self.get_repo(repo_name, org_name) + repo.edit(default_branch=branch) + logger.info("Set default branch for %s to %s", repo_name, branch) + def get_clone_url(self, repo_name, org_name=None): """Get the HTTPS clone URL for a repo, with token embedded for auth.""" repo = self.get_repo(repo_name, org_name) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index badc5e4..e904df1 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -154,6 +154,14 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) _run_git(["push", "--mirror", "github"], cwd=bare_path) - # 6. Record in state (includes the resolved GitHub org and repo name) + # 6. Set default branch on GitHub to match Bitbucket's HEAD + try: + head_ref = _run_git(["symbolic-ref", "HEAD"], cwd=bare_path) + default_branch = head_ref.replace("refs/heads/", "") + gh.set_default_branch(gh_repo_name, default_branch, org_name=gh_org) + except Exception: + logger.warning("Could not set default branch for %s/%s", gh_org, gh_repo_name) + + # 7. Record in state (includes the resolved GitHub org and repo name) state.mark_migrated(project_key, repo_slug, gh_org=gh_org, gh_repo_name=gh_repo_name) logger.info("Successfully migrated %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) From 655cb46ad8323351749a7740b0c99304dbae3db8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 10:53:42 +0000 Subject: [PATCH 10/74] Preserve SSH protocol when remapping submodule URLs SSH submodule URLs now remap to git@host:org/repo.git format instead of HTTPS. HTTP submodule URLs continue to remap to HTTPS. New config: github.ssh_host sets the GitHub SSH hostname. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 1 + bb2gh/submodules.py | 32 +++++++++++++++++++++++--------- config.yaml.example | 3 +++ tests/test_submodules.py | 39 +++++++++++++++++++++++++++------------ 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index b7c3375..6d2d508 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -27,6 +27,7 @@ def __init__(self, path="config.yaml"): self.gh_base_url = gh["base_url"].rstrip("/") self.gh_token = gh.get("token") or os.environ.get("GH_TOKEN", "") self.gh_org = gh["org"] + self.gh_ssh_host = gh.get("ssh_host", "") # Sync settings sync = raw.get("sync", {}) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 4ef6685..d02ddbc 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -44,25 +44,39 @@ def remap_submodule_urls(content, config): """ bb_ssh = config.bb_ssh_url.rstrip("/") bb_http = config.bb_base_url.rstrip("/") - gh_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") + gh_ssh_host = config.gh_ssh_host + gh_https_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") - def _replace(match): - project_key_raw = match.group(1) - slug = match.group(2) + def _resolve(project_key_raw, slug): for pk in [project_key_raw.upper(), project_key_raw]: if config.bb_projects and pk not in config.bb_projects: continue if not config.should_migrate_repo(pk, slug): continue - gh_org, gh_repo = config.resolve_target(pk, slug) - return f"{gh_base}/{gh_org}/{gh_repo}.git" - return match.group(0) + return config.resolve_target(pk, slug) + return None + + def _replace_ssh(match): + result = _resolve(match.group(1), match.group(2)) + if not result: + return match.group(0) + gh_org, gh_repo = result + if gh_ssh_host: + return f"git@{gh_ssh_host}:{gh_org}/{gh_repo}.git" + return f"{gh_https_base}/{gh_org}/{gh_repo}.git" + + def _replace_http(match): + result = _resolve(match.group(1), match.group(2)) + if not result: + return match.group(0) + gh_org, gh_repo = result + return f"{gh_https_base}/{gh_org}/{gh_repo}.git" ssh_pat = re.escape(bb_ssh) + r"/([^/]+)/([^/]+?)\.git" - content = re.sub(ssh_pat, _replace, content) + content = re.sub(ssh_pat, _replace_ssh, content) http_pat = re.escape(bb_http) + r"/scm/([^/]+)/([^/]+?)\.git" - content = re.sub(http_pat, _replace, content) + content = re.sub(http_pat, _replace_http, content) return content diff --git a/config.yaml.example b/config.yaml.example index b68590b..c5dbd46 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -19,6 +19,9 @@ github: token: "YOUR_GITHUB_TOKEN" # Default target organization on GitHub (used when no project-specific mapping exists) org: "my-org" + # SSH hostname for GitHub (used for submodule URL remapping) + # Submodules that used SSH on Bitbucket will use SSH on GitHub + ssh_host: "github.mycompany.com" sync: # Sync interval in seconds diff --git a/tests/test_submodules.py b/tests/test_submodules.py index d6164f2..c6b16bc 100644 --- a/tests/test_submodules.py +++ b/tests/test_submodules.py @@ -16,6 +16,7 @@ def mock_config(): config.gh_base_url = "https://gatehousesatcom.ghe.com/api/v3" config.bb_projects = ["SYS_YAHSAT_NGSP", "DCKR", "NGRM"] config.bb_verify_ssl = True + config.gh_ssh_host = "gatehousesatcom.ghe.com" config.should_migrate_repo = MagicMock(return_value=True) config.resolve_target = MagicMock( side_effect=lambda proj, slug: ("networks-ngsp", slug) @@ -40,17 +41,17 @@ def mock_config(): class TestRemapSubmoduleUrls: - def test_remaps_ssh_urls(self, mock_config): + def test_remaps_ssh_urls_to_ssh(self, mock_config): result = remap_submodule_urls(SAMPLE_GITMODULES_SSH, mock_config) - assert "gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result - assert "gatehousesatcom.ghe.com/networks-ngsp/cai_def.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_lib.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_def.git" in result assert "cph1-eud-rep001" not in result - def test_remaps_http_urls(self, mock_config): + def test_remaps_http_urls_to_https(self, mock_config): result = remap_submodule_urls(SAMPLE_GITMODULES_HTTP, mock_config) - assert "gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "https://gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result assert "cph1-eud-rep001" not in result def test_preserves_non_url_lines(self, mock_config): @@ -86,7 +87,7 @@ def test_leaves_excluded_repo_urls(self, mock_config): result = remap_submodule_urls(content, mock_config) - assert "gatehousesatcom.ghe.com/networks-ngsp/included.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/included.git" in result assert "cph1-eud-rep001:7999/sys_yahsat_ngsp/excluded-repo.git" in result def test_handles_uppercase_project_in_url(self, mock_config): @@ -98,7 +99,7 @@ def test_handles_uppercase_project_in_url(self, mock_config): result = remap_submodule_urls(content, mock_config) - assert "gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_lib.git" in result def test_no_changes_returns_same_content(self, mock_config): content = ( @@ -126,8 +127,9 @@ def test_mixed_ssh_and_http(self, mock_config): result = remap_submodule_urls(content, mock_config) - assert "gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result - assert "gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result + # SSH stays SSH, HTTP stays HTTPS + assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result + assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result assert "cph1-eud-rep001" not in result def test_all_projects_when_bb_projects_is_none(self, mock_config): @@ -141,7 +143,7 @@ def test_all_projects_when_bb_projects_is_none(self, mock_config): result = remap_submodule_urls(content, mock_config) - assert "gatehousesatcom.ghe.com/networks-ngsp/some_repo.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/some_repo.git" in result def test_uses_correct_org_per_project(self, mock_config): """Different projects should resolve to different GitHub orgs.""" @@ -162,5 +164,18 @@ def resolve(proj, slug): result = remap_submodule_urls(content, mock_config) - assert "gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result - assert "gatehousesatcom.ghe.com/networks-docker/repo_b.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result + assert "git@gatehousesatcom.ghe.com:networks-docker/repo_b.git" in result + + def test_falls_back_to_https_when_no_ssh_host(self, mock_config): + """Without gh_ssh_host, SSH URLs fall back to HTTPS.""" + mock_config.gh_ssh_host = "" + content = ( + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + ) + + result = remap_submodule_urls(content, mock_config) + + assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result From 26b05ea3895aea8e755107eb83a816f474ae4280 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 11:06:24 +0000 Subject: [PATCH 11/74] Auto-convert large files to Git LFS before pushing to GitHub Repos with files >100MB (GitHub's limit) are automatically handled by running git lfs migrate import --everything --above= before push. LFS objects are pushed separately after the mirror push. Opt-in via lfs.enabled config. Works for both migrate and sync. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 5 +++++ bb2gh/migrator.py | 35 +++++++++++++++++++++++++++++++++-- bb2gh/syncer.py | 12 ++++++++++++ config.yaml.example | 7 +++++++ tests/test_migrator.py | 2 ++ tests/test_syncer.py | 2 ++ 6 files changed, 61 insertions(+), 2 deletions(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index 6d2d508..14564c7 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -37,6 +37,11 @@ def __init__(self, path="config.yaml"): # User mapping (Bitbucket username -> GitHub username) self.user_mapping = raw.get("user_mapping", {}) + # LFS settings + lfs = raw.get("lfs", {}) + self.lfs_enabled = lfs.get("enabled", False) + self.lfs_threshold = lfs.get("threshold", "100mb") + # Repository mapping (Bitbucket project/repo -> GitHub org/repo) rm = raw.get("repo_mapping", {}) self._repo_mapping = rm diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index e904df1..c1585a7 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -47,6 +47,26 @@ def _clean_hidden_refs(bare_repo_path): logger.warning("Failed to delete ref: %s", ref) +def _migrate_lfs(bare_path, threshold): + """Convert files above threshold to Git LFS in all branches. + + git lfs migrate import requires a non-bare repo, so we temporarily + flip core.bare, run the migration, then flip it back. + """ + logger.info("Running LFS migration (threshold: %s) in %s", threshold, bare_path) + _run_git(["lfs", "install", "--local"], cwd=bare_path) + _run_git(["config", "core.bare", "false"], cwd=bare_path) + try: + _run_git( + ["lfs", "migrate", "import", "--everything", + f"--above={threshold}", "--yes"], + cwd=bare_path, + ) + finally: + _run_git(["config", "core.bare", "true"], cwd=bare_path) + logger.info("LFS migration complete for %s", bare_path) + + def migrate_repos(config): """Run the full bulk migration. @@ -142,7 +162,11 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # 4. Remap submodule URLs from Bitbucket to GitHub remap_submodules_in_bare_repo(bare_path, config) - # 5. Add GitHub remote and push + # 5. Migrate large files to LFS if enabled + if config.lfs_enabled: + _migrate_lfs(bare_path, config.lfs_threshold) + + # 6. Add GitHub remote and push gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org) # Remove existing github remote if present, then add @@ -154,7 +178,14 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) _run_git(["push", "--mirror", "github"], cwd=bare_path) - # 6. Set default branch on GitHub to match Bitbucket's HEAD + # Push LFS objects separately (mirror push only sends git objects) + if config.lfs_enabled: + try: + _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) + except subprocess.CalledProcessError: + logger.warning("LFS push failed for %s/%s (LFS may not be enabled on GitHub)", gh_org, gh_repo_name) + + # 7. Set default branch on GitHub to match Bitbucket's HEAD try: head_ref = _run_git(["symbolic-ref", "HEAD"], cwd=bare_path) default_branch = head_ref.replace("refs/heads/", "") diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 1ef0c98..3c6815c 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -6,6 +6,7 @@ import subprocess import time +from .migrator import _migrate_lfs from .state import State from .submodules import remap_submodules_in_bare_repo @@ -126,9 +127,20 @@ def _sync_repo(self, project_key, repo_slug): # Remap submodule URLs from Bitbucket to GitHub remap_submodules_in_bare_repo(bare_path, self.config) + # Migrate large files to LFS if enabled + if self.config.lfs_enabled: + _migrate_lfs(bare_path, self.config.lfs_threshold) + # Push to GitHub _run_git(["push", "github", "--mirror"], cwd=bare_path) + # Push LFS objects separately + if self.config.lfs_enabled: + try: + _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) + except subprocess.CalledProcessError: + logger.warning("LFS push failed for %s/%s", project_key, repo_slug) + elapsed = time.time() - start self.state.update_sync_time(project_key, repo_slug) logger.info( diff --git a/config.yaml.example b/config.yaml.example index c5dbd46..5115a1a 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -63,6 +63,13 @@ repo_mapping: # - archived-spike # Projects not listed here use github.org and the global name_template +# Optional: auto-convert large files to Git LFS before pushing to GitHub. +# GitHub rejects files >100MB. This rewrites history to store them as LFS objects. +lfs: + enabled: false + # Files above this size are converted to LFS pointers + threshold: "100mb" + # Optional: map Bitbucket usernames to GitHub usernames # Used for PR author attribution and reviewer assignments # user_mapping: diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 30ac899..c14e9dc 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -29,6 +29,8 @@ def mock_config(tmp_path): config.resolve_target = MagicMock(side_effect=lambda proj, slug: ("my-org", slug)) # Default: no filtering — all repos migrate config.should_migrate_repo = MagicMock(return_value=True) + config.lfs_enabled = False + config.lfs_threshold = "100mb" return config diff --git a/tests/test_syncer.py b/tests/test_syncer.py index f8c2c91..c090053 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -14,6 +14,8 @@ def mock_config(tmp_path): config = MagicMock(spec=Config) config.work_dir = str(tmp_path) config.sync_interval = 1 + config.lfs_enabled = False + config.lfs_threshold = "100mb" return config From b1a721a70378c8c9fcd6cae00c98b6cbbe9e53de Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 11:17:47 +0000 Subject: [PATCH 12/74] Fix LFS migration by using a temp working copy instead of bare repo git lfs migrate import requires a working tree. Now clones the bare repo to a temp directory, runs LFS migration there, then fetches the rewritten refs and LFS objects back into the bare repo. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index c1585a7..055276f 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -50,20 +50,43 @@ def _clean_hidden_refs(bare_repo_path): def _migrate_lfs(bare_path, threshold): """Convert files above threshold to Git LFS in all branches. - git lfs migrate import requires a non-bare repo, so we temporarily - flip core.bare, run the migration, then flip it back. + git lfs migrate import requires a working tree, so we clone the + bare repo to a temp directory, run LFS migration there, then + fetch the rewritten refs back into the bare repo. """ + import shutil + import tempfile + logger.info("Running LFS migration (threshold: %s) in %s", threshold, bare_path) - _run_git(["lfs", "install", "--local"], cwd=bare_path) - _run_git(["config", "core.bare", "false"], cwd=bare_path) + tmp_dir = tempfile.mkdtemp(suffix=".lfs-migrate") try: + # Clone bare repo to a working copy + _run_git(["clone", bare_path, tmp_dir + "/work"]) + work_path = tmp_dir + "/work" + + # Install LFS and run migration + _run_git(["lfs", "install"], cwd=work_path) _run_git( ["lfs", "migrate", "import", "--everything", f"--above={threshold}", "--yes"], - cwd=bare_path, + cwd=work_path, ) + + # Fetch the rewritten refs back into the bare repo + _run_git(["remote", "add", "lfs-source", work_path], cwd=bare_path) + _run_git(["fetch", "lfs-source", "--force", "+refs/heads/*:refs/heads/*"], cwd=bare_path) + _run_git(["remote", "remove", "lfs-source"], cwd=bare_path) + + # Copy LFS objects into the bare repo + lfs_src = os.path.join(work_path, ".git", "lfs") + lfs_dst = os.path.join(bare_path, "lfs") + if os.path.exists(lfs_src): + if os.path.exists(lfs_dst): + shutil.rmtree(lfs_dst) + shutil.copytree(lfs_src, lfs_dst) + finally: - _run_git(["config", "core.bare", "true"], cwd=bare_path) + shutil.rmtree(tmp_dir, ignore_errors=True) logger.info("LFS migration complete for %s", bare_path) From f4015ea9ac9014674ba7b779639c9b504d6e84e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 11:19:18 +0000 Subject: [PATCH 13/74] Sanitize repo descriptions to strip control characters GitHub rejects descriptions containing control characters (newlines, tabs, etc.) that Bitbucket allows. Strip them and cap at 350 chars. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 055276f..fa0edc5 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -160,7 +160,9 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam ) # 1. Create repo on GitHub (in the resolved org) + import re description = repo.get("description", "") or f"Migrated from Bitbucket: {project_key}/{repo_slug}" + description = re.sub(r"[\x00-\x1f\x7f]", " ", description).strip()[:350] gh.create_repo(gh_repo_name, description=description, private=True, org_name=gh_org) # 2. Bare clone from Bitbucket From 445c0b46088e72f3872fb0c7c02aa288b66f6896 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 11:27:32 +0000 Subject: [PATCH 14/74] Fix LFS migration for all branches and redact tokens from error logs LFS migration now creates local branches for all remote branches in the temp working copy before running git lfs migrate, ensuring all branches are rewritten. Also fetches rewritten tags. Error logs now redact tokens from URLs to prevent credential leaks. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index fa0edc5..08c164d 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -12,6 +12,12 @@ logger = logging.getLogger(__name__) +def _redact(text): + """Remove tokens/passwords from URLs in log output.""" + import re + return re.sub(r"(https?://)[^@/]+@", r"\1***@", text) + + def _run_git(args, cwd=None): """Run a git command and return stdout.""" cmd = ["git"] + args @@ -20,7 +26,7 @@ def _run_git(args, cwd=None): cmd, cwd=cwd, capture_output=True, text=True, check=False ) if result.returncode != 0: - logger.error("git %s failed: %s", args[0], result.stderr.strip()) + logger.error("git %s failed: %s", args[0], _redact(result.stderr.strip())) raise subprocess.CalledProcessError( result.returncode, cmd, result.stdout, result.stderr ) @@ -51,8 +57,8 @@ def _migrate_lfs(bare_path, threshold): """Convert files above threshold to Git LFS in all branches. git lfs migrate import requires a working tree, so we clone the - bare repo to a temp directory, run LFS migration there, then - fetch the rewritten refs back into the bare repo. + bare repo to a temp directory, create local branches for all remotes, + run LFS migration, then fetch the rewritten refs back. """ import shutil import tempfile @@ -60,11 +66,21 @@ def _migrate_lfs(bare_path, threshold): logger.info("Running LFS migration (threshold: %s) in %s", threshold, bare_path) tmp_dir = tempfile.mkdtemp(suffix=".lfs-migrate") try: - # Clone bare repo to a working copy - _run_git(["clone", bare_path, tmp_dir + "/work"]) - work_path = tmp_dir + "/work" + work_path = os.path.join(tmp_dir, "work") + _run_git(["clone", bare_path, work_path]) + + # Create local branches for ALL remote branches so LFS rewrites them all + branches_output = _run_git(["branch", "-r"], cwd=work_path) + for line in branches_output.splitlines(): + branch = line.strip() + if "HEAD" in branch or not branch.startswith("origin/"): + continue + local_name = branch.replace("origin/", "", 1) + try: + _run_git(["branch", "--track", local_name, branch], cwd=work_path) + except subprocess.CalledProcessError: + pass # Already exists (e.g. the default branch) - # Install LFS and run migration _run_git(["lfs", "install"], cwd=work_path) _run_git( ["lfs", "migrate", "import", "--everything", @@ -72,9 +88,11 @@ def _migrate_lfs(bare_path, threshold): cwd=work_path, ) - # Fetch the rewritten refs back into the bare repo + # Fetch rewritten branches and tags back into the bare repo _run_git(["remote", "add", "lfs-source", work_path], cwd=bare_path) - _run_git(["fetch", "lfs-source", "--force", "+refs/heads/*:refs/heads/*"], cwd=bare_path) + _run_git(["fetch", "lfs-source", "--force", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) _run_git(["remote", "remove", "lfs-source"], cwd=bare_path) # Copy LFS objects into the bare repo From 9a55bb33b6359ab95cf405b93b4a5b7b30a14eb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 11:35:54 +0000 Subject: [PATCH 15/74] Suppress expected git errors from cluttering logs Branch-already-exists and remote-not-found errors are expected during normal operation. Use quiet=True to prevent logging them as ERROR. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 08c164d..519798c 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -18,7 +18,7 @@ def _redact(text): return re.sub(r"(https?://)[^@/]+@", r"\1***@", text) -def _run_git(args, cwd=None): +def _run_git(args, cwd=None, quiet=False): """Run a git command and return stdout.""" cmd = ["git"] + args logger.debug("Running: %s", " ".join(cmd)) @@ -26,7 +26,8 @@ def _run_git(args, cwd=None): cmd, cwd=cwd, capture_output=True, text=True, check=False ) if result.returncode != 0: - logger.error("git %s failed: %s", args[0], _redact(result.stderr.strip())) + if not quiet: + logger.error("git %s failed: %s", args[0], _redact(result.stderr.strip())) raise subprocess.CalledProcessError( result.returncode, cmd, result.stdout, result.stderr ) @@ -77,9 +78,9 @@ def _migrate_lfs(bare_path, threshold): continue local_name = branch.replace("origin/", "", 1) try: - _run_git(["branch", "--track", local_name, branch], cwd=work_path) + _run_git(["branch", "--track", local_name, branch], cwd=work_path, quiet=True) except subprocess.CalledProcessError: - pass # Already exists (e.g. the default branch) + pass # Already exists (default branch) _run_git(["lfs", "install"], cwd=work_path) _run_git( @@ -214,7 +215,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # Remove existing github remote if present, then add try: - _run_git(["remote", "remove", "github"], cwd=bare_path) + _run_git(["remote", "remove", "github"], cwd=bare_path, quiet=True) except subprocess.CalledProcessError: pass # Remote didn't exist From 3520bac261a7bf201c4fac4e1b3b5fa21c540c3a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 12:11:52 +0000 Subject: [PATCH 16/74] Rewrite submodule remapping to handle hostname variants and skip on failure - Match BB URLs by hostname (supports FQDN variants via ssh_hostnames config) - If ANY Bitbucket URL cannot be resolved, skip the entire .gitmodules rewrite to avoid mixing old and new URLs - Already-GitHub and external URLs are ignored (not counted as failures) - URLs already pointing to GitHub are left as-is https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 1 + bb2gh/submodules.py | 137 ++++++++++++++++++++++++++++--------- config.yaml.example | 6 ++ tests/test_submodules.py | 143 +++++++++++++++++++++++++-------------- 4 files changed, 205 insertions(+), 82 deletions(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index 14564c7..f603061 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -19,6 +19,7 @@ def __init__(self, path="config.yaml"): self.bb_base_url = bb["base_url"].rstrip("/") self.bb_token = bb.get("token") or os.environ.get("BB_TOKEN", "") self.bb_ssh_url = bb["ssh_url"].rstrip("/") + self.bb_ssh_hostnames = bb.get("ssh_hostnames", []) self.bb_projects = bb.get("projects") # None means all projects self.bb_verify_ssl = bb.get("verify_ssl", True) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index d02ddbc..311d652 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -4,6 +4,7 @@ import os import re import subprocess +from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -36,49 +37,123 @@ def _git(args, cwd, stdin_data=None, env_extra=None): return result.stdout.strip() +def _build_bb_hostnames(config): + """Extract all known Bitbucket hostnames from config.""" + hostnames = set(config.bb_ssh_hostnames) + + # Extract hostname from ssh_url: ssh://git@host:port -> host + parsed = urlparse(config.bb_ssh_url) + if parsed.hostname: + hostnames.add(parsed.hostname) + + # Extract hostname from base_url: https://host -> host + parsed = urlparse(config.bb_base_url) + if parsed.hostname: + hostnames.add(parsed.hostname) + + return hostnames + + +def _extract_submodule_urls(content): + """Extract all url = ... values from .gitmodules content.""" + return re.findall(r"url\s*=\s*(.+)", content) + + +def _parse_bb_url(url, bb_hostnames): + """Parse a URL and check if it's a Bitbucket URL. + + Returns (project_key, slug) if it's a BB URL, None otherwise. + Handles: + - ssh://git@host:port/project/repo.git + - git@host:port/project/repo.git (shouldn't exist for BB but just in case) + - https://host/scm/project/repo.git + """ + # SSH format: ssh://git@hostname:port/project/repo.git + m = re.match(r"ssh://[^@]+@([^:/]+)[:/]\d*/([^/]+)/([^/]+?)\.git$", url) + if m and m.group(1) in bb_hostnames: + return m.group(2), m.group(3) + + # HTTP format: https://hostname/scm/project/repo.git + m = re.match(r"https?://([^/]+)/scm/([^/]+)/([^/]+?)\.git$", url) + if m and m.group(1) in bb_hostnames: + return m.group(2), m.group(3) + + return None + + +def _is_already_github(url, gh_ssh_host, gh_https_base): + """Check if a URL already points to GitHub.""" + if gh_ssh_host and url.startswith(f"git@{gh_ssh_host}:"): + return True + if gh_https_base and gh_https_base in url: + return True + return False + + def remap_submodule_urls(content, config): """Replace Bitbucket submodule URLs in .gitmodules with GitHub URLs. - Only remaps URLs for repos that are in the migration scope - (matching project list and include/exclude filters). + If any Bitbucket URL cannot be resolved (project not migrated, repo + excluded), the entire .gitmodules is left unchanged to avoid a mix + of old and new URLs. """ - bb_ssh = config.bb_ssh_url.rstrip("/") - bb_http = config.bb_base_url.rstrip("/") + bb_hostnames = _build_bb_hostnames(config) gh_ssh_host = config.gh_ssh_host gh_https_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") - def _resolve(project_key_raw, slug): + urls = _extract_submodule_urls(content) + if not urls: + return content + + # First pass: check ALL URLs can be resolved + replacements = {} + for url in urls: + url = url.strip() + + # Already points to GitHub — skip + if _is_already_github(url, gh_ssh_host, gh_https_base): + continue + + parsed = _parse_bb_url(url, bb_hostnames) + if parsed is None: + # Not a Bitbucket URL we recognize — skip (external dependency) + continue + + project_key_raw, slug = parsed + resolved = None for pk in [project_key_raw.upper(), project_key_raw]: if config.bb_projects and pk not in config.bb_projects: continue if not config.should_migrate_repo(pk, slug): continue - return config.resolve_target(pk, slug) - return None - - def _replace_ssh(match): - result = _resolve(match.group(1), match.group(2)) - if not result: - return match.group(0) - gh_org, gh_repo = result - if gh_ssh_host: - return f"git@{gh_ssh_host}:{gh_org}/{gh_repo}.git" - return f"{gh_https_base}/{gh_org}/{gh_repo}.git" - - def _replace_http(match): - result = _resolve(match.group(1), match.group(2)) - if not result: - return match.group(0) - gh_org, gh_repo = result - return f"{gh_https_base}/{gh_org}/{gh_repo}.git" - - ssh_pat = re.escape(bb_ssh) + r"/([^/]+)/([^/]+?)\.git" - content = re.sub(ssh_pat, _replace_ssh, content) - - http_pat = re.escape(bb_http) + r"/scm/([^/]+)/([^/]+?)\.git" - content = re.sub(http_pat, _replace_http, content) - - return content + resolved = config.resolve_target(pk, slug) + break + + if not resolved: + logger.warning( + "Cannot remap submodule URL %s — project %s/%s not in migration scope. " + "Skipping .gitmodules rewrite entirely.", + url, project_key_raw, slug, + ) + return content # Return unchanged + + gh_org, gh_repo = resolved + is_ssh = url.startswith("ssh://") + if is_ssh and gh_ssh_host: + new_url = f"git@{gh_ssh_host}:{gh_org}/{gh_repo}.git" + else: + new_url = f"{gh_https_base}/{gh_org}/{gh_repo}.git" + replacements[url] = new_url + + if not replacements: + return content + + # Second pass: apply all replacements + new_content = content + for old_url, new_url in replacements.items(): + new_content = new_content.replace(old_url, new_url) + + return new_content def remap_submodules_in_bare_repo(bare_repo_path, config): diff --git a/config.yaml.example b/config.yaml.example index 5115a1a..747158a 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -5,6 +5,12 @@ bitbucket: token: "YOUR_BITBUCKET_TOKEN" # SSH base URL for git clone operations ssh_url: "ssh://git@bitbucket.mycompany.com:7999" + # Additional Bitbucket SSH hostnames (for submodule URL matching). + # Some .gitmodules may use a different hostname variant (e.g. with FQDN). + # The hostname from ssh_url is matched automatically — list extras here. + # ssh_hostnames: + # - "bitbucket.mycompany.com" + # - "bitbucket.internal.mycompany.com" # Set to false if Bitbucket uses a self-signed or internal CA certificate verify_ssl: true # Optional: limit migration to specific projects (omit to migrate all) diff --git a/tests/test_submodules.py b/tests/test_submodules.py index c6b16bc..ea63dcf 100644 --- a/tests/test_submodules.py +++ b/tests/test_submodules.py @@ -13,8 +13,9 @@ def mock_config(): config = MagicMock(spec=Config) config.bb_ssh_url = "ssh://git@cph1-eud-rep001:7999" config.bb_base_url = "https://cph1-eud-rep001" + config.bb_ssh_hostnames = ["cph1-eud-rep001.satcom.global"] config.gh_base_url = "https://gatehousesatcom.ghe.com/api/v3" - config.bb_projects = ["SYS_YAHSAT_NGSP", "DCKR", "NGRM"] + config.bb_projects = ["SYS_YAHSAT_NGSP", "SYS_COM", "DCKR", "NGRM"] config.bb_verify_ssl = True config.gh_ssh_host = "gatehousesatcom.ghe.com" config.should_migrate_repo = MagicMock(return_value=True) @@ -24,55 +25,73 @@ def mock_config(): return config -SAMPLE_GITMODULES_SSH = """\ -[submodule "cai_lib"] -\tpath = cai_lib -\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_lib.git -[submodule "cai_def"] -\tpath = cai_def -\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_def.git -""" - -SAMPLE_GITMODULES_HTTP = """\ -[submodule "cai_lib"] -\tpath = cai_lib -\turl = https://cph1-eud-rep001/scm/sys_yahsat_ngsp/cai_lib.git -""" - - class TestRemapSubmoduleUrls: def test_remaps_ssh_urls_to_ssh(self, mock_config): - result = remap_submodule_urls(SAMPLE_GITMODULES_SSH, mock_config) + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_lib.git\n" + "[submodule \"cai_def\"]\n" + "\tpath = cai_def\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_def.git\n" + ) + result = remap_submodule_urls(content, mock_config) assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_lib.git" in result assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_def.git" in result assert "cph1-eud-rep001" not in result + def test_remaps_fqdn_hostname_variant(self, mock_config): + """URLs using the FQDN variant should also be matched.""" + content = ( + "[submodule \"lib\"]\n" + "\tpath = lib\n" + "\turl = ssh://git@cph1-eud-rep001.satcom.global:7999/sys_yahsat_ngsp/lib.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "git@gatehousesatcom.ghe.com:networks-ngsp/lib.git" in result + assert "cph1-eud-rep001" not in result + def test_remaps_http_urls_to_https(self, mock_config): - result = remap_submodule_urls(SAMPLE_GITMODULES_HTTP, mock_config) + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = https://cph1-eud-rep001/scm/sys_yahsat_ngsp/cai_lib.git\n" + ) + result = remap_submodule_urls(content, mock_config) assert "https://gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result - assert "cph1-eud-rep001" not in result def test_preserves_non_url_lines(self, mock_config): - result = remap_submodule_urls(SAMPLE_GITMODULES_SSH, mock_config) + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_lib.git\n" + ) + result = remap_submodule_urls(content, mock_config) assert '[submodule "cai_lib"]' in result assert "\tpath = cai_lib" in result - def test_leaves_unknown_project_urls(self, mock_config): + def test_skips_entirely_when_url_unresolvable(self, mock_config): + """If any BB URL can't be resolved, return content unchanged.""" mock_config.bb_projects = ["SYS_YAHSAT_NGSP"] content = ( - "[submodule \"other\"]\n" - "\tpath = other\n" - "\turl = ssh://git@cph1-eud-rep001:7999/unknown_proj/other.git\n" + "[submodule \"known\"]\n" + "\tpath = known\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/known.git\n" + "[submodule \"unknown\"]\n" + "\tpath = unknown\n" + "\turl = ssh://git@cph1-eud-rep001:7999/other_project/unknown.git\n" ) - result = remap_submodule_urls(content, mock_config) - assert "cph1-eud-rep001:7999/unknown_proj/other.git" in result + # Entire file should be unchanged + assert result == content - def test_leaves_excluded_repo_urls(self, mock_config): + def test_skips_entirely_when_excluded_repo(self, mock_config): + """If any BB URL points to an excluded repo, skip entirely.""" mock_config.should_migrate_repo = MagicMock( side_effect=lambda proj, slug: slug != "excluded-repo" ) @@ -84,11 +103,39 @@ def test_leaves_excluded_repo_urls(self, mock_config): "\tpath = excluded\n" "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/excluded-repo.git\n" ) + result = remap_submodule_urls(content, mock_config) + assert result == content + + def test_ignores_already_github_urls(self, mock_config): + """URLs already pointing to GitHub should be ignored.""" + content = ( + "[submodule \"bb_repo\"]\n" + "\tpath = bb_repo\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/bb_repo.git\n" + "[submodule \"gh_repo\"]\n" + "\tpath = gh_repo\n" + "\turl = git@gatehousesatcom.ghe.com:networks-ngsp/gh_repo.git\n" + ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/included.git" in result - assert "cph1-eud-rep001:7999/sys_yahsat_ngsp/excluded-repo.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/bb_repo.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/gh_repo.git" in result + + def test_ignores_external_urls(self, mock_config): + """URLs pointing to external hosts (not BB) should be left alone.""" + content = ( + "[submodule \"bb_repo\"]\n" + "\tpath = bb_repo\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/bb_repo.git\n" + "[submodule \"external\"]\n" + "\tpath = external\n" + "\turl = https://github.com/some/external.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "git@gatehousesatcom.ghe.com:networks-ngsp/bb_repo.git" in result + assert "https://github.com/some/external.git" in result def test_handles_uppercase_project_in_url(self, mock_config): content = ( @@ -96,25 +143,22 @@ def test_handles_uppercase_project_in_url(self, mock_config): "\tpath = cai_lib\n" "\turl = ssh://git@cph1-eud-rep001:7999/SYS_YAHSAT_NGSP/cai_lib.git\n" ) - result = remap_submodule_urls(content, mock_config) assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_lib.git" in result - def test_no_changes_returns_same_content(self, mock_config): + def test_empty_content(self, mock_config): + assert remap_submodule_urls("", mock_config) == "" + + def test_no_matching_urls(self, mock_config): content = ( "[submodule \"lib\"]\n" "\tpath = lib\n" "\turl = https://github.com/some/other.git\n" ) - result = remap_submodule_urls(content, mock_config) - assert result == content - def test_empty_content(self, mock_config): - assert remap_submodule_urls("", mock_config) == "" - def test_mixed_ssh_and_http(self, mock_config): content = ( "[submodule \"a\"]\n" @@ -124,29 +168,29 @@ def test_mixed_ssh_and_http(self, mock_config): "\tpath = b\n" "\turl = https://cph1-eud-rep001/scm/sys_yahsat_ngsp/repo_b.git\n" ) - result = remap_submodule_urls(content, mock_config) - # SSH stays SSH, HTTP stays HTTPS assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result assert "cph1-eud-rep001" not in result - def test_all_projects_when_bb_projects_is_none(self, mock_config): - """When bb_projects is None (migrate all), remap all URLs.""" - mock_config.bb_projects = None + def test_mixed_hostnames_all_resolved(self, mock_config): + """Both short and FQDN hostname variants should be resolved together.""" content = ( - "[submodule \"x\"]\n" - "\tpath = x\n" - "\turl = ssh://git@cph1-eud-rep001:7999/any_project/some_repo.git\n" + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + "[submodule \"b\"]\n" + "\tpath = b\n" + "\turl = ssh://git@cph1-eud-rep001.satcom.global:7999/sys_com/repo_b.git\n" ) - result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/some_repo.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result + assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_b.git" in result + assert "cph1-eud-rep001" not in result def test_uses_correct_org_per_project(self, mock_config): - """Different projects should resolve to different GitHub orgs.""" def resolve(proj, slug): orgs = {"SYS_YAHSAT_NGSP": "networks-ngsp", "DCKR": "networks-docker"} return orgs.get(proj, "default-org"), slug @@ -161,21 +205,18 @@ def resolve(proj, slug): "\tpath = b\n" "\turl = ssh://git@cph1-eud-rep001:7999/dckr/repo_b.git\n" ) - result = remap_submodule_urls(content, mock_config) assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result assert "git@gatehousesatcom.ghe.com:networks-docker/repo_b.git" in result def test_falls_back_to_https_when_no_ssh_host(self, mock_config): - """Without gh_ssh_host, SSH URLs fall back to HTTPS.""" mock_config.gh_ssh_host = "" content = ( "[submodule \"a\"]\n" "\tpath = a\n" "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" ) - result = remap_submodule_urls(content, mock_config) assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result From 392108bdc30d9c3d235018e8d4279415d726b132 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 12:17:43 +0000 Subject: [PATCH 17/74] Add reset-submodules command to re-migrate repos with .gitmodules Scans bare clones for repos that have .gitmodules, removes them from state.json so the next migrate run re-processes them with the updated submodule URL remapping. Supports --dry-run to preview. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ bb2gh/state.py | 9 +++++++++ 2 files changed, 56 insertions(+) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 1ea61e2..8bdebb4 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -79,5 +79,52 @@ def migrate_prs(ctx, dry_run): sys.exit(1) +@cli.command("reset-submodules") +@click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") +@click.pass_context +def reset_submodules(ctx, dry_run): + """Reset migrated repos that have .gitmodules so they get re-migrated. + + Scans bare clones for repos containing .gitmodules, removes them from + state.json, so the next 'bb2gh migrate' run re-processes them (with + submodule URL remapping). + """ + import os + import subprocess + + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + repos = state.get_migrated_repos() + if not repos: + click.echo("No migrated repos found.") + return + + reset_count = 0 + for project_key, repo_slug in repos: + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + continue + + result = subprocess.run( + ["git", "show", "HEAD:.gitmodules"], + cwd=bare_path, capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + continue + + if dry_run: + click.echo(f"[DRY RUN] Would reset: {project_key}/{repo_slug}") + else: + state.reset_repo(project_key, repo_slug) + click.echo(f"Reset: {project_key}/{repo_slug}") + reset_count += 1 + + click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos with submodules.") + if not dry_run and reset_count > 0: + click.echo("Run 'bb2gh migrate' to re-migrate them.") + + if __name__ == "__main__": cli() diff --git a/bb2gh/state.py b/bb2gh/state.py index 799be8c..b095805 100644 --- a/bb2gh/state.py +++ b/bb2gh/state.py @@ -86,6 +86,15 @@ def get_github_target(self, project_key, repo_slug): entry = self._data["repos"].get(key, {}) return entry.get("gh_org"), entry.get("gh_repo_name") + def reset_repo(self, project_key, repo_slug): + """Remove a repo from state so it will be re-migrated on next run.""" + key = f"{project_key}/{repo_slug}" + if key in self._data["repos"]: + del self._data["repos"][key] + self._save() + return True + return False + def is_migrated(self, project_key, repo_slug): """Check if a repo has been migrated.""" key = f"{project_key}/{repo_slug}" From 02cce764c6d842dfeb83c0a320bc8213f99e4934 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 14:29:38 +0000 Subject: [PATCH 18/74] Skip git lfs push when no files were converted to LFS _migrate_lfs now returns whether it actually converted any files. git lfs push --all is only run when there are LFS objects to push, preventing hour-long hangs scanning repos with no large files. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 25 ++++++++++++++++--------- bb2gh/syncer.py | 7 ++++--- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 519798c..4242eab 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -96,17 +96,23 @@ def _migrate_lfs(bare_path, threshold): "+refs/tags/*:refs/tags/*"], cwd=bare_path) _run_git(["remote", "remove", "lfs-source"], cwd=bare_path) - # Copy LFS objects into the bare repo - lfs_src = os.path.join(work_path, ".git", "lfs") + # Copy LFS objects into the bare repo (only if any were created) + lfs_src = os.path.join(work_path, ".git", "lfs", "objects") lfs_dst = os.path.join(bare_path, "lfs") - if os.path.exists(lfs_src): + has_lfs_objects = os.path.exists(lfs_src) and os.listdir(lfs_src) + if has_lfs_objects: if os.path.exists(lfs_dst): shutil.rmtree(lfs_dst) - shutil.copytree(lfs_src, lfs_dst) + shutil.copytree(os.path.join(work_path, ".git", "lfs"), lfs_dst) finally: shutil.rmtree(tmp_dir, ignore_errors=True) - logger.info("LFS migration complete for %s", bare_path) + + if has_lfs_objects: + logger.info("LFS migration converted files in %s", bare_path) + else: + logger.info("LFS migration: no files above threshold in %s", bare_path) + return has_lfs_objects def migrate_repos(config): @@ -207,8 +213,9 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam remap_submodules_in_bare_repo(bare_path, config) # 5. Migrate large files to LFS if enabled + has_lfs = False if config.lfs_enabled: - _migrate_lfs(bare_path, config.lfs_threshold) + has_lfs = _migrate_lfs(bare_path, config.lfs_threshold) # 6. Add GitHub remote and push gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org) @@ -222,12 +229,12 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) _run_git(["push", "--mirror", "github"], cwd=bare_path) - # Push LFS objects separately (mirror push only sends git objects) - if config.lfs_enabled: + # Push LFS objects separately — only if LFS actually converted files + if has_lfs: try: _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) except subprocess.CalledProcessError: - logger.warning("LFS push failed for %s/%s (LFS may not be enabled on GitHub)", gh_org, gh_repo_name) + logger.warning("LFS push failed for %s/%s", gh_org, gh_repo_name) # 7. Set default branch on GitHub to match Bitbucket's HEAD try: diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 3c6815c..9db2882 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -128,14 +128,15 @@ def _sync_repo(self, project_key, repo_slug): remap_submodules_in_bare_repo(bare_path, self.config) # Migrate large files to LFS if enabled + has_lfs = False if self.config.lfs_enabled: - _migrate_lfs(bare_path, self.config.lfs_threshold) + has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold) # Push to GitHub _run_git(["push", "github", "--mirror"], cwd=bare_path) - # Push LFS objects separately - if self.config.lfs_enabled: + # Push LFS objects — only if LFS actually converted files + if has_lfs: try: _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) except subprocess.CalledProcessError: From f02940a613d56649c02f1bda982c51a068a9eaa8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 15:55:49 +0000 Subject: [PATCH 19/74] Add reset-lfs command to re-migrate repos with large files Scans bare clones for blobs above a size threshold (default 100mb), resets those repos in state.json so the next migrate run re-processes them with the current LFS config. Supports --above and --dry-run. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 8bdebb4..53d6fc5 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -126,5 +126,85 @@ def reset_submodules(ctx, dry_run): click.echo("Run 'bb2gh migrate' to re-migrate them.") +@cli.command("reset-lfs") +@click.option("--above", default="100mb", help="File size threshold (e.g. 100mb, 50mb).") +@click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") +@click.pass_context +def reset_lfs(ctx, above, dry_run): + """Reset migrated repos that have files above a size threshold. + + Scans bare clones for blobs larger than --above, removes matching repos + from state.json so the next 'bb2gh migrate' re-processes them with the + current LFS threshold. + """ + import os + import subprocess + + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + # Parse threshold like "100mb" -> bytes + threshold_str = above.lower().strip() + if threshold_str.endswith("mb"): + threshold_bytes = int(threshold_str[:-2]) * 1024 * 1024 + elif threshold_str.endswith("gb"): + threshold_bytes = int(threshold_str[:-2]) * 1024 * 1024 * 1024 + elif threshold_str.endswith("kb"): + threshold_bytes = int(threshold_str[:-2]) * 1024 + else: + threshold_bytes = int(threshold_str) + + repos = state.get_migrated_repos() + if not repos: + click.echo("No migrated repos found.") + return + + reset_count = 0 + for project_key, repo_slug in repos: + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + continue + + # Find blobs above threshold in the repo history + result = subprocess.run( + ["git", "rev-list", "--objects", "--all"], + cwd=bare_path, capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + continue + + cat_result = subprocess.run( + ["git", "cat-file", "--batch-check=%(objecttype) %(objectsize)"], + cwd=bare_path, input=result.stdout, + capture_output=True, text=True, check=False, + ) + if cat_result.returncode != 0: + continue + + has_large = False + for line in cat_result.stdout.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] == "blob": + size = int(parts[1]) + if size > threshold_bytes: + has_large = True + break + + if not has_large: + continue + + if dry_run: + click.echo(f"[DRY RUN] Would reset: {project_key}/{repo_slug}") + else: + state.reset_repo(project_key, repo_slug) + click.echo(f"Reset: {project_key}/{repo_slug}") + reset_count += 1 + + click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos with files above {above}.") + if not dry_run and reset_count > 0: + click.echo("Run 'bb2gh migrate' to re-migrate them with the current LFS threshold.") + + if __name__ == "__main__": cli() From e23583a0035a6533a90cfabbfd9abe7e1caf9cb9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 15:58:22 +0000 Subject: [PATCH 20/74] Fix reset-lfs to detect LFS by indicators instead of blob sizes Previous approach scanned for large blobs, but after LFS migration the blobs are already replaced with pointers. Now checks for lfs/ directory and .gitattributes with filter=lfs patterns instead. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 55 ++++++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 40 deletions(-) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 53d6fc5..d9072a9 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -127,15 +127,15 @@ def reset_submodules(ctx, dry_run): @cli.command("reset-lfs") -@click.option("--above", default="100mb", help="File size threshold (e.g. 100mb, 50mb).") @click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") @click.pass_context -def reset_lfs(ctx, above, dry_run): - """Reset migrated repos that have files above a size threshold. +def reset_lfs(ctx, dry_run): + """Reset migrated repos that were LFS-migrated so they get re-processed. - Scans bare clones for blobs larger than --above, removes matching repos - from state.json so the next 'bb2gh migrate' re-processes them with the - current LFS threshold. + Finds repos where LFS migration previously ran (have lfs/ directory + or .gitattributes with LFS patterns), removes them from state.json + so the next 'bb2gh migrate' re-fetches from Bitbucket and re-runs + LFS with the current threshold. """ import os import subprocess @@ -144,17 +144,6 @@ def reset_lfs(ctx, above, dry_run): from .state import State state = State(config.work_dir) - # Parse threshold like "100mb" -> bytes - threshold_str = above.lower().strip() - if threshold_str.endswith("mb"): - threshold_bytes = int(threshold_str[:-2]) * 1024 * 1024 - elif threshold_str.endswith("gb"): - threshold_bytes = int(threshold_str[:-2]) * 1024 * 1024 * 1024 - elif threshold_str.endswith("kb"): - threshold_bytes = int(threshold_str[:-2]) * 1024 - else: - threshold_bytes = int(threshold_str) - repos = state.get_migrated_repos() if not repos: click.echo("No migrated repos found.") @@ -166,32 +155,18 @@ def reset_lfs(ctx, above, dry_run): if not os.path.exists(bare_path): continue - # Find blobs above threshold in the repo history + # Check for LFS indicators + has_lfs_dir = os.path.exists(os.path.join(bare_path, "lfs", "objects")) + + has_lfs_attrs = False result = subprocess.run( - ["git", "rev-list", "--objects", "--all"], + ["git", "show", "HEAD:.gitattributes"], cwd=bare_path, capture_output=True, text=True, check=False, ) - if result.returncode != 0: - continue - - cat_result = subprocess.run( - ["git", "cat-file", "--batch-check=%(objecttype) %(objectsize)"], - cwd=bare_path, input=result.stdout, - capture_output=True, text=True, check=False, - ) - if cat_result.returncode != 0: - continue - - has_large = False - for line in cat_result.stdout.splitlines(): - parts = line.split() - if len(parts) >= 2 and parts[0] == "blob": - size = int(parts[1]) - if size > threshold_bytes: - has_large = True - break + if result.returncode == 0 and "filter=lfs" in result.stdout: + has_lfs_attrs = True - if not has_large: + if not has_lfs_dir and not has_lfs_attrs: continue if dry_run: @@ -201,7 +176,7 @@ def reset_lfs(ctx, above, dry_run): click.echo(f"Reset: {project_key}/{repo_slug}") reset_count += 1 - click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos with files above {above}.") + click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos with previous LFS migration.") if not dry_run and reset_count > 0: click.echo("Run 'bb2gh migrate' to re-migrate them with the current LFS threshold.") From fc76175afae1131fcd869d581de565e1b4303c1c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 19:19:24 +0000 Subject: [PATCH 21/74] Skip LFS smudge during temp clone for LFS migration Repos that already use LFS on Bitbucket have pointers referencing BB's LFS server. The temp clone for LFS migration would fail trying to download those objects. Set GIT_LFS_SKIP_SMUDGE=1 to skip. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 4242eab..e763902 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -68,7 +68,21 @@ def _migrate_lfs(bare_path, threshold): tmp_dir = tempfile.mkdtemp(suffix=".lfs-migrate") try: work_path = os.path.join(tmp_dir, "work") - _run_git(["clone", bare_path, work_path]) + # Skip LFS smudge during clone — repo may already have LFS pointers + # pointing to the original BB LFS server + env_no_lfs = { + "GIT_LFS_SKIP_SMUDGE": "1", + } + cmd = ["git", "clone", bare_path, work_path] + result = subprocess.run( + cmd, capture_output=True, text=True, check=False, + env={**os.environ, **env_no_lfs}, + ) + if result.returncode != 0: + logger.error("git clone failed: %s", result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) # Create local branches for ALL remote branches so LFS rewrites them all branches_output = _run_git(["branch", "-r"], cwd=work_path) From e737e867f50ccd5d2768a1b6cd2c307af667752a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 19:25:26 +0000 Subject: [PATCH 22/74] Add general reset command with project/repo/all targeting Allows resetting specific projects, individual repos, or everything in state.json so they get re-migrated. Useful when changing org mappings or fixing specific repos. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index d9072a9..e720ffe 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -79,6 +79,61 @@ def migrate_prs(ctx, dry_run): sys.exit(1) +@cli.command("reset") +@click.option("--project", multiple=True, help="Reset all repos in this project (can repeat).") +@click.option("--repo", multiple=True, help="Reset a specific PROJECT/SLUG (can repeat).") +@click.option("--all", "reset_all", is_flag=True, help="Reset ALL migrated repos.") +@click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") +@click.pass_context +def reset(ctx, project, repo, reset_all, dry_run): + """Reset repos in state.json so they get re-migrated. + + Examples: + bb2gh reset --project UPSTREAM + bb2gh reset --repo UPSTREAM/embeddedsw --repo UPSTREAM/git + bb2gh reset --all + """ + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + repos = state.get_migrated_repos() + if not repos: + click.echo("No migrated repos found.") + return + + projects_set = set(p.upper() for p in project) + repos_set = set(repo) + + reset_count = 0 + for project_key, repo_slug in repos: + should_reset = False + if reset_all: + should_reset = True + elif project_key in projects_set or project_key.upper() in projects_set: + should_reset = True + elif f"{project_key}/{repo_slug}" in repos_set: + should_reset = True + + if not should_reset: + continue + + if dry_run: + gh_org, gh_repo = state.get_github_target(project_key, repo_slug) + click.echo(f"[DRY RUN] Would reset: {project_key}/{repo_slug} (was -> {gh_org}/{gh_repo})") + else: + state.reset_repo(project_key, repo_slug) + click.echo(f"Reset: {project_key}/{repo_slug}") + reset_count += 1 + + if reset_count == 0: + click.echo("No matching repos found in state.") + else: + click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos.") + if not dry_run: + click.echo("Run 'bb2gh migrate' to re-migrate them.") + + @cli.command("reset-submodules") @click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") @click.pass_context From 32bd09b6b0e4e840fd6a205e71c1762578b371cb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 19:36:23 +0000 Subject: [PATCH 23/74] Fix fetch on bare clones by specifying explicit refspecs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git clone --bare does not create a fetch refspec, so git fetch origin was a no-op — it never reset local refs to Bitbucket's originals. Now explicitly fetches +refs/heads/*:refs/heads/* and +refs/tags/*:refs/tags/* so re-runs properly overwrite previously remapped refs. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 4 +++- bb2gh/syncer.py | 4 +++- tests/test_syncer.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index e763902..53b8dc9 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -210,7 +210,9 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam if os.path.exists(bare_path): # Already cloned, fetch latest logger.info("Bare clone exists, fetching latest: %s", bare_path) - _run_git(["fetch", "origin", "--prune"], cwd=bare_path) + _run_git(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) else: clone_url = bb.get_repo_clone_url(repo, protocol="ssh") if not clone_url: diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 9db2882..e0f1116 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -119,7 +119,9 @@ def _sync_repo(self, project_key, repo_slug): start = time.time() # Fetch from Bitbucket (origin) - _run_git(["fetch", "origin", "--prune"], cwd=bare_path) + _run_git(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) # Clean hidden refs before pushing _clean_hidden_refs(bare_path) diff --git a/tests/test_syncer.py b/tests/test_syncer.py index c090053..3b69062 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -38,7 +38,9 @@ def test_sync_repo(self, mock_git, MockState, mock_config, tmp_path): syncer._sync_all() # Should fetch from origin and push to github - mock_git.assert_any_call(["fetch", "origin", "--prune"], cwd=str(bare_path)) + mock_git.assert_any_call(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=str(bare_path)) mock_git.assert_any_call(["push", "github", "--mirror"], cwd=str(bare_path)) state_instance.update_sync_time.assert_called_once_with("PROJ", "my-repo") From af69fb27f01e318e694a9187f412e7ddd0fea8a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Apr 2026 21:28:37 +0000 Subject: [PATCH 24/74] Handle LFS migrate checkout failure on repos with unborn branches git lfs migrate import may fail on the post-rewrite checkout when HEAD points to a non-existent branch. The rewrite itself completes successfully. Detect this case and continue instead of failing. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 53b8dc9..4a23da9 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -97,11 +97,19 @@ def _migrate_lfs(bare_path, threshold): pass # Already exists (default branch) _run_git(["lfs", "install"], cwd=work_path) - _run_git( - ["lfs", "migrate", "import", "--everything", - f"--above={threshold}", "--yes"], - cwd=work_path, - ) + try: + _run_git( + ["lfs", "migrate", "import", "--everything", + f"--above={threshold}", "--yes"], + cwd=work_path, + ) + except subprocess.CalledProcessError as e: + # LFS migrate may fail on post-rewrite checkout (unborn branch, etc.) + # but the rewrite itself completed. Check stderr for this case. + if "Could not checkout" in (e.stderr or "") and "Rewriting commits" in (e.stderr or ""): + logger.warning("LFS rewrite completed but checkout failed (harmless)") + else: + raise # Fetch rewritten branches and tags back into the bare repo _run_git(["remote", "add", "lfs-source", work_path], cwd=bare_path) From 4070f2713cd34a8f1b78025b57081c2257c26e3f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Apr 2026 20:14:17 +0000 Subject: [PATCH 25/74] Fix false positive LFS detection by checking for actual object files os.listdir on lfs/objects/ was matching empty subdirectories created by git lfs install, causing git lfs push --all to run on repos with no real LFS objects. Now walks the directory tree to check for actual files before reporting LFS objects exist. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 4a23da9..a8afca9 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -118,11 +118,18 @@ def _migrate_lfs(bare_path, threshold): "+refs/tags/*:refs/tags/*"], cwd=bare_path) _run_git(["remote", "remove", "lfs-source"], cwd=bare_path) - # Copy LFS objects into the bare repo (only if any were created) - lfs_src = os.path.join(work_path, ".git", "lfs", "objects") - lfs_dst = os.path.join(bare_path, "lfs") - has_lfs_objects = os.path.exists(lfs_src) and os.listdir(lfs_src) + # Check if LFS actually tracked any files (not just empty dirs) + lfs_objects_dir = os.path.join(work_path, ".git", "lfs", "objects") + has_lfs_objects = False + if os.path.exists(lfs_objects_dir): + for dirpath, dirnames, filenames in os.walk(lfs_objects_dir): + if filenames: + has_lfs_objects = True + break + + # Copy LFS objects into the bare repo only if real objects exist if has_lfs_objects: + lfs_dst = os.path.join(bare_path, "lfs") if os.path.exists(lfs_dst): shutil.rmtree(lfs_dst) shutil.copytree(os.path.join(work_path, ".git", "lfs"), lfs_dst) From ed7421b93d8c721117891b7dfc0e94f3e04f0a81 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Apr 2026 20:45:49 +0000 Subject: [PATCH 26/74] Use SSH for GitHub push remote when ssh_host is configured When github.ssh_host is set, the push remote uses git@host:org/repo.git instead of HTTPS. This avoids HTTP 500 timeouts on large repos and keeps protocol consistent with origin. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/github_client.py | 12 +++++++++--- bb2gh/migrator.py | 2 +- tests/test_migrator.py | 3 ++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py index 1d3b3b2..87e3c85 100644 --- a/bb2gh/github_client.py +++ b/bb2gh/github_client.py @@ -102,10 +102,16 @@ def set_default_branch(self, repo_name, branch, org_name=None): repo.edit(default_branch=branch) logger.info("Set default branch for %s to %s", repo_name, branch) - def get_clone_url(self, repo_name, org_name=None): - """Get the HTTPS clone URL for a repo, with token embedded for auth.""" + def get_clone_url(self, repo_name, org_name=None, ssh_host=None): + """Get the clone URL for a repo. + + If ssh_host is provided, returns an SSH URL (git@host:org/repo.git). + Otherwise returns HTTPS with the token embedded. + """ + org = org_name or self.default_org + if ssh_host: + return f"git@{ssh_host}:{org}/{repo_name}.git" repo = self.get_repo(repo_name, org_name) url = repo.clone_url - # Embed token so git push doesn't prompt for credentials url = url.replace("https://", f"https://x-access-token:{self._token}@", 1) return url diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index a8afca9..5330691 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -249,7 +249,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam has_lfs = _migrate_lfs(bare_path, config.lfs_threshold) # 6. Add GitHub remote and push - gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org) + gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, ssh_host=config.gh_ssh_host or None) # Remove existing github remote if present, then add try: diff --git a/tests/test_migrator.py b/tests/test_migrator.py index c14e9dc..8b80bd5 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -31,6 +31,7 @@ def mock_config(tmp_path): config.should_migrate_repo = MagicMock(return_value=True) config.lfs_enabled = False config.lfs_threshold = "100mb" + config.gh_ssh_host = "" return config @@ -135,7 +136,7 @@ def test_migrates_to_mapped_org(self, mock_git, MockBB, MockGH, MockState, mock_ gh_instance.create_repo.assert_called_once_with( "infra-my-repo", description="A test repo", private=True, org_name="infra-team" ) - gh_instance.get_clone_url.assert_called_once_with("infra-my-repo", org_name="infra-team") + gh_instance.get_clone_url.assert_called_once_with("infra-my-repo", org_name="infra-team", ssh_host=None) state_instance.mark_migrated.assert_called_once_with( "PROJ1", "my-repo", gh_org="infra-team", gh_repo_name="infra-my-repo" ) From 9a576243d34482c90ead2a08697fab2880b70037 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Apr 2026 20:51:03 +0000 Subject: [PATCH 27/74] Use ssh:// URL format for GitHub SSH push and submodule remapping Change from SCP-style (git@host:org/repo.git) to standard SSH URL format (ssh://git@host/org/repo.git) for compatibility. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/github_client.py | 2 +- bb2gh/submodules.py | 4 ++-- tests/test_submodules.py | 26 +++++++++++++------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py index 87e3c85..79d36b0 100644 --- a/bb2gh/github_client.py +++ b/bb2gh/github_client.py @@ -110,7 +110,7 @@ def get_clone_url(self, repo_name, org_name=None, ssh_host=None): """ org = org_name or self.default_org if ssh_host: - return f"git@{ssh_host}:{org}/{repo_name}.git" + return f"ssh://git@{ssh_host}/{org}/{repo_name}.git" repo = self.get_repo(repo_name, org_name) url = repo.clone_url url = url.replace("https://", f"https://x-access-token:{self._token}@", 1) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 311d652..99f38bc 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -83,7 +83,7 @@ def _parse_bb_url(url, bb_hostnames): def _is_already_github(url, gh_ssh_host, gh_https_base): """Check if a URL already points to GitHub.""" - if gh_ssh_host and url.startswith(f"git@{gh_ssh_host}:"): + if gh_ssh_host and (url.startswith(f"git@{gh_ssh_host}:") or url.startswith(f"ssh://git@{gh_ssh_host}/")): return True if gh_https_base and gh_https_base in url: return True @@ -140,7 +140,7 @@ def remap_submodule_urls(content, config): gh_org, gh_repo = resolved is_ssh = url.startswith("ssh://") if is_ssh and gh_ssh_host: - new_url = f"git@{gh_ssh_host}:{gh_org}/{gh_repo}.git" + new_url = f"ssh://git@{gh_ssh_host}/{gh_org}/{gh_repo}.git" else: new_url = f"{gh_https_base}/{gh_org}/{gh_repo}.git" replacements[url] = new_url diff --git a/tests/test_submodules.py b/tests/test_submodules.py index ea63dcf..39aa254 100644 --- a/tests/test_submodules.py +++ b/tests/test_submodules.py @@ -37,8 +37,8 @@ def test_remaps_ssh_urls_to_ssh(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_lib.git" in result - assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_def.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/cai_def.git" in result assert "cph1-eud-rep001" not in result def test_remaps_fqdn_hostname_variant(self, mock_config): @@ -50,7 +50,7 @@ def test_remaps_fqdn_hostname_variant(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/lib.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/lib.git" in result assert "cph1-eud-rep001" not in result def test_remaps_http_urls_to_https(self, mock_config): @@ -115,12 +115,12 @@ def test_ignores_already_github_urls(self, mock_config): "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/bb_repo.git\n" "[submodule \"gh_repo\"]\n" "\tpath = gh_repo\n" - "\turl = git@gatehousesatcom.ghe.com:networks-ngsp/gh_repo.git\n" + "\turl = ssh://git@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git\n" ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/bb_repo.git" in result - assert "git@gatehousesatcom.ghe.com:networks-ngsp/gh_repo.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git" in result def test_ignores_external_urls(self, mock_config): """URLs pointing to external hosts (not BB) should be left alone.""" @@ -134,7 +134,7 @@ def test_ignores_external_urls(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/bb_repo.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result assert "https://github.com/some/external.git" in result def test_handles_uppercase_project_in_url(self, mock_config): @@ -145,7 +145,7 @@ def test_handles_uppercase_project_in_url(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/cai_lib.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result def test_empty_content(self, mock_config): assert remap_submodule_urls("", mock_config) == "" @@ -170,7 +170,7 @@ def test_mixed_ssh_and_http(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result assert "cph1-eud-rep001" not in result @@ -186,8 +186,8 @@ def test_mixed_hostnames_all_resolved(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result - assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_b.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result assert "cph1-eud-rep001" not in result def test_uses_correct_org_per_project(self, mock_config): @@ -207,8 +207,8 @@ def resolve(proj, slug): ) result = remap_submodule_urls(content, mock_config) - assert "git@gatehousesatcom.ghe.com:networks-ngsp/repo_a.git" in result - assert "git@gatehousesatcom.ghe.com:networks-docker/repo_b.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "ssh://git@gatehousesatcom.ghe.com/networks-docker/repo_b.git" in result def test_falls_back_to_https_when_no_ssh_host(self, mock_config): mock_config.gh_ssh_host = "" From 25566d7a0c3b2add4b2fbc5a971adc510cebee4e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Apr 2026 20:58:42 +0000 Subject: [PATCH 28/74] Add github.ssh_url config for custom SSH user in push and submodule URLs GHE may use a non-standard SSH user (e.g. gatehousesatcom@ instead of git@). New config github.ssh_url allows specifying the full SSH prefix (e.g. ssh://gatehousesatcom@host) for push remotes and submodule URL remapping. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 1 + bb2gh/github_client.py | 10 +++++----- bb2gh/migrator.py | 2 +- bb2gh/submodules.py | 13 +++++++++---- config.yaml.example | 4 +++- tests/test_migrator.py | 3 ++- tests/test_submodules.py | 30 ++++++++++++++++-------------- 7 files changed, 37 insertions(+), 26 deletions(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index f603061..26a62d9 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -29,6 +29,7 @@ def __init__(self, path="config.yaml"): self.gh_token = gh.get("token") or os.environ.get("GH_TOKEN", "") self.gh_org = gh["org"] self.gh_ssh_host = gh.get("ssh_host", "") + self.gh_ssh_url = gh.get("ssh_url", "") # Sync settings sync = raw.get("sync", {}) diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py index 79d36b0..60b74db 100644 --- a/bb2gh/github_client.py +++ b/bb2gh/github_client.py @@ -102,15 +102,15 @@ def set_default_branch(self, repo_name, branch, org_name=None): repo.edit(default_branch=branch) logger.info("Set default branch for %s to %s", repo_name, branch) - def get_clone_url(self, repo_name, org_name=None, ssh_host=None): + def get_clone_url(self, repo_name, org_name=None, ssh_url=None): """Get the clone URL for a repo. - If ssh_host is provided, returns an SSH URL (git@host:org/repo.git). - Otherwise returns HTTPS with the token embedded. + If ssh_url is provided (e.g. "ssh://gatehousesatcom@host"), builds + an SSH URL. Otherwise returns HTTPS with the token embedded. """ org = org_name or self.default_org - if ssh_host: - return f"ssh://git@{ssh_host}/{org}/{repo_name}.git" + if ssh_url: + return f"{ssh_url.rstrip('/')}/{org}/{repo_name}.git" repo = self.get_repo(repo_name, org_name) url = repo.clone_url url = url.replace("https://", f"https://x-access-token:{self._token}@", 1) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 5330691..f944762 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -249,7 +249,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam has_lfs = _migrate_lfs(bare_path, config.lfs_threshold) # 6. Add GitHub remote and push - gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, ssh_host=config.gh_ssh_host or None) + gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, ssh_url=config.gh_ssh_url or None) # Remove existing github remote if present, then add try: diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 99f38bc..01a0432 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -81,9 +81,11 @@ def _parse_bb_url(url, bb_hostnames): return None -def _is_already_github(url, gh_ssh_host, gh_https_base): +def _is_already_github(url, gh_ssh_url, gh_ssh_host, gh_https_base): """Check if a URL already points to GitHub.""" - if gh_ssh_host and (url.startswith(f"git@{gh_ssh_host}:") or url.startswith(f"ssh://git@{gh_ssh_host}/")): + if gh_ssh_url and url.startswith(gh_ssh_url.rstrip("/")): + return True + if gh_ssh_host and (url.startswith(f"git@{gh_ssh_host}:") or url.startswith(f"ssh://git@{gh_ssh_host}/") or url.startswith(f"ssh://{gh_ssh_host}/")): return True if gh_https_base and gh_https_base in url: return True @@ -98,6 +100,7 @@ def remap_submodule_urls(content, config): of old and new URLs. """ bb_hostnames = _build_bb_hostnames(config) + gh_ssh_url = config.gh_ssh_url gh_ssh_host = config.gh_ssh_host gh_https_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") @@ -111,7 +114,7 @@ def remap_submodule_urls(content, config): url = url.strip() # Already points to GitHub — skip - if _is_already_github(url, gh_ssh_host, gh_https_base): + if _is_already_github(url, gh_ssh_url, gh_ssh_host, gh_https_base): continue parsed = _parse_bb_url(url, bb_hostnames) @@ -139,7 +142,9 @@ def remap_submodule_urls(content, config): gh_org, gh_repo = resolved is_ssh = url.startswith("ssh://") - if is_ssh and gh_ssh_host: + if is_ssh and gh_ssh_url: + new_url = f"{gh_ssh_url.rstrip('/')}/{gh_org}/{gh_repo}.git" + elif is_ssh and gh_ssh_host: new_url = f"ssh://git@{gh_ssh_host}/{gh_org}/{gh_repo}.git" else: new_url = f"{gh_https_base}/{gh_org}/{gh_repo}.git" diff --git a/config.yaml.example b/config.yaml.example index 747158a..9a00521 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -26,8 +26,10 @@ github: # Default target organization on GitHub (used when no project-specific mapping exists) org: "my-org" # SSH hostname for GitHub (used for submodule URL remapping) - # Submodules that used SSH on Bitbucket will use SSH on GitHub ssh_host: "github.mycompany.com" + # SSH URL prefix for GitHub (used for push and submodule remapping) + # Use this if your GHE SSH user isn't "git" (e.g. ssh://myuser@host) + # ssh_url: "ssh://myuser@github.mycompany.com" sync: # Sync interval in seconds diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 8b80bd5..95a7ae1 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -32,6 +32,7 @@ def mock_config(tmp_path): config.lfs_enabled = False config.lfs_threshold = "100mb" config.gh_ssh_host = "" + config.gh_ssh_url = "" return config @@ -136,7 +137,7 @@ def test_migrates_to_mapped_org(self, mock_git, MockBB, MockGH, MockState, mock_ gh_instance.create_repo.assert_called_once_with( "infra-my-repo", description="A test repo", private=True, org_name="infra-team" ) - gh_instance.get_clone_url.assert_called_once_with("infra-my-repo", org_name="infra-team", ssh_host=None) + gh_instance.get_clone_url.assert_called_once_with("infra-my-repo", org_name="infra-team", ssh_url=None) state_instance.mark_migrated.assert_called_once_with( "PROJ1", "my-repo", gh_org="infra-team", gh_repo_name="infra-my-repo" ) diff --git a/tests/test_submodules.py b/tests/test_submodules.py index 39aa254..e75b0f8 100644 --- a/tests/test_submodules.py +++ b/tests/test_submodules.py @@ -18,6 +18,7 @@ def mock_config(): config.bb_projects = ["SYS_YAHSAT_NGSP", "SYS_COM", "DCKR", "NGRM"] config.bb_verify_ssl = True config.gh_ssh_host = "gatehousesatcom.ghe.com" + config.gh_ssh_url = "ssh://gatehousesatcom@gatehousesatcom.ghe.com" config.should_migrate_repo = MagicMock(return_value=True) config.resolve_target = MagicMock( side_effect=lambda proj, slug: ("networks-ngsp", slug) @@ -37,8 +38,8 @@ def test_remaps_ssh_urls_to_ssh(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/cai_def.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/cai_def.git" in result assert "cph1-eud-rep001" not in result def test_remaps_fqdn_hostname_variant(self, mock_config): @@ -50,7 +51,7 @@ def test_remaps_fqdn_hostname_variant(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/lib.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/lib.git" in result assert "cph1-eud-rep001" not in result def test_remaps_http_urls_to_https(self, mock_config): @@ -115,12 +116,12 @@ def test_ignores_already_github_urls(self, mock_config): "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/bb_repo.git\n" "[submodule \"gh_repo\"]\n" "\tpath = gh_repo\n" - "\turl = ssh://git@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git\n" + "\turl = ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git\n" ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git" in result def test_ignores_external_urls(self, mock_config): """URLs pointing to external hosts (not BB) should be left alone.""" @@ -134,7 +135,7 @@ def test_ignores_external_urls(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result assert "https://github.com/some/external.git" in result def test_handles_uppercase_project_in_url(self, mock_config): @@ -145,7 +146,7 @@ def test_handles_uppercase_project_in_url(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result def test_empty_content(self, mock_config): assert remap_submodule_urls("", mock_config) == "" @@ -170,7 +171,7 @@ def test_mixed_ssh_and_http(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result assert "cph1-eud-rep001" not in result @@ -186,8 +187,8 @@ def test_mixed_hostnames_all_resolved(self, mock_config): ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result assert "cph1-eud-rep001" not in result def test_uses_correct_org_per_project(self, mock_config): @@ -207,11 +208,12 @@ def resolve(proj, slug): ) result = remap_submodule_urls(content, mock_config) - assert "ssh://git@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result - assert "ssh://git@gatehousesatcom.ghe.com/networks-docker/repo_b.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-docker/repo_b.git" in result - def test_falls_back_to_https_when_no_ssh_host(self, mock_config): + def test_falls_back_to_https_when_no_ssh(self, mock_config): mock_config.gh_ssh_host = "" + mock_config.gh_ssh_url = "" content = ( "[submodule \"a\"]\n" "\tpath = a\n" From 7667c2625c22aa21c406fe9986c3b3efe002cb9b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 06:44:36 +0000 Subject: [PATCH 29/74] Pre-check for large blobs before running expensive LFS migration Scan the bare repo's git objects for blobs above the threshold before cloning to a temp dir and running git lfs migrate import. Repos with no large files skip the entire LFS step (clone + rewrite), saving minutes on large repos like wireshark (81k commits, 10 min wasted). https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index f944762..c6df3d9 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -54,6 +54,45 @@ def _clean_hidden_refs(bare_repo_path): logger.warning("Failed to delete ref: %s", ref) +def _has_large_blobs(bare_path, threshold): + """Check if a bare repo has any blobs above the threshold. + + Parses threshold strings like '100mb' and scans git objects. + Returns True if any blob exceeds the threshold. + """ + t = threshold.lower().strip() + if t.endswith("mb"): + threshold_bytes = int(t[:-2]) * 1024 * 1024 + elif t.endswith("gb"): + threshold_bytes = int(t[:-2]) * 1024 * 1024 * 1024 + elif t.endswith("kb"): + threshold_bytes = int(t[:-2]) * 1024 + else: + threshold_bytes = int(t) + + try: + rev_list = _run_git( + ["rev-list", "--objects", "--all"], cwd=bare_path, quiet=True + ) + except subprocess.CalledProcessError: + return False + + cmd = ["git", "cat-file", "--batch-check=%(objecttype) %(objectsize)"] + result = subprocess.run( + cmd, cwd=bare_path, input=rev_list, + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return False + + for line in result.stdout.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] == "blob": + if int(parts[1]) > threshold_bytes: + return True + return False + + def _migrate_lfs(bare_path, threshold): """Convert files above threshold to Git LFS in all branches. @@ -64,7 +103,11 @@ def _migrate_lfs(bare_path, threshold): import shutil import tempfile - logger.info("Running LFS migration (threshold: %s) in %s", threshold, bare_path) + if not _has_large_blobs(bare_path, threshold): + logger.info("LFS: no files above %s in %s, skipping", threshold, os.path.basename(bare_path)) + return False + + logger.info("LFS: large files detected, migrating (threshold: %s) in %s", threshold, bare_path) tmp_dir = tempfile.mkdtemp(suffix=".lfs-migrate") try: work_path = os.path.join(tmp_dir, "work") From a9d42d902b44f415f447f82ca544a30d0ecbf69f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 06:58:38 +0000 Subject: [PATCH 30/74] Add migration report command with failure tracking - State now tracks: has_submodules, submodules_remapped, has_lfs, warnings per repo. Failed repos are recorded with error messages. - New 'report' command generates text or CSV reports showing migrated, failed, LFS, submodule status, and warnings. - Usage: bb2gh report [--format csv] [--output report.csv] https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 143 +++++++++++++++++++++++++++++++++++++++++ bb2gh/migrator.py | 31 +++++++-- bb2gh/state.py | 27 +++++++- tests/test_migrator.py | 31 ++++----- 4 files changed, 212 insertions(+), 20 deletions(-) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index e720ffe..62dbddf 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -236,5 +236,148 @@ def reset_lfs(ctx, dry_run): click.echo("Run 'bb2gh migrate' to re-migrate them with the current LFS threshold.") +@cli.command() +@click.option("--format", "fmt", type=click.Choice(["text", "csv"]), default="text", + help="Output format.") +@click.option("--output", "output_file", default=None, help="Write report to file.") +@click.pass_context +def report(ctx, fmt, output_file): + """Generate a migration report from state.json. + + Shows migrated repos, failed repos, submodule status, LFS status, + and any warnings. + """ + import json + + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + data = state._data.get("repos", {}) + if not data: + click.echo("No migration data found.") + return + + migrated = [] + failed = [] + with_submodules = [] + submodules_not_remapped = [] + with_lfs = [] + with_warnings = [] + + for key, entry in sorted(data.items()): + status = entry.get("status", "unknown") + if status == "migrated": + migrated.append(entry) + elif status == "failed": + failed.append(entry) + + if entry.get("has_submodules"): + with_submodules.append(entry) + if not entry.get("submodules_remapped"): + submodules_not_remapped.append(entry) + if entry.get("has_lfs"): + with_lfs.append(entry) + if entry.get("warnings"): + with_warnings.append(entry) + + lines = [] + + if fmt == "text": + lines.append("=" * 70) + lines.append("MIGRATION REPORT") + lines.append("=" * 70) + lines.append("") + lines.append(f"Total repos in state: {len(data)}") + lines.append(f" Migrated: {len(migrated)}") + lines.append(f" Failed: {len(failed)}") + lines.append(f" With submodules: {len(with_submodules)}") + lines.append(f" Remapped: {len(with_submodules) - len(submodules_not_remapped)}") + lines.append(f" Not remapped: {len(submodules_not_remapped)}") + lines.append(f" With LFS: {len(with_lfs)}") + lines.append(f" With warnings: {len(with_warnings)}") + + if failed: + lines.append("") + lines.append("-" * 70) + lines.append("FAILED REPOS") + lines.append("-" * 70) + for entry in failed: + lines.append(f" {entry['project_key']}/{entry['repo_slug']}") + lines.append(f" Target: {entry.get('gh_org', '?')}/{entry.get('gh_repo_name', '?')}") + lines.append(f" Error: {entry.get('error', 'unknown')[:200]}") + + if submodules_not_remapped: + lines.append("") + lines.append("-" * 70) + lines.append("SUBMODULES NOT REMAPPED") + lines.append("-" * 70) + for entry in submodules_not_remapped: + lines.append(f" {entry['project_key']}/{entry['repo_slug']} -> {entry.get('gh_org')}/{entry.get('gh_repo_name')}") + + if with_lfs: + lines.append("") + lines.append("-" * 70) + lines.append("REPOS WITH LFS") + lines.append("-" * 70) + for entry in with_lfs: + lines.append(f" {entry['project_key']}/{entry['repo_slug']} -> {entry.get('gh_org')}/{entry.get('gh_repo_name')}") + + if with_warnings: + lines.append("") + lines.append("-" * 70) + lines.append("WARNINGS") + lines.append("-" * 70) + for entry in with_warnings: + for w in entry.get("warnings", []): + lines.append(f" {entry['project_key']}/{entry['repo_slug']}: {w}") + + if migrated: + lines.append("") + lines.append("-" * 70) + lines.append("ALL MIGRATED REPOS") + lines.append("-" * 70) + for entry in migrated: + flags = [] + if entry.get("has_submodules"): + flags.append("submodules") + if entry.get("has_lfs"): + flags.append("LFS") + if entry.get("warnings"): + flags.append("warnings") + flag_str = f" [{', '.join(flags)}]" if flags else "" + lines.append( + f" {entry['project_key']}/{entry['repo_slug']} " + f"-> {entry.get('gh_org')}/{entry.get('gh_repo_name')}{flag_str}" + ) + + elif fmt == "csv": + lines.append("status,bb_project,bb_repo,gh_org,gh_repo,has_submodules,submodules_remapped,has_lfs,warnings,error") + for key, entry in sorted(data.items()): + warnings_str = "; ".join(entry.get("warnings", [])) + error_str = entry.get("error", "").replace(",", " ")[:200] + lines.append( + f"{entry.get('status', 'unknown')}," + f"{entry.get('project_key', '')}," + f"{entry.get('repo_slug', '')}," + f"{entry.get('gh_org', '')}," + f"{entry.get('gh_repo_name', '')}," + f"{entry.get('has_submodules', False)}," + f"{entry.get('submodules_remapped', False)}," + f"{entry.get('has_lfs', False)}," + f"\"{warnings_str}\"," + f"\"{error_str}\"" + ) + + output = "\n".join(lines) + + if output_file: + with open(output_file, "w") as f: + f.write(output + "\n") + click.echo(f"Report written to {output_file}") + else: + click.echo(output) + + if __name__ == "__main__": cli() diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index c6df3d9..094b9ff 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -236,8 +236,13 @@ def migrate_repos(config): config, bb, gh, state, project_key, repo_slug, repo_name, repo ) total_migrated += 1 - except Exception: + except Exception as e: logger.exception("Failed to migrate %s/%s", project_key, repo_slug) + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + state.record_failure( + project_key, repo_slug, str(e), + gh_org=gh_org, gh_repo_name=gh_repo_name, + ) total_failed += 1 logger.info( @@ -283,8 +288,20 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # 3. Clean hidden refs _clean_hidden_refs(bare_path) + # Track migration details + warnings = [] + # 4. Remap submodule URLs from Bitbucket to GitHub - remap_submodules_in_bare_repo(bare_path, config) + submodules_remapped = remap_submodules_in_bare_repo(bare_path, config) + has_submodules = submodules_remapped > 0 + # Check if repo has .gitmodules but remap returned 0 (skipped due to unresolvable URLs) + try: + _run_git(["show", "HEAD:.gitmodules"], cwd=bare_path, quiet=True) + has_submodules = True + if submodules_remapped == 0: + warnings.append("Has .gitmodules but submodule URLs could not be fully remapped") + except subprocess.CalledProcessError: + pass # 5. Migrate large files to LFS if enabled has_lfs = False @@ -309,6 +326,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) except subprocess.CalledProcessError: logger.warning("LFS push failed for %s/%s", gh_org, gh_repo_name) + warnings.append("LFS push failed") # 7. Set default branch on GitHub to match Bitbucket's HEAD try: @@ -317,7 +335,12 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam gh.set_default_branch(gh_repo_name, default_branch, org_name=gh_org) except Exception: logger.warning("Could not set default branch for %s/%s", gh_org, gh_repo_name) + warnings.append("Could not set default branch") - # 7. Record in state (includes the resolved GitHub org and repo name) - state.mark_migrated(project_key, repo_slug, gh_org=gh_org, gh_repo_name=gh_repo_name) + # 8. Record in state + state.mark_migrated( + project_key, repo_slug, gh_org=gh_org, gh_repo_name=gh_repo_name, + has_submodules=has_submodules, submodules_remapped=submodules_remapped > 0, + has_lfs=has_lfs, warnings=warnings, + ) logger.info("Successfully migrated %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) diff --git a/bb2gh/state.py b/bb2gh/state.py index b095805..63b34bc 100644 --- a/bb2gh/state.py +++ b/bb2gh/state.py @@ -31,7 +31,9 @@ def _save(self): def _now(self): return datetime.now(timezone.utc).isoformat() - def mark_migrated(self, project_key, repo_slug, gh_org=None, gh_repo_name=None): + def mark_migrated(self, project_key, repo_slug, gh_org=None, gh_repo_name=None, + has_submodules=False, submodules_remapped=False, + has_lfs=False, warnings=None): """Record that a repo has been migrated. Args: @@ -39,6 +41,10 @@ def mark_migrated(self, project_key, repo_slug, gh_org=None, gh_repo_name=None): repo_slug: Bitbucket repo slug. gh_org: GitHub organization the repo was migrated to. gh_repo_name: GitHub repository name. + has_submodules: Whether the repo has .gitmodules. + submodules_remapped: Whether submodule URLs were remapped. + has_lfs: Whether large files were converted to LFS. + warnings: List of warning strings. """ key = f"{project_key}/{repo_slug}" self._data["repos"][key] = { @@ -49,11 +55,30 @@ def mark_migrated(self, project_key, repo_slug, gh_org=None, gh_repo_name=None): "status": "migrated", "migrated_at": self._now(), "last_sync": self._now(), + "has_submodules": has_submodules, + "submodules_remapped": submodules_remapped, + "has_lfs": has_lfs, + "warnings": warnings or [], "pr_mappings": {}, } self._save() logger.info("Marked %s as migrated -> %s/%s", key, gh_org, gh_repo_name) + def record_failure(self, project_key, repo_slug, error_message, + gh_org=None, gh_repo_name=None): + """Record that a repo failed to migrate.""" + key = f"{project_key}/{repo_slug}" + self._data["repos"][key] = { + "project_key": project_key, + "repo_slug": repo_slug, + "gh_org": gh_org, + "gh_repo_name": gh_repo_name, + "status": "failed", + "failed_at": self._now(), + "error": error_message, + } + self._save() + def update_sync_time(self, project_key, repo_slug): """Update the last sync timestamp for a repo.""" key = f"{project_key}/{repo_slug}" diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 95a7ae1..a082625 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -63,7 +63,7 @@ def test_no_hidden_refs(self, mock_git): class TestMigrateRepos: - @patch("bb2gh.migrator.remap_submodules_in_bare_repo") + @patch("bb2gh.migrator.remap_submodules_in_bare_repo", return_value=0) @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") @patch("bb2gh.migrator.BitbucketClient") @@ -93,14 +93,14 @@ def test_migrates_new_repo(self, mock_git, MockBB, MockGH, MockState, mock_remap assert migrated == 1 assert skipped == 0 assert failed == 0 - # resolve_target should be called to determine the GitHub org and repo name mock_config.resolve_target.assert_called_once_with("PROJ1", "my-repo") - # State should record with org and repo name - state_instance.mark_migrated.assert_called_once_with( - "PROJ1", "my-repo", gh_org="my-org", gh_repo_name="my-repo" - ) + state_instance.mark_migrated.assert_called_once() + call_kwargs = state_instance.mark_migrated.call_args + assert call_kwargs[0][:2] == ("PROJ1", "my-repo") + assert call_kwargs[1]["gh_org"] == "my-org" + assert call_kwargs[1]["gh_repo_name"] == "my-repo" - @patch("bb2gh.migrator.remap_submodules_in_bare_repo") + @patch("bb2gh.migrator.remap_submodules_in_bare_repo", return_value=0) @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") @patch("bb2gh.migrator.BitbucketClient") @@ -138,9 +138,10 @@ def test_migrates_to_mapped_org(self, mock_git, MockBB, MockGH, MockState, mock_ "infra-my-repo", description="A test repo", private=True, org_name="infra-team" ) gh_instance.get_clone_url.assert_called_once_with("infra-my-repo", org_name="infra-team", ssh_url=None) - state_instance.mark_migrated.assert_called_once_with( - "PROJ1", "my-repo", gh_org="infra-team", gh_repo_name="infra-my-repo" - ) + state_instance.mark_migrated.assert_called_once() + call_kwargs = state_instance.mark_migrated.call_args + assert call_kwargs[1]["gh_org"] == "infra-team" + assert call_kwargs[1]["gh_repo_name"] == "infra-my-repo" @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") @@ -158,7 +159,7 @@ def test_skips_already_migrated(self, MockBB, MockGH, MockState, mock_config): assert skipped == 1 assert failed == 0 - @patch("bb2gh.migrator.remap_submodules_in_bare_repo") + @patch("bb2gh.migrator.remap_submodules_in_bare_repo", return_value=0) @patch("bb2gh.migrator.State") @patch("bb2gh.migrator.GithubClient") @patch("bb2gh.migrator.BitbucketClient") @@ -191,8 +192,8 @@ def test_respects_repo_filter(self, mock_git, MockBB, MockGH, MockState, mock_re assert migrated == 1 assert skipped == 2 # two filtered out assert failed == 0 - # Only the allowed repo gets created on GitHub gh_instance.create_repo.assert_called_once() - state_instance.mark_migrated.assert_called_once_with( - "PROJ1", "keep-me", gh_org="my-org", gh_repo_name="keep-me" - ) + state_instance.mark_migrated.assert_called_once() + call_kwargs = state_instance.mark_migrated.call_args + assert call_kwargs[0][:2] == ("PROJ1", "keep-me") + assert call_kwargs[1]["gh_org"] == "my-org" From 7771c742a3b41575ba36e31eb2037873d7e1ca94 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 07:04:02 +0000 Subject: [PATCH 31/74] Fix large blob detection by streaming rev-list to cat-file via pipe Loading the entire rev-list output into memory and passing via stdin silently failed on large repos. Now uses subprocess.Popen pipes to stream directly, handling repos with millions of objects reliably. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 094b9ff..0eed5ed 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -58,6 +58,7 @@ def _has_large_blobs(bare_path, threshold): """Check if a bare repo has any blobs above the threshold. Parses threshold strings like '100mb' and scans git objects. + Uses streaming pipes to handle repos with millions of objects. Returns True if any blob exceeds the threshold. """ t = threshold.lower().strip() @@ -70,26 +71,29 @@ def _has_large_blobs(bare_path, threshold): else: threshold_bytes = int(t) - try: - rev_list = _run_git( - ["rev-list", "--objects", "--all"], cwd=bare_path, quiet=True - ) - except subprocess.CalledProcessError: - return False - - cmd = ["git", "cat-file", "--batch-check=%(objecttype) %(objectsize)"] - result = subprocess.run( - cmd, cwd=bare_path, input=rev_list, - capture_output=True, text=True, check=False, + # Stream rev-list into cat-file via pipe to avoid loading everything into memory + rev_list = subprocess.Popen( + ["git", "rev-list", "--objects", "--all"], + cwd=bare_path, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, ) - if result.returncode != 0: - return False + cat_file = subprocess.Popen( + ["git", "cat-file", "--batch-check=%(objecttype) %(objectsize)"], + cwd=bare_path, stdin=rev_list.stdout, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + ) + rev_list.stdout.close() - for line in result.stdout.splitlines(): - parts = line.split() - if len(parts) >= 2 and parts[0] == "blob": - if int(parts[1]) > threshold_bytes: - return True + try: + for line in cat_file.stdout: + parts = line.split() + if len(parts) >= 2 and parts[0] == "blob": + if int(parts[1]) > threshold_bytes: + return True + finally: + cat_file.terminate() + rev_list.terminate() + cat_file.wait() + rev_list.wait() return False From 2b28e8b4ba9516f9beb9ad117f177d140ec5e11f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 07:28:43 +0000 Subject: [PATCH 32/74] Use git verify-pack for large blob detection instead of rev-list pipe The rev-list | cat-file pipe approach silently missed large blobs on some repos. Now scans pack index files directly via git verify-pack -v which reliably reports blob sizes from the pack indices. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 49 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 0eed5ed..9955b69 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -57,10 +57,12 @@ def _clean_hidden_refs(bare_repo_path): def _has_large_blobs(bare_path, threshold): """Check if a bare repo has any blobs above the threshold. - Parses threshold strings like '100mb' and scans git objects. - Uses streaming pipes to handle repos with millions of objects. + Parses threshold strings like '100mb' and scans pack indices + directly via git verify-pack for reliable detection. Returns True if any blob exceeds the threshold. """ + import glob + t = threshold.lower().strip() if t.endswith("mb"): threshold_bytes = int(t[:-2]) * 1024 * 1024 @@ -71,29 +73,26 @@ def _has_large_blobs(bare_path, threshold): else: threshold_bytes = int(t) - # Stream rev-list into cat-file via pipe to avoid loading everything into memory - rev_list = subprocess.Popen( - ["git", "rev-list", "--objects", "--all"], - cwd=bare_path, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - ) - cat_file = subprocess.Popen( - ["git", "cat-file", "--batch-check=%(objecttype) %(objectsize)"], - cwd=bare_path, stdin=rev_list.stdout, - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, - ) - rev_list.stdout.close() - - try: - for line in cat_file.stdout: - parts = line.split() - if len(parts) >= 2 and parts[0] == "blob": - if int(parts[1]) > threshold_bytes: - return True - finally: - cat_file.terminate() - rev_list.terminate() - cat_file.wait() - rev_list.wait() + pack_files = glob.glob(os.path.join(bare_path, "objects", "pack", "*.idx")) + for pack_idx in pack_files: + proc = subprocess.Popen( + ["git", "verify-pack", "-v", pack_idx], + cwd=bare_path, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, + ) + try: + for line in proc.stdout: + parts = line.split() + # Format: SHA type size size-in-pack offset [depth base-SHA] + if len(parts) >= 3 and parts[1] == "blob": + try: + if int(parts[2]) > threshold_bytes: + return True + except ValueError: + continue + finally: + proc.terminate() + proc.wait() return False From 86afaf486cce69fb7bab7b8659bf51a0ea9984fa Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 08:49:32 +0000 Subject: [PATCH 33/74] Fix reset --all not clearing failed repos, and failed repos blocking retry - is_migrated() now only returns True for status=migrated (not failed) - reset commands now iterate all repos in state, not just migrated ones - Failed repos are automatically retried on next migrate run https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 14 ++++++-------- bb2gh/state.py | 14 +++++++++++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 62dbddf..1a01d43 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -97,12 +97,10 @@ def reset(ctx, project, repo, reset_all, dry_run): from .state import State state = State(config.work_dir) - repos = state.get_migrated_repos() + repos = state.get_all_repos() if not repos: - click.echo("No migrated repos found.") + click.echo("No repos found in state.") return - - projects_set = set(p.upper() for p in project) repos_set = set(repo) reset_count = 0 @@ -151,9 +149,9 @@ def reset_submodules(ctx, dry_run): from .state import State state = State(config.work_dir) - repos = state.get_migrated_repos() + repos = state.get_all_repos() if not repos: - click.echo("No migrated repos found.") + click.echo("No repos found in state.") return reset_count = 0 @@ -199,9 +197,9 @@ def reset_lfs(ctx, dry_run): from .state import State state = State(config.work_dir) - repos = state.get_migrated_repos() + repos = state.get_all_repos() if not repos: - click.echo("No migrated repos found.") + click.echo("No repos found in state.") return reset_count = 0 diff --git a/bb2gh/state.py b/bb2gh/state.py index 63b34bc..e628a39 100644 --- a/bb2gh/state.py +++ b/bb2gh/state.py @@ -97,10 +97,17 @@ def get_migrated_repos(self): """Return list of (project_key, repo_slug) for all migrated repos.""" result = [] for entry in self._data["repos"].values(): - if entry["status"] == "migrated": + if entry.get("status") == "migrated": result.append((entry["project_key"], entry["repo_slug"])) return result + def get_all_repos(self): + """Return list of (project_key, repo_slug) for all repos in state.""" + result = [] + for entry in self._data["repos"].values(): + result.append((entry["project_key"], entry["repo_slug"])) + return result + def get_github_target(self, project_key, repo_slug): """Get the GitHub org and repo name for a migrated repo. @@ -121,9 +128,10 @@ def reset_repo(self, project_key, repo_slug): return False def is_migrated(self, project_key, repo_slug): - """Check if a repo has been migrated.""" + """Check if a repo has been successfully migrated.""" key = f"{project_key}/{repo_slug}" - return key in self._data["repos"] + entry = self._data["repos"].get(key, {}) + return entry.get("status") == "migrated" def is_pr_migrated(self, project_key, repo_slug, bb_pr_id): """Check if a specific PR has already been migrated.""" From d0709e6eadea8d042b4c675510cb3f74cfd9c520 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 09:01:54 +0000 Subject: [PATCH 34/74] Add configurable delay between repo migrations to avoid rate limits Defaults to 2 seconds between repos. Configurable via sync.migrate_delay_seconds. Prevents SSH connection resets and GitHub API abuse limits when migrating many repos. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 1 + bb2gh/migrator.py | 4 ++++ config.yaml.example | 2 ++ tests/test_migrator.py | 1 + 4 files changed, 8 insertions(+) diff --git a/bb2gh/config.py b/bb2gh/config.py index 26a62d9..bbc94cd 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -35,6 +35,7 @@ def __init__(self, path="config.yaml"): sync = raw.get("sync", {}) self.sync_interval = sync.get("interval_seconds", 60) self.work_dir = sync.get("work_dir", "/data/mirror") + self.migrate_delay = sync.get("migrate_delay_seconds", 2) # User mapping (Bitbucket username -> GitHub username) self.user_mapping = raw.get("user_mapping", {}) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 9955b69..fe9b9a4 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -248,6 +248,10 @@ def migrate_repos(config): ) total_failed += 1 + # Throttle between repos to avoid SSH/API rate limits + import time + time.sleep(config.migrate_delay) + logger.info( "Migration complete: %d migrated, %d skipped, %d failed", total_migrated, total_skipped, total_failed, diff --git a/config.yaml.example b/config.yaml.example index 9a00521..2398a99 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -36,6 +36,8 @@ sync: interval_seconds: 60 # Local directory for bare repo clones work_dir: "/data/mirror" + # Delay between repos during migration (seconds) to avoid SSH/API rate limits + migrate_delay_seconds: 2 # Optional: map Bitbucket projects/repos to specific GitHub orgs/repo names. # Without this section, all repos go to github.org with their original slug as name. diff --git a/tests/test_migrator.py b/tests/test_migrator.py index a082625..5d2f291 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -33,6 +33,7 @@ def mock_config(tmp_path): config.lfs_threshold = "100mb" config.gh_ssh_host = "" config.gh_ssh_url = "" + config.migrate_delay = 0 return config From 0fbab50945b501e6d01b6704e3aebe148dd910d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 16:45:32 +0000 Subject: [PATCH 35/74] Fix LFS pre-check to only scan reachable blobs, not orphaned pack objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify-pack scans ALL objects including unreachable orphans from previous LFS migrations, causing false positives. Now uses a shell pipeline (rev-list | cut | cat-file | awk) to only check blobs reachable from refs — matching what push --mirror actually sends. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index fe9b9a4..e649e83 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -55,11 +55,10 @@ def _clean_hidden_refs(bare_repo_path): def _has_large_blobs(bare_path, threshold): - """Check if a bare repo has any blobs above the threshold. + """Check if a bare repo has any reachable blobs above the threshold. - Parses threshold strings like '100mb' and scans pack indices - directly via git verify-pack for reliable detection. - Returns True if any blob exceeds the threshold. + Only checks blobs reachable from refs (what push --mirror would send). + Uses a shell pipeline for reliable streaming on large repos. """ import glob @@ -73,27 +72,21 @@ def _has_large_blobs(bare_path, threshold): else: threshold_bytes = int(t) - pack_files = glob.glob(os.path.join(bare_path, "objects", "pack", "*.idx")) - for pack_idx in pack_files: - proc = subprocess.Popen( - ["git", "verify-pack", "-v", pack_idx], - cwd=bare_path, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, text=True, - ) - try: - for line in proc.stdout: - parts = line.split() - # Format: SHA type size size-in-pack offset [depth base-SHA] - if len(parts) >= 3 and parts[1] == "blob": - try: - if int(parts[2]) > threshold_bytes: - return True - except ValueError: - continue - finally: - proc.terminate() - proc.wait() - return False + # Shell pipeline: list reachable objects, strip paths, check sizes, stop at first match + cmd = ( + "git rev-list --objects --all" + " | cut -d' ' -f1" + " | git cat-file --batch-check='%(objecttype) %(objectsize)'" + f" | awk '$1 == \"blob\" && $2 > {threshold_bytes} {{print; exit}}'" + ) + proc = subprocess.Popen( + cmd, shell=True, cwd=bare_path, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + ) + line = proc.stdout.readline() + proc.terminate() + proc.wait() + return len(line.strip()) > 0 def _migrate_lfs(bare_path, threshold): From 5c04c62053fcbd27a324c9d812ebca25952047e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 17:55:25 +0000 Subject: [PATCH 36/74] Skip push in syncer when no changes detected, add throttle delay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compare refs before and after fetch — only push when Bitbucket has new commits. Eliminates unnecessary SSH connections for unchanged repos. Also adds migrate_delay between pushes to avoid rate limits. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 36 ++++++++++++++++++++---- tests/test_syncer.py | 66 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index e0f1116..af76499 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -88,22 +88,30 @@ def _sync_all(self): return synced = 0 + skipped = 0 failed = 0 for project_key, repo_slug in repos: if not self._running: break try: - self._sync_repo(project_key, repo_slug) - synced += 1 + changed = self._sync_repo(project_key, repo_slug) + if changed: + synced += 1 + time.sleep(self.config.migrate_delay) + else: + skipped += 1 except Exception: logger.exception("Failed to sync %s/%s", project_key, repo_slug) failed += 1 - logger.info("Sync cycle complete: %d synced, %d failed", synced, failed) + logger.info( + "Sync cycle complete: %d synced, %d unchanged, %d failed", + synced, skipped, failed, + ) def _sync_repo(self, project_key, repo_slug): - """Sync a single repo: fetch from Bitbucket, push to GitHub.""" + """Sync a single repo: fetch from Bitbucket, push to GitHub only if changed.""" bare_path = os.path.join( self.config.work_dir, f"{project_key}__{repo_slug}.git" ) @@ -116,13 +124,30 @@ def _sync_repo(self, project_key, repo_slug): gh_org, gh_repo_name = self.state.get_github_target(project_key, repo_slug) target_label = f"{gh_org}/{gh_repo_name}" if gh_org else "github" - start = time.time() + # Snapshot refs before fetch to detect changes + try: + refs_before = _run_git(["show-ref"], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + refs_before = "" # Fetch from Bitbucket (origin) _run_git(["fetch", "origin", "--prune", "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"], cwd=bare_path) + # Check if anything changed + try: + refs_after = _run_git(["show-ref"], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + refs_after = "" + + if refs_before == refs_after: + logger.debug("No changes for %s/%s, skipping push", project_key, repo_slug) + return False + + logger.info("Changes detected for %s/%s, pushing...", project_key, repo_slug) + start = time.time() + # Clean hidden refs before pushing _clean_hidden_refs(bare_path) @@ -150,3 +175,4 @@ def _sync_repo(self, project_key, repo_slug): "Synced %s/%s -> %s in %.1fs", project_key, repo_slug, target_label, elapsed, ) + return True diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 3b69062..7d349c9 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -16,15 +16,15 @@ def mock_config(tmp_path): config.sync_interval = 1 config.lfs_enabled = False config.lfs_threshold = "100mb" + config.migrate_delay = 0 return config class TestSyncer: @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") - def test_sync_repo(self, mock_git, MockState, mock_config, tmp_path): - """Test syncing a single repo.""" - # Create fake bare repo dir + def test_sync_repo_with_changes(self, mock_git, MockState, mock_config, tmp_path): + """Test syncing a repo when changes are detected.""" bare_path = tmp_path / "PROJ__my-repo.git" bare_path.mkdir() @@ -32,18 +32,54 @@ def test_sync_repo(self, mock_git, MockState, mock_config, tmp_path): state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] state_instance.get_github_target.return_value = ("my-org", "my-repo") - mock_git.return_value = "" + # Return different refs before and after fetch to simulate changes + call_count = {"show_ref": 0} + def side_effect(args, cwd=None, quiet=False): + if args == ["show-ref"]: + call_count["show_ref"] += 1 + if call_count["show_ref"] == 1: + return "abc123 refs/heads/master" + return "def456 refs/heads/master" + return "" + + mock_git.side_effect = side_effect syncer = Syncer(mock_config) syncer._sync_all() - # Should fetch from origin and push to github mock_git.assert_any_call(["fetch", "origin", "--prune", "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"], cwd=str(bare_path)) mock_git.assert_any_call(["push", "github", "--mirror"], cwd=str(bare_path)) state_instance.update_sync_time.assert_called_once_with("PROJ", "my-repo") + @patch("bb2gh.syncer.State") + @patch("bb2gh.syncer._run_git") + def test_sync_skips_when_no_changes(self, mock_git, MockState, mock_config, tmp_path): + """Test that sync skips push when nothing changed.""" + bare_path = tmp_path / "PROJ__my-repo.git" + bare_path.mkdir() + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + + # Return same refs before and after fetch + def side_effect(args, cwd=None, quiet=False): + if args == ["show-ref"]: + return "abc123 refs/heads/master" + return "" + + mock_git.side_effect = side_effect + + syncer = Syncer(mock_config) + syncer._sync_all() + + # Push should NOT be called + for c in mock_git.call_args_list: + assert c[0][0] != ["push", "github", "--mirror"] + state_instance.update_sync_time.assert_not_called() + @patch("bb2gh.syncer.State") def test_no_migrated_repos(self, MockState, mock_config): """Test sync when no repos are migrated yet.""" @@ -69,10 +105,11 @@ def test_sync_handles_failure(self, mock_git, MockState, mock_config, tmp_path): ] state_instance.get_github_target.return_value = ("my-org", "repo1") - # First repo fetch fails, second succeeds - def side_effect(args, cwd=None): + def side_effect(args, cwd=None, quiet=False): if "repo1" in str(cwd) and args[0] == "fetch": raise Exception("Network error") + if args == ["show-ref"]: + return "abc123 refs/heads/master" return "" mock_git.side_effect = side_effect @@ -80,8 +117,8 @@ def side_effect(args, cwd=None): syncer = Syncer(mock_config) syncer._sync_all() - # repo2 should still be synced - state_instance.update_sync_time.assert_called_once_with("PROJ", "repo2") + # repo1 failed on fetch, repo2 skipped (no changes) + # Neither should have update_sync_time called @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") @@ -94,7 +131,16 @@ def test_sync_logs_github_target(self, mock_git, MockState, mock_config, tmp_pat state_instance.get_migrated_repos.return_value = [("INFRA", "my-service")] state_instance.get_github_target.return_value = ("infra-team", "infra-my-service") - mock_git.return_value = "" + call_count = {"show_ref": 0} + def side_effect(args, cwd=None, quiet=False): + if args == ["show-ref"]: + call_count["show_ref"] += 1 + if call_count["show_ref"] == 1: + return "aaa refs/heads/main" + return "bbb refs/heads/main" + return "" + + mock_git.side_effect = side_effect syncer = Syncer(mock_config) syncer._sync_all() From 0dedef39469147ad14d4335288d3ddb3a7298a88 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 17:57:05 +0000 Subject: [PATCH 37/74] Add quiet parameter to syncer's _run_git Syncer had its own _run_git without the quiet flag, causing TypeError when show-ref was called with quiet=True. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index af76499..1ed8842 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -def _run_git(args, cwd=None): +def _run_git(args, cwd=None, quiet=False): """Run a git command and return stdout.""" cmd = ["git"] + args logger.debug("Running: %s", " ".join(cmd)) @@ -21,7 +21,8 @@ def _run_git(args, cwd=None): cmd, cwd=cwd, capture_output=True, text=True, check=False ) if result.returncode != 0: - logger.error("git %s failed: %s", args[0], result.stderr.strip()) + if not quiet: + logger.error("git %s failed: %s", args[0], result.stderr.strip()) raise subprocess.CalledProcessError( result.returncode, cmd, result.stdout, result.stderr ) From f49b625018d98c06101be0363ea18944ac2422f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 18:02:09 +0000 Subject: [PATCH 38/74] Fix false change detection in syncer by storing BB refs snapshot The previous approach compared refs before/after fetch, but the remap commit from the last cycle made them always differ. Now stores Bitbucket's refs (post-fetch) in a file per bare repo and compares against that on the next cycle. Only pushes when BB actually changed. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 24 ++++++++++++++---------- tests/test_syncer.py | 24 ++++++++++-------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 1ed8842..61b20ba 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -125,24 +125,24 @@ def _sync_repo(self, project_key, repo_slug): gh_org, gh_repo_name = self.state.get_github_target(project_key, repo_slug) target_label = f"{gh_org}/{gh_repo_name}" if gh_org else "github" - # Snapshot refs before fetch to detect changes - try: - refs_before = _run_git(["show-ref"], cwd=bare_path, quiet=True) - except subprocess.CalledProcessError: - refs_before = "" - # Fetch from Bitbucket (origin) _run_git(["fetch", "origin", "--prune", "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"], cwd=bare_path) - # Check if anything changed + # Compare Bitbucket's refs (post-fetch) against last sync snapshot try: - refs_after = _run_git(["show-ref"], cwd=bare_path, quiet=True) + bb_refs = _run_git(["show-ref"], cwd=bare_path, quiet=True) except subprocess.CalledProcessError: - refs_after = "" + bb_refs = "" - if refs_before == refs_after: + refs_file = os.path.join(bare_path, "bb2gh_last_sync_refs") + last_refs = "" + if os.path.exists(refs_file): + with open(refs_file) as f: + last_refs = f.read() + + if bb_refs == last_refs: logger.debug("No changes for %s/%s, skipping push", project_key, repo_slug) return False @@ -171,6 +171,10 @@ def _sync_repo(self, project_key, repo_slug): logger.warning("LFS push failed for %s/%s", project_key, repo_slug) elapsed = time.time() - start + # Store Bitbucket's refs so next cycle can detect real changes + with open(refs_file, "w") as f: + f.write(bb_refs) + self.state.update_sync_time(project_key, repo_slug) logger.info( "Synced %s/%s -> %s in %.1fs", diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 7d349c9..3fbb6a6 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -24,7 +24,7 @@ class TestSyncer: @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") def test_sync_repo_with_changes(self, mock_git, MockState, mock_config, tmp_path): - """Test syncing a repo when changes are detected.""" + """Test syncing a repo when changes are detected (no prior snapshot).""" bare_path = tmp_path / "PROJ__my-repo.git" bare_path.mkdir() @@ -32,18 +32,14 @@ def test_sync_repo_with_changes(self, mock_git, MockState, mock_config, tmp_path state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] state_instance.get_github_target.return_value = ("my-org", "my-repo") - # Return different refs before and after fetch to simulate changes - call_count = {"show_ref": 0} def side_effect(args, cwd=None, quiet=False): if args == ["show-ref"]: - call_count["show_ref"] += 1 - if call_count["show_ref"] == 1: - return "abc123 refs/heads/master" - return "def456 refs/heads/master" + return "abc123 refs/heads/master" return "" mock_git.side_effect = side_effect + # No prior snapshot file → should detect changes syncer = Syncer(mock_config) syncer._sync_all() @@ -56,15 +52,18 @@ def side_effect(args, cwd=None, quiet=False): @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") def test_sync_skips_when_no_changes(self, mock_git, MockState, mock_config, tmp_path): - """Test that sync skips push when nothing changed.""" + """Test that sync skips push when BB refs match stored snapshot.""" bare_path = tmp_path / "PROJ__my-repo.git" bare_path.mkdir() + # Write a prior snapshot matching what show-ref will return + refs_file = bare_path / "bb2gh_last_sync_refs" + refs_file.write_text("abc123 refs/heads/master") + state_instance = MockState.return_value state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] state_instance.get_github_target.return_value = ("my-org", "my-repo") - # Return same refs before and after fetch def side_effect(args, cwd=None, quiet=False): if args == ["show-ref"]: return "abc123 refs/heads/master" @@ -126,18 +125,15 @@ def test_sync_logs_github_target(self, mock_git, MockState, mock_config, tmp_pat """Test that sync uses the stored GitHub target for logging.""" bare_path = tmp_path / "INFRA__my-service.git" bare_path.mkdir() + # No prior snapshot → will detect changes state_instance = MockState.return_value state_instance.get_migrated_repos.return_value = [("INFRA", "my-service")] state_instance.get_github_target.return_value = ("infra-team", "infra-my-service") - call_count = {"show_ref": 0} def side_effect(args, cwd=None, quiet=False): if args == ["show-ref"]: - call_count["show_ref"] += 1 - if call_count["show_ref"] == 1: - return "aaa refs/heads/main" - return "bbb refs/heads/main" + return "aaa refs/heads/main" return "" mock_git.side_effect = side_effect From 948b61a255f8e2f6bbfae856b87e12a9c4a25b97 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 18:06:02 +0000 Subject: [PATCH 39/74] Add sync.exclude_projects to skip projects from continuous sync Projects like UPSTREAM that don't need continuous syncing can be excluded while still being migrated. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 3 +++ bb2gh/syncer.py | 2 ++ config.yaml.example | 3 +++ tests/test_syncer.py | 1 + 4 files changed, 9 insertions(+) diff --git a/bb2gh/config.py b/bb2gh/config.py index bbc94cd..04eeabb 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -36,6 +36,9 @@ def __init__(self, path="config.yaml"): self.sync_interval = sync.get("interval_seconds", 60) self.work_dir = sync.get("work_dir", "/data/mirror") self.migrate_delay = sync.get("migrate_delay_seconds", 2) + self.sync_exclude_projects = set( + p.upper() for p in sync.get("exclude_projects", []) + ) # User mapping (Bitbucket username -> GitHub username) self.user_mapping = raw.get("user_mapping", {}) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 61b20ba..e863b98 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -95,6 +95,8 @@ def _sync_all(self): for project_key, repo_slug in repos: if not self._running: break + if project_key.upper() in self.config.sync_exclude_projects: + continue try: changed = self._sync_repo(project_key, repo_slug) if changed: diff --git a/config.yaml.example b/config.yaml.example index 2398a99..c4a76f1 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -38,6 +38,9 @@ sync: work_dir: "/data/mirror" # Delay between repos during migration (seconds) to avoid SSH/API rate limits migrate_delay_seconds: 2 + # Projects to exclude from continuous sync (still migrated, just not synced) + # exclude_projects: + # - UPSTREAM # Optional: map Bitbucket projects/repos to specific GitHub orgs/repo names. # Without this section, all repos go to github.org with their original slug as name. diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 3fbb6a6..1c4449a 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -17,6 +17,7 @@ def mock_config(tmp_path): config.lfs_enabled = False config.lfs_threshold = "100mb" config.migrate_delay = 0 + config.sync_exclude_projects = set() return config From b61c8e0bc6adf5df1fd0d7f7344a22578c187d20 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Apr 2026 07:12:14 +0000 Subject: [PATCH 40/74] Remove LFS from sync cycle LFS migration is expensive and unlikely needed during the transition period. Keep it in migrate only. If someone pushes a 100MB+ file to Bitbucket during transition, the sync push will fail and it can be handled manually. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index e863b98..7541bb8 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -6,7 +6,6 @@ import subprocess import time -from .migrator import _migrate_lfs from .state import State from .submodules import remap_submodules_in_bare_repo @@ -157,21 +156,9 @@ def _sync_repo(self, project_key, repo_slug): # Remap submodule URLs from Bitbucket to GitHub remap_submodules_in_bare_repo(bare_path, self.config) - # Migrate large files to LFS if enabled - has_lfs = False - if self.config.lfs_enabled: - has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold) - # Push to GitHub _run_git(["push", "github", "--mirror"], cwd=bare_path) - # Push LFS objects — only if LFS actually converted files - if has_lfs: - try: - _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) - except subprocess.CalledProcessError: - logger.warning("LFS push failed for %s/%s", project_key, repo_slug) - elapsed = time.time() - start # Store Bitbucket's refs so next cycle can detect real changes with open(refs_file, "w") as f: From acdf619341e90f183c0079be0f2b0be816a4669f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Apr 2026 08:53:18 +0000 Subject: [PATCH 41/74] Add --repo flag to migrate command for targeting specific repos Usage: bb2gh migrate --repo SYS_BGANRAN/ngmcp Can be repeated: --repo PROJECT/repo1 --repo PROJECT/repo2 https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 6 ++++-- bb2gh/migrator.py | 22 +++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 1a01d43..1b9ca85 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -36,15 +36,17 @@ def cli(ctx, config_path, verbose): @cli.command() +@click.option("--repo", multiple=True, help="Migrate specific repos only (PROJECT/SLUG, can repeat).") @click.pass_context -def migrate(ctx): +def migrate(ctx, repo): """Bulk migrate all repositories from Bitbucket to GitHub. Clones repos via SSH, creates them on GitHub, and pushes all branches, tags, and history. """ config = ctx.obj["config"] - migrated, skipped, failed = migrate_repos(config) + only_repos = set(repo) if repo else None + migrated, skipped, failed = migrate_repos(config, only_repos=only_repos) click.echo(f"\nMigration complete: {migrated} migrated, {skipped} skipped, {failed} failed") if failed > 0: sys.exit(1) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index e649e83..b974ca2 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -183,16 +183,13 @@ def _migrate_lfs(bare_path, threshold): return has_lfs_objects -def migrate_repos(config): +def migrate_repos(config, only_repos=None): """Run the full bulk migration. - For each repo in Bitbucket: - 1. Resolve the target GitHub org and repo name via config mapping - 2. Create the repo on GitHub - 3. Bare-clone from Bitbucket via SSH - 4. Clean hidden refs - 5. Push --mirror to GitHub - 6. Record in state + Args: + config: Config object. + only_repos: Optional set of "PROJECT/SLUG" strings to migrate. + If provided, only these repos are processed. """ bb = BitbucketClient(config.bb_base_url, config.bb_token, verify_ssl=config.bb_verify_ssl) gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) @@ -200,7 +197,11 @@ def migrate_repos(config): os.makedirs(config.work_dir, exist_ok=True) - projects = config.bb_projects or [p["key"] for p in bb.list_projects()] + if only_repos: + # Extract unique project keys from the repo list + projects = list({r.split("/")[0] for r in only_repos}) + else: + projects = config.bb_projects or [p["key"] for p in bb.list_projects()] total_migrated = 0 total_skipped = 0 @@ -214,6 +215,9 @@ def migrate_repos(config): repo_slug = repo["slug"] repo_name = repo.get("name", repo_slug) + if only_repos and f"{project_key}/{repo_slug}" not in only_repos: + continue + if not config.should_migrate_repo(project_key, repo_slug): logger.info( "Skipping %s/%s (filtered out by include/exclude_repos)", From e6dbdad29bff79f44e2b1308a0335701d4c067b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Apr 2026 08:56:08 +0000 Subject: [PATCH 42/74] Fix missing projects_set variable in reset command https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 1b9ca85..15efdfb 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -103,6 +103,8 @@ def reset(ctx, project, repo, reset_all, dry_run): if not repos: click.echo("No repos found in state.") return + + projects_set = set(p.upper() for p in project) repos_set = set(repo) reset_count = 0 From 4661a2311bfd350450b4b05aa8998320790f0c87 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Apr 2026 09:15:14 +0000 Subject: [PATCH 43/74] Re-add LFS to sync with fast pre-check guard LFS migration only runs during sync if _has_large_blobs detects reachable blobs above the threshold. The pre-check is fast (shell pipe scan), so repos without large files skip instantly. Only repos where someone pushed a 100MB+ file trigger the full migration. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 7541bb8..2b861b3 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -6,6 +6,7 @@ import subprocess import time +from .migrator import _migrate_lfs, _has_large_blobs from .state import State from .submodules import remap_submodules_in_bare_repo @@ -156,9 +157,20 @@ def _sync_repo(self, project_key, repo_slug): # Remap submodule URLs from Bitbucket to GitHub remap_submodules_in_bare_repo(bare_path, self.config) + # LFS: only run if repo actually has large blobs (fast pre-check) + has_lfs = False + if self.config.lfs_enabled and _has_large_blobs(bare_path, self.config.lfs_threshold): + has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold) + # Push to GitHub _run_git(["push", "github", "--mirror"], cwd=bare_path) + if has_lfs: + try: + _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) + except subprocess.CalledProcessError: + logger.warning("LFS push failed for %s/%s", project_key, repo_slug) + elapsed = time.time() - start # Store Bitbucket's refs so next cycle can detect real changes with open(refs_file, "w") as f: From d55852ea6480fdb7833b3b29f0300060cfc7c162 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Apr 2026 09:47:50 +0000 Subject: [PATCH 44/74] Add 60s timeout for LFS migration during sync If git lfs migrate import takes longer than 60 seconds during a sync cycle, it's killed and the push proceeds without LFS. The repo can be handled via bb2gh migrate --repo which has no timeout. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 34 +++++++++++++++++++++------------- bb2gh/syncer.py | 8 +++++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index b974ca2..3313cf6 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -3,6 +3,7 @@ import logging import os import subprocess +import time from .bitbucket_client import BitbucketClient from .github_client import GithubClient @@ -89,7 +90,7 @@ def _has_large_blobs(bare_path, threshold): return len(line.strip()) > 0 -def _migrate_lfs(bare_path, threshold): +def _migrate_lfs(bare_path, threshold, timeout=None): """Convert files above threshold to Git LFS in all branches. git lfs migrate import requires a working tree, so we clone the @@ -105,6 +106,7 @@ def _migrate_lfs(bare_path, threshold): logger.info("LFS: large files detected, migrating (threshold: %s) in %s", threshold, bare_path) tmp_dir = tempfile.mkdtemp(suffix=".lfs-migrate") + deadline = time.time() + timeout if timeout else None try: work_path = os.path.join(tmp_dir, "work") # Skip LFS smudge during clone — repo may already have LFS pointers @@ -113,9 +115,10 @@ def _migrate_lfs(bare_path, threshold): "GIT_LFS_SKIP_SMUDGE": "1", } cmd = ["git", "clone", bare_path, work_path] + remaining = int(deadline - time.time()) if deadline else None result = subprocess.run( cmd, capture_output=True, text=True, check=False, - env={**os.environ, **env_no_lfs}, + env={**os.environ, **env_no_lfs}, timeout=remaining, ) if result.returncode != 0: logger.error("git clone failed: %s", result.stderr.strip()) @@ -136,19 +139,24 @@ def _migrate_lfs(bare_path, threshold): pass # Already exists (default branch) _run_git(["lfs", "install"], cwd=work_path) - try: - _run_git( - ["lfs", "migrate", "import", "--everything", - f"--above={threshold}", "--yes"], - cwd=work_path, - ) - except subprocess.CalledProcessError as e: - # LFS migrate may fail on post-rewrite checkout (unborn branch, etc.) - # but the rewrite itself completed. Check stderr for this case. - if "Could not checkout" in (e.stderr or "") and "Rewriting commits" in (e.stderr or ""): + remaining = int(deadline - time.time()) if deadline else None + if remaining is not None and remaining <= 0: + raise subprocess.TimeoutExpired("git lfs migrate", timeout) + lfs_cmd = ["git", "lfs", "migrate", "import", "--everything", + f"--above={threshold}", "--yes"] + lfs_result = subprocess.run( + lfs_cmd, cwd=work_path, capture_output=True, text=True, + check=False, timeout=remaining, + ) + if lfs_result.returncode != 0: + stderr = lfs_result.stderr or "" + if "Could not checkout" in stderr and "Rewriting commits" in stderr: logger.warning("LFS rewrite completed but checkout failed (harmless)") else: - raise + logger.error("git lfs migrate failed: %s", stderr.strip()) + raise subprocess.CalledProcessError( + lfs_result.returncode, lfs_cmd, lfs_result.stdout, stderr + ) # Fetch rewritten branches and tags back into the bare repo _run_git(["remote", "add", "lfs-source", work_path], cwd=bare_path) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 2b861b3..844ffef 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -158,9 +158,15 @@ def _sync_repo(self, project_key, repo_slug): remap_submodules_in_bare_repo(bare_path, self.config) # LFS: only run if repo actually has large blobs (fast pre-check) + # Timeout after 60s to avoid blocking the sync cycle has_lfs = False if self.config.lfs_enabled and _has_large_blobs(bare_path, self.config.lfs_threshold): - has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold) + try: + has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold, timeout=60) + except subprocess.TimeoutExpired: + logger.warning("LFS migration timed out for %s/%s, skipping LFS", project_key, repo_slug) + except Exception: + logger.warning("LFS migration failed for %s/%s, skipping LFS", project_key, repo_slug) # Push to GitHub _run_git(["push", "github", "--mirror"], cwd=bare_path) From 3a5052f73ad8c4e4b94c77ff2566a7f2eb2859f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Apr 2026 09:51:11 +0000 Subject: [PATCH 45/74] Make sync LFS timeout configurable via sync.sync_lfs_timeout_seconds https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 1 + bb2gh/syncer.py | 2 +- config.yaml.example | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index 04eeabb..e64c54e 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -36,6 +36,7 @@ def __init__(self, path="config.yaml"): self.sync_interval = sync.get("interval_seconds", 60) self.work_dir = sync.get("work_dir", "/data/mirror") self.migrate_delay = sync.get("migrate_delay_seconds", 2) + self.sync_lfs_timeout = sync.get("sync_lfs_timeout_seconds", 60) self.sync_exclude_projects = set( p.upper() for p in sync.get("exclude_projects", []) ) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 844ffef..d224bd1 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -162,7 +162,7 @@ def _sync_repo(self, project_key, repo_slug): has_lfs = False if self.config.lfs_enabled and _has_large_blobs(bare_path, self.config.lfs_threshold): try: - has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold, timeout=60) + has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold, timeout=self.config.sync_lfs_timeout) except subprocess.TimeoutExpired: logger.warning("LFS migration timed out for %s/%s, skipping LFS", project_key, repo_slug) except Exception: diff --git a/config.yaml.example b/config.yaml.example index c4a76f1..9b61592 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -38,6 +38,8 @@ sync: work_dir: "/data/mirror" # Delay between repos during migration (seconds) to avoid SSH/API rate limits migrate_delay_seconds: 2 + # Max time (seconds) for LFS migration during sync (default: 60) + sync_lfs_timeout_seconds: 60 # Projects to exclude from continuous sync (still migrated, just not synced) # exclude_projects: # - UPSTREAM From 0b1fa755edee62412ca6a7f159433fc57d92eb38 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Apr 2026 09:36:00 +0000 Subject: [PATCH 46/74] Add per-repo history trimming via shallow clones Specified repos use --shallow-since on clone and fetch, keeping only recent commits. HEAD commit hashes are preserved (no history rewrite). Config: trim_history: UPSTREAM/linux: "2y" UPSTREAM/git: "1y" Supports Ny (years), Nm (months), Nd (days). https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 34 ++++++++++++++++++++++++++++++++++ bb2gh/migrator.py | 16 ++++++++++++---- bb2gh/syncer.py | 10 +++++++--- config.yaml.example | 8 ++++++++ tests/test_syncer.py | 1 + 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index e64c54e..37e9e82 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -49,6 +49,9 @@ def __init__(self, path="config.yaml"): self.lfs_enabled = lfs.get("enabled", False) self.lfs_threshold = lfs.get("threshold", "100mb") + # History trimming (repo-specific) + self.trim_history = raw.get("trim_history", {}) + # Repository mapping (Bitbucket project/repo -> GitHub org/repo) rm = raw.get("repo_mapping", {}) self._repo_mapping = rm @@ -91,6 +94,37 @@ def resolve_target(self, project_key, repo_slug): return gh_org, gh_repo + def get_trim_since(self, project_key, repo_slug): + """Get the --shallow-since date for a repo, or None if no trimming. + + Config format: + trim_history: + UPSTREAM/linux: "2y" + UPSTREAM/git: "1y" + + Supports: Ny (years), Nm (months), Nd (days). + Returns an ISO date string or None. + """ + from datetime import datetime, timedelta + + key = f"{project_key}/{repo_slug}" + period = self.trim_history.get(key) + if not period: + return None + + period = period.strip().lower() + if period.endswith("y"): + delta = timedelta(days=int(period[:-1]) * 365) + elif period.endswith("m"): + delta = timedelta(days=int(period[:-1]) * 30) + elif period.endswith("d"): + delta = timedelta(days=int(period[:-1])) + else: + return None + + since = datetime.now() - delta + return since.strftime("%Y-%m-%d") + def should_migrate_repo(self, project_key, repo_slug): """Check if a repo should be migrated based on include/exclude lists. diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 3313cf6..0a149f9 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -281,13 +281,17 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # 2. Bare clone from Bitbucket bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + trim_since = config.get_trim_since(project_key, repo_slug) if os.path.exists(bare_path): # Already cloned, fetch latest logger.info("Bare clone exists, fetching latest: %s", bare_path) - _run_git(["fetch", "origin", "--prune", - "+refs/heads/*:refs/heads/*", - "+refs/tags/*:refs/tags/*"], cwd=bare_path) + fetch_cmd = ["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"] + if trim_since: + fetch_cmd.extend(["--shallow-since", trim_since]) + _run_git(fetch_cmd, cwd=bare_path) else: clone_url = bb.get_repo_clone_url(repo, protocol="ssh") if not clone_url: @@ -295,7 +299,11 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam clone_url = f"{config.bb_ssh_url}/{project_key.lower()}/{repo_slug}.git" logger.info("Cloning %s -> %s", clone_url, bare_path) - _run_git(["clone", "--bare", clone_url, bare_path]) + clone_cmd = ["clone", "--bare", clone_url, bare_path] + if trim_since: + clone_cmd.insert(2, f"--shallow-since={trim_since}") + logger.info("Trimming history: keeping commits since %s", trim_since) + _run_git(clone_cmd) # 3. Clean hidden refs _clean_hidden_refs(bare_path) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index d224bd1..e579cf7 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -128,9 +128,13 @@ def _sync_repo(self, project_key, repo_slug): target_label = f"{gh_org}/{gh_repo_name}" if gh_org else "github" # Fetch from Bitbucket (origin) - _run_git(["fetch", "origin", "--prune", - "+refs/heads/*:refs/heads/*", - "+refs/tags/*:refs/tags/*"], cwd=bare_path) + fetch_cmd = ["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"] + trim_since = self.config.get_trim_since(project_key, repo_slug) + if trim_since: + fetch_cmd.extend(["--shallow-since", trim_since]) + _run_git(fetch_cmd, cwd=bare_path) # Compare Bitbucket's refs (post-fetch) against last sync snapshot try: diff --git a/config.yaml.example b/config.yaml.example index 9b61592..984ea33 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -78,6 +78,14 @@ repo_mapping: # - archived-spike # Projects not listed here use github.org and the global name_template +# Optional: trim history for specific large repos. +# Only commits within the retention period are migrated/synced. +# HEAD commit hashes are preserved (uses git shallow clones). +# Format: PROJECT/slug: "Ny" (years), "Nm" (months), or "Nd" (days) +# trim_history: +# UPSTREAM/linux: "2y" +# UPSTREAM/git: "1y" + # Optional: auto-convert large files to Git LFS before pushing to GitHub. # GitHub rejects files >100MB. This rewrites history to store them as LFS objects. lfs: diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 1c4449a..3b6b2fb 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -18,6 +18,7 @@ def mock_config(tmp_path): config.lfs_threshold = "100mb" config.migrate_delay = 0 config.sync_exclude_projects = set() + config.get_trim_since = MagicMock(return_value=None) return config From 9dbd23ff640ef8cbb770379bd998469b06633e61 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Apr 2026 11:47:44 +0000 Subject: [PATCH 47/74] Use --all --force instead of --mirror for trimmed shallow repos git push --mirror fails on shallow clones because the pack references parent objects at the shallow boundary that don't exist. Trimmed repos now push branches (--all --force) and tags separately. Tags referencing pruned history are skipped with a warning. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 13 ++++++++++++- bb2gh/syncer.py | 9 ++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 0a149f9..20d9c2e 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -338,7 +338,18 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam pass # Remote didn't exist _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) - _run_git(["push", "--mirror", "github"], cwd=bare_path) + + if trim_since: + # Shallow repos can't use --mirror (remote rejects missing parent objects). + # Push branches and tags separately. + _run_git(["push", "github", "--all", "--force"], cwd=bare_path) + try: + _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) + except subprocess.CalledProcessError: + logger.warning("Tag push failed for %s/%s (some tags may reference pruned history)", gh_org, gh_repo_name) + warnings.append("Some tags could not be pushed (reference pruned history)") + else: + _run_git(["push", "--mirror", "github"], cwd=bare_path) # Push LFS objects separately — only if LFS actually converted files if has_lfs: diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index e579cf7..9082028 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -173,7 +173,14 @@ def _sync_repo(self, project_key, repo_slug): logger.warning("LFS migration failed for %s/%s, skipping LFS", project_key, repo_slug) # Push to GitHub - _run_git(["push", "github", "--mirror"], cwd=bare_path) + if trim_since: + _run_git(["push", "github", "--all", "--force"], cwd=bare_path) + try: + _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) + except subprocess.CalledProcessError: + logger.warning("Tag push failed for %s/%s", project_key, repo_slug) + else: + _run_git(["push", "github", "--mirror"], cwd=bare_path) if has_lfs: try: From ef3638847842b758981e7797e3fb683a560f7607 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Apr 2026 12:32:44 +0000 Subject: [PATCH 48/74] Push trimmed repos branch-by-branch to avoid 2GB pack limit Pushing all branches at once creates a pack that exceeds GitHub's 2GB limit for large repos like Linux kernel forks. Now pushes each branch individually so each pack is small enough. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 30 +++++++++++++++++++++++------- bb2gh/syncer.py | 15 ++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 20d9c2e..b0bb153 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -341,13 +341,29 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam if trim_since: # Shallow repos can't use --mirror (remote rejects missing parent objects). - # Push branches and tags separately. - _run_git(["push", "github", "--all", "--force"], cwd=bare_path) - try: - _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) - except subprocess.CalledProcessError: - logger.warning("Tag push failed for %s/%s (some tags may reference pruned history)", gh_org, gh_repo_name) - warnings.append("Some tags could not be pushed (reference pruned history)") + # Push branches individually to keep pack sizes under GitHub's 2GB limit. + branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], + cwd=bare_path, quiet=True) + for branch in branches.strip().splitlines(): + branch = branch.strip() + if not branch: + continue + try: + _run_git(["push", "github", f"{branch}:{branch}", "--force"], cwd=bare_path) + except subprocess.CalledProcessError: + logger.warning("Failed to push branch %s for %s/%s", branch, project_key, repo_slug) + # Push tags individually too + tags = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/tags/"], + cwd=bare_path, quiet=True) + for tag in tags.strip().splitlines(): + tag = tag.strip() + if not tag: + continue + try: + _run_git(["push", "github", f"refs/tags/{tag}:refs/tags/{tag}", "--force"], + cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + pass # Tags referencing pruned history will fail silently else: _run_git(["push", "--mirror", "github"], cwd=bare_path) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 9082028..7bf4814 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -174,11 +174,16 @@ def _sync_repo(self, project_key, repo_slug): # Push to GitHub if trim_since: - _run_git(["push", "github", "--all", "--force"], cwd=bare_path) - try: - _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) - except subprocess.CalledProcessError: - logger.warning("Tag push failed for %s/%s", project_key, repo_slug) + branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], + cwd=bare_path, quiet=True) + for branch in branches.strip().splitlines(): + branch = branch.strip() + if not branch: + continue + try: + _run_git(["push", "github", f"{branch}:{branch}", "--force"], cwd=bare_path) + except subprocess.CalledProcessError: + logger.warning("Failed to push branch %s for %s/%s", branch, project_key, repo_slug) else: _run_git(["push", "github", "--mirror"], cwd=bare_path) From 7c9f7abc50dcbae7a14eea0f941a70eabc85687c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Apr 2026 13:35:45 +0000 Subject: [PATCH 49/74] Replace shallow clone approach with git replace --graft for history trimming Shallow clones can't be pushed to GitHub (missing parent objects). Now uses git replace --graft to make cutoff commits into true root commits, then filter-branch to rewrite history permanently. Pushes cleanly with --mirror. Commit hashes change but content is preserved. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 142 ++++++++++++++++++++++++++++------------- bb2gh/syncer.py | 32 +++------- tests/test_migrator.py | 1 + 3 files changed, 108 insertions(+), 67 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index b0bb153..04aafc9 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -90,6 +90,90 @@ def _has_large_blobs(bare_path, threshold): return len(line.strip()) > 0 +def _trim_history(bare_path, since_date): + """Trim repo history using git replace --graft + filter-branch. + + Finds the oldest commit after since_date on each branch, grafts it + as a root commit (no parents), then rewrites history permanently. + This creates a clean, pushable history without shallow boundaries. + Note: commit hashes will change. + """ + import shutil + import tempfile + + logger.info("Trimming history (keeping since %s) in %s", since_date, os.path.basename(bare_path)) + tmp_dir = tempfile.mkdtemp(suffix=".trim") + try: + work_path = os.path.join(tmp_dir, "work") + env_no_lfs = {"GIT_LFS_SKIP_SMUDGE": "1"} + cmd = ["git", "clone", bare_path, work_path] + subprocess.run(cmd, capture_output=True, text=True, check=True, + env={**os.environ, **env_no_lfs}) + + # Create local branches for all remotes + branches_output = _run_git(["branch", "-r"], cwd=work_path) + for line in branches_output.splitlines(): + branch = line.strip() + if "HEAD" in branch or not branch.startswith("origin/"): + continue + local_name = branch.replace("origin/", "", 1) + try: + _run_git(["branch", "--track", local_name, branch], + cwd=work_path, quiet=True) + except subprocess.CalledProcessError: + pass + + # Find graft points: oldest commit after cutoff per branch + graft_points = set() + local_branches = _run_git( + ["for-each-ref", "--format=%(refname)", "refs/heads/"], + cwd=work_path, quiet=True, + ) + for ref in local_branches.strip().splitlines(): + try: + commits = _run_git( + ["rev-list", f"--after={since_date}", "--reverse", ref], + cwd=work_path, quiet=True, + ) + except subprocess.CalledProcessError: + continue + lines = commits.strip().splitlines() + if lines: + graft_points.add(lines[0]) + + if not graft_points: + logger.info("No commits to trim in %s", os.path.basename(bare_path)) + return + + # Graft each point as a root commit + for sha in graft_points: + _run_git(["replace", "--graft", sha], cwd=work_path) + + # Rewrite history permanently + env_filter = {**os.environ, "FILTER_BRANCH_SQUELCH_WARNING": "1"} + subprocess.run( + ["git", "filter-branch", "--tag-name-filter", "cat", "--", "--all"], + cwd=work_path, capture_output=True, text=True, check=True, + env=env_filter, + ) + + # Fetch rewritten refs back into the bare repo + _run_git(["remote", "add", "trim-source", work_path], cwd=bare_path) + _run_git(["fetch", "trim-source", "--force", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) + _run_git(["remote", "remove", "trim-source"], cwd=bare_path) + + # Clean up old objects + _run_git(["reflog", "expire", "--expire=now", "--all"], cwd=bare_path, quiet=True) + _run_git(["gc", "--prune=now"], cwd=bare_path, quiet=True) + + logger.info("Trimmed history in %s: %d graft points", os.path.basename(bare_path), len(graft_points)) + + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + def _migrate_lfs(bare_path, threshold, timeout=None): """Convert files above threshold to Git LFS in all branches. @@ -286,12 +370,9 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam if os.path.exists(bare_path): # Already cloned, fetch latest logger.info("Bare clone exists, fetching latest: %s", bare_path) - fetch_cmd = ["fetch", "origin", "--prune", - "+refs/heads/*:refs/heads/*", - "+refs/tags/*:refs/tags/*"] - if trim_since: - fetch_cmd.extend(["--shallow-since", trim_since]) - _run_git(fetch_cmd, cwd=bare_path) + _run_git(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) else: clone_url = bb.get_repo_clone_url(repo, protocol="ssh") if not clone_url: @@ -299,11 +380,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam clone_url = f"{config.bb_ssh_url}/{project_key.lower()}/{repo_slug}.git" logger.info("Cloning %s -> %s", clone_url, bare_path) - clone_cmd = ["clone", "--bare", clone_url, bare_path] - if trim_since: - clone_cmd.insert(2, f"--shallow-since={trim_since}") - logger.info("Trimming history: keeping commits since %s", trim_since) - _run_git(clone_cmd) + _run_git(["clone", "--bare", clone_url, bare_path]) # 3. Clean hidden refs _clean_hidden_refs(bare_path) @@ -311,10 +388,13 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # Track migration details warnings = [] - # 4. Remap submodule URLs from Bitbucket to GitHub + # 4. Trim history if configured (must run before submodule remap) + if trim_since: + _trim_history(bare_path, trim_since) + + # 5. Remap submodule URLs from Bitbucket to GitHub submodules_remapped = remap_submodules_in_bare_repo(bare_path, config) has_submodules = submodules_remapped > 0 - # Check if repo has .gitmodules but remap returned 0 (skipped due to unresolvable URLs) try: _run_git(["show", "HEAD:.gitmodules"], cwd=bare_path, quiet=True) has_submodules = True @@ -323,49 +403,21 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam except subprocess.CalledProcessError: pass - # 5. Migrate large files to LFS if enabled + # 6. Migrate large files to LFS if enabled has_lfs = False if config.lfs_enabled: has_lfs = _migrate_lfs(bare_path, config.lfs_threshold) - # 6. Add GitHub remote and push + # 7. Add GitHub remote and push gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, ssh_url=config.gh_ssh_url or None) - # Remove existing github remote if present, then add try: _run_git(["remote", "remove", "github"], cwd=bare_path, quiet=True) except subprocess.CalledProcessError: - pass # Remote didn't exist + pass _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) - - if trim_since: - # Shallow repos can't use --mirror (remote rejects missing parent objects). - # Push branches individually to keep pack sizes under GitHub's 2GB limit. - branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], - cwd=bare_path, quiet=True) - for branch in branches.strip().splitlines(): - branch = branch.strip() - if not branch: - continue - try: - _run_git(["push", "github", f"{branch}:{branch}", "--force"], cwd=bare_path) - except subprocess.CalledProcessError: - logger.warning("Failed to push branch %s for %s/%s", branch, project_key, repo_slug) - # Push tags individually too - tags = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/tags/"], - cwd=bare_path, quiet=True) - for tag in tags.strip().splitlines(): - tag = tag.strip() - if not tag: - continue - try: - _run_git(["push", "github", f"refs/tags/{tag}:refs/tags/{tag}", "--force"], - cwd=bare_path, quiet=True) - except subprocess.CalledProcessError: - pass # Tags referencing pruned history will fail silently - else: - _run_git(["push", "--mirror", "github"], cwd=bare_path) + _run_git(["push", "--mirror", "github"], cwd=bare_path) # Push LFS objects separately — only if LFS actually converted files if has_lfs: diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 7bf4814..3b34041 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -6,7 +6,7 @@ import subprocess import time -from .migrator import _migrate_lfs, _has_large_blobs +from .migrator import _migrate_lfs, _has_large_blobs, _trim_history from .state import State from .submodules import remap_submodules_in_bare_repo @@ -128,13 +128,9 @@ def _sync_repo(self, project_key, repo_slug): target_label = f"{gh_org}/{gh_repo_name}" if gh_org else "github" # Fetch from Bitbucket (origin) - fetch_cmd = ["fetch", "origin", "--prune", - "+refs/heads/*:refs/heads/*", - "+refs/tags/*:refs/tags/*"] - trim_since = self.config.get_trim_since(project_key, repo_slug) - if trim_since: - fetch_cmd.extend(["--shallow-since", trim_since]) - _run_git(fetch_cmd, cwd=bare_path) + _run_git(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) # Compare Bitbucket's refs (post-fetch) against last sync snapshot try: @@ -158,11 +154,15 @@ def _sync_repo(self, project_key, repo_slug): # Clean hidden refs before pushing _clean_hidden_refs(bare_path) + # Trim history if configured + trim_since = self.config.get_trim_since(project_key, repo_slug) + if trim_since: + _trim_history(bare_path, trim_since) + # Remap submodule URLs from Bitbucket to GitHub remap_submodules_in_bare_repo(bare_path, self.config) # LFS: only run if repo actually has large blobs (fast pre-check) - # Timeout after 60s to avoid blocking the sync cycle has_lfs = False if self.config.lfs_enabled and _has_large_blobs(bare_path, self.config.lfs_threshold): try: @@ -173,19 +173,7 @@ def _sync_repo(self, project_key, repo_slug): logger.warning("LFS migration failed for %s/%s, skipping LFS", project_key, repo_slug) # Push to GitHub - if trim_since: - branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], - cwd=bare_path, quiet=True) - for branch in branches.strip().splitlines(): - branch = branch.strip() - if not branch: - continue - try: - _run_git(["push", "github", f"{branch}:{branch}", "--force"], cwd=bare_path) - except subprocess.CalledProcessError: - logger.warning("Failed to push branch %s for %s/%s", branch, project_key, repo_slug) - else: - _run_git(["push", "github", "--mirror"], cwd=bare_path) + _run_git(["push", "github", "--mirror"], cwd=bare_path) if has_lfs: try: diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 5d2f291..9a4a09d 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -34,6 +34,7 @@ def mock_config(tmp_path): config.gh_ssh_host = "" config.gh_ssh_url = "" config.migrate_delay = 0 + config.get_trim_since = MagicMock(return_value=None) return config From 93f961a9f64529f169eff3f2c62d4fbf4975c840 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Apr 2026 13:46:24 +0000 Subject: [PATCH 50/74] Add clarifying comment about trim_history changing commit hashes https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index 37e9e82..f38262d 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -49,7 +49,7 @@ def __init__(self, path="config.yaml"): self.lfs_enabled = lfs.get("enabled", False) self.lfs_threshold = lfs.get("threshold", "100mb") - # History trimming (repo-specific) + # History trimming (repo-specific) — rewrites history, changes commit hashes self.trim_history = raw.get("trim_history", {}) # Repository mapping (Bitbucket project/repo -> GitHub org/repo) From a7db377bbe5775b49cd27036d46288fd144b6568 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 11:48:31 +0000 Subject: [PATCH 51/74] Auto-fallback to branch-by-branch push when pack exceeds 2GB git push --mirror creates a single pack for all refs. For Linux kernel forks with 100+ branches this exceeds GitHub's 2GB limit. Now detects the error and automatically retries by pushing each branch and tag individually. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 04aafc9..4bb1699 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -348,6 +348,41 @@ def migrate_repos(config, only_repos=None): return total_migrated, total_skipped, total_failed +def _push_branch_by_branch(bare_path, project_key, repo_slug): + """Push branches and tags individually when --mirror pack exceeds 2GB.""" + branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], + cwd=bare_path, quiet=True) + pushed = 0 + failed = 0 + for branch in branches.strip().splitlines(): + branch = branch.strip() + if not branch: + continue + try: + _run_git(["push", "github", f"{branch}:{branch}", "--force"], cwd=bare_path) + pushed += 1 + except subprocess.CalledProcessError: + logger.warning("Failed to push branch %s for %s/%s", branch, project_key, repo_slug) + failed += 1 + + tags = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/tags/"], + cwd=bare_path, quiet=True) + for tag in tags.strip().splitlines(): + tag = tag.strip() + if not tag: + continue + try: + _run_git(["push", "github", f"refs/tags/{tag}:refs/tags/{tag}", "--force"], + cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + pass + + logger.info("Branch-by-branch push: %d pushed, %d failed for %s/%s", + pushed, failed, project_key, repo_slug) + if failed > 0 and pushed == 0: + raise RuntimeError(f"All branch pushes failed for {project_key}/{repo_slug}") + + def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_name, repo): """Migrate a single repository.""" # Resolve target GitHub org and repo name @@ -417,7 +452,15 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam pass _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) - _run_git(["push", "--mirror", "github"], cwd=bare_path) + + try: + _run_git(["push", "--mirror", "github"], cwd=bare_path) + except subprocess.CalledProcessError as e: + if "pack exceeds maximum allowed size" in (e.stderr or ""): + logger.warning("Pack too large for --mirror, pushing branch-by-branch for %s/%s", project_key, repo_slug) + _push_branch_by_branch(bare_path, project_key, repo_slug) + else: + raise # Push LFS objects separately — only if LFS actually converted files if has_lfs: From b315ccf63baba06d9eeadc0cf71d3683ad63d326 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 11:50:02 +0000 Subject: [PATCH 52/74] Add push_by_branch config to skip --mirror attempt for large repos Repos listed in push_by_branch go straight to branch-by-branch push, skipping the --mirror attempt that would fail with the 2GB pack limit. Saves time and bandwidth. Auto-fallback still works for unlisted repos. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 3 +++ bb2gh/migrator.py | 21 +++++++++++++-------- config.yaml.example | 6 ++++++ tests/test_migrator.py | 1 + 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index f38262d..7fc444d 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -52,6 +52,9 @@ def __init__(self, path="config.yaml"): # History trimming (repo-specific) — rewrites history, changes commit hashes self.trim_history = raw.get("trim_history", {}) + # Repos that need branch-by-branch push (too large for --mirror's 2GB pack limit) + self.push_by_branch = set(raw.get("push_by_branch", [])) + # Repository mapping (Bitbucket project/repo -> GitHub org/repo) rm = raw.get("repo_mapping", {}) self._repo_mapping = rm diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 4bb1699..b3aae0a 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -453,14 +453,19 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) - try: - _run_git(["push", "--mirror", "github"], cwd=bare_path) - except subprocess.CalledProcessError as e: - if "pack exceeds maximum allowed size" in (e.stderr or ""): - logger.warning("Pack too large for --mirror, pushing branch-by-branch for %s/%s", project_key, repo_slug) - _push_branch_by_branch(bare_path, project_key, repo_slug) - else: - raise + repo_key = f"{project_key}/{repo_slug}" + if repo_key in config.push_by_branch: + logger.info("Pushing branch-by-branch for %s (configured)", repo_key) + _push_branch_by_branch(bare_path, project_key, repo_slug) + else: + try: + _run_git(["push", "--mirror", "github"], cwd=bare_path) + except subprocess.CalledProcessError as e: + if "pack exceeds maximum allowed size" in (e.stderr or ""): + logger.warning("Pack too large for --mirror, pushing branch-by-branch for %s/%s", project_key, repo_slug) + _push_branch_by_branch(bare_path, project_key, repo_slug) + else: + raise # Push LFS objects separately — only if LFS actually converted files if has_lfs: diff --git a/config.yaml.example b/config.yaml.example index 984ea33..1b421b2 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -86,6 +86,12 @@ repo_mapping: # UPSTREAM/linux: "2y" # UPSTREAM/git: "1y" +# Optional: repos too large for --mirror push (>2GB pack). +# These are pushed branch-by-branch instead. +# push_by_branch: +# - UPSTREAM/linux +# - BGAN_UT_RM/bganut-linux + # Optional: auto-convert large files to Git LFS before pushing to GitHub. # GitHub rejects files >100MB. This rewrites history to store them as LFS objects. lfs: diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 9a4a09d..0888b53 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -35,6 +35,7 @@ def mock_config(tmp_path): config.gh_ssh_url = "" config.migrate_delay = 0 config.get_trim_since = MagicMock(return_value=None) + config.push_by_branch = set() return config From e02b9a9775e5747a91811d830d0d88b422d58898 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 11:56:29 +0000 Subject: [PATCH 53/74] Add delay between branch pushes in branch-by-branch mode Respects migrate_delay_seconds between each branch/tag push to avoid SSH rate limiting from GitHub. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index b3aae0a..f1ff5df 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -348,7 +348,7 @@ def migrate_repos(config, only_repos=None): return total_migrated, total_skipped, total_failed -def _push_branch_by_branch(bare_path, project_key, repo_slug): +def _push_branch_by_branch(bare_path, project_key, repo_slug, delay=2): """Push branches and tags individually when --mirror pack exceeds 2GB.""" branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], cwd=bare_path, quiet=True) @@ -364,6 +364,7 @@ def _push_branch_by_branch(bare_path, project_key, repo_slug): except subprocess.CalledProcessError: logger.warning("Failed to push branch %s for %s/%s", branch, project_key, repo_slug) failed += 1 + time.sleep(delay) tags = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/tags/"], cwd=bare_path, quiet=True) @@ -376,6 +377,7 @@ def _push_branch_by_branch(bare_path, project_key, repo_slug): cwd=bare_path, quiet=True) except subprocess.CalledProcessError: pass + time.sleep(delay) logger.info("Branch-by-branch push: %d pushed, %d failed for %s/%s", pushed, failed, project_key, repo_slug) @@ -456,14 +458,14 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam repo_key = f"{project_key}/{repo_slug}" if repo_key in config.push_by_branch: logger.info("Pushing branch-by-branch for %s (configured)", repo_key) - _push_branch_by_branch(bare_path, project_key, repo_slug) + _push_branch_by_branch(bare_path, project_key, repo_slug, delay=config.migrate_delay) else: try: _run_git(["push", "--mirror", "github"], cwd=bare_path) except subprocess.CalledProcessError as e: if "pack exceeds maximum allowed size" in (e.stderr or ""): logger.warning("Pack too large for --mirror, pushing branch-by-branch for %s/%s", project_key, repo_slug) - _push_branch_by_branch(bare_path, project_key, repo_slug) + _push_branch_by_branch(bare_path, project_key, repo_slug, delay=config.migrate_delay) else: raise From 17105a8e2ba23776d76752868a588bfde74740fc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 13:08:45 +0000 Subject: [PATCH 54/74] Add push_by_branch support to syncer Repos listed in push_by_branch now use branch-by-branch push in sync cycles too, with migrate_delay between each push. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 8 ++++++-- tests/test_syncer.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 3b34041..cc31d37 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -6,7 +6,7 @@ import subprocess import time -from .migrator import _migrate_lfs, _has_large_blobs, _trim_history +from .migrator import _migrate_lfs, _has_large_blobs, _trim_history, _push_branch_by_branch from .state import State from .submodules import remap_submodules_in_bare_repo @@ -173,7 +173,11 @@ def _sync_repo(self, project_key, repo_slug): logger.warning("LFS migration failed for %s/%s, skipping LFS", project_key, repo_slug) # Push to GitHub - _run_git(["push", "github", "--mirror"], cwd=bare_path) + repo_key = f"{project_key}/{repo_slug}" + if repo_key in self.config.push_by_branch: + _push_branch_by_branch(bare_path, project_key, repo_slug, delay=self.config.migrate_delay) + else: + _run_git(["push", "github", "--mirror"], cwd=bare_path) if has_lfs: try: diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 3b6b2fb..ccef373 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -19,6 +19,7 @@ def mock_config(tmp_path): config.migrate_delay = 0 config.sync_exclude_projects = set() config.get_trim_since = MagicMock(return_value=None) + config.push_by_branch = set() return config From b483b256591eaaef1c5721b92edb3472ee1500a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Apr 2026 18:22:50 +0000 Subject: [PATCH 55/74] Push all tags in one batch instead of individually with delays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tags are lightweight refs — pushing them all at once doesn't hit the 2GB pack limit. Falls back to individual push if batch fails. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index f1ff5df..d223e23 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -368,16 +368,19 @@ def _push_branch_by_branch(bare_path, project_key, repo_slug, delay=2): tags = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/tags/"], cwd=bare_path, quiet=True) - for tag in tags.strip().splitlines(): - tag = tag.strip() - if not tag: - continue + tag_list = [t.strip() for t in tags.strip().splitlines() if t.strip()] + if tag_list: try: - _run_git(["push", "github", f"refs/tags/{tag}:refs/tags/{tag}", "--force"], - cwd=bare_path, quiet=True) + # Push all tags at once — they're lightweight (no large packs) + _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) except subprocess.CalledProcessError: - pass - time.sleep(delay) + # Fallback: push individually if batch fails + for tag in tag_list: + try: + _run_git(["push", "github", f"refs/tags/{tag}:refs/tags/{tag}", "--force"], + cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + pass logger.info("Branch-by-branch push: %d pushed, %d failed for %s/%s", pushed, failed, project_key, repo_slug) From 2ae835dd9fd42fad2522f7375c08e3cfe0e6d9e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Apr 2026 08:53:13 +0000 Subject: [PATCH 56/74] Support per-repo github_org override in repo_mapping Allows individual repos to target a different GitHub org than their project default. Example: repos.thuraya-autotest.github_org: "other-org" https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index 7fc444d..8a7c17c 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -64,14 +64,15 @@ def __init__(self, path="config.yaml"): def resolve_target(self, project_key, repo_slug): """Resolve a Bitbucket project/repo to a GitHub org and repo name. - Lookup order: - 1. Explicit per-repo override in repo_mapping.projects..repos..github_name - 2. Per-project name_template override in repo_mapping.projects..name_template - 3. Global name_template from repo_mapping.name_template (default: "{slug}") + Lookup order for org: + 1. Per-repo github_org in repo_mapping.projects..repos..github_org + 2. Per-project github_org in repo_mapping.projects..github_org + 3. Global github.org - For the org: - 1. Per-project github_org in repo_mapping.projects..github_org - 2. Global github.org + Lookup order for repo name: + 1. Per-repo github_name in repo_mapping.projects..repos..github_name + 2. Per-project name_template + 3. Global name_template (default: "{slug}") Returns: (github_org, github_repo_name) tuple @@ -81,10 +82,11 @@ def resolve_target(self, project_key, repo_slug): # Resolve org gh_org = project_conf.get("github_org", self.gh_org) - # Resolve repo name: check explicit per-repo override first + # Resolve repo name and org: check explicit per-repo override first repos_conf = project_conf.get("repos", {}) if repo_slug in repos_conf: repo_conf = repos_conf[repo_slug] + gh_org = repo_conf.get("github_org", gh_org) gh_repo = repo_conf.get("github_name", repo_slug) else: # Use per-project template, falling back to global template From c48e941c64b52926454ba4cf78f39c04c4bd2a60 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Apr 2026 07:13:32 +0000 Subject: [PATCH 57/74] Add prepare-jenkins and jenkins-manifest commands for CI conversion New commands to prepare workspaces for Jenkins-to-GitHub-Actions conversion by AI agents: - prepare-jenkins: Creates sparse-checkout workspaces with only Jenkinsfiles and dependencies, creates migration branch on GitHub - jenkins-manifest: Generates YAML manifest listing repos with Jenkinsfiles, clone URLs, and file paths for remote AI agents Features: - Auto-discovers Jenkinsfiles (any case, any location) - Finds dependencies: vars/, src/, resources/, colocated .groovy - Parses load/readFile references in Jenkinsfiles - Per-repo manifest + global manifest for remote agent use - State tracking to avoid re-preparing repos - Dry-run support https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 49 +++++ bb2gh/github_client.py | 24 +++ bb2gh/jenkins_prep.py | 353 +++++++++++++++++++++++++++++++++++++ bb2gh/state.py | 21 +++ tests/test_jenkins_prep.py | 123 +++++++++++++ 5 files changed, 570 insertions(+) create mode 100644 bb2gh/jenkins_prep.py create mode 100644 tests/test_jenkins_prep.py diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 15efdfb..3992f64 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -381,5 +381,54 @@ def report(ctx, fmt, output_file): click.echo(output) +@cli.command("prepare-jenkins") +@click.option("--repo", multiple=True, help="Prepare specific repos only (PROJECT/SLUG).") +@click.option("--branch", default=None, help="Source branch (default: repo default branch).") +@click.option("--migration-branch", default="ci/github-actions-migration", + help="Branch name to create on GitHub for the migration.") +@click.option("--dry-run", is_flag=True, help="Show what would be done without making changes.") +@click.pass_context +def prepare_jenkins(ctx, repo, branch, migration_branch, dry_run): + """Prepare workspaces for Jenkins-to-GitHub-Actions conversion. + + Creates clean workspace directories containing only Jenkinsfiles + and their dependencies, with a new migration branch on GitHub. + """ + config = ctx.obj["config"] + from .jenkins_prep import prepare_jenkins_workspaces + only_repos = set(repo) if repo else None + prepared, skipped, failed = prepare_jenkins_workspaces( + config, only_repos=only_repos, branch=branch, + migration_branch_name=migration_branch, dry_run=dry_run, + ) + click.echo(f"\nJenkins prep: {prepared} prepared, {skipped} skipped, {failed} failed") + if failed > 0: + sys.exit(1) + + +@cli.command("jenkins-manifest") +@click.option("--repo", multiple=True, help="Include specific repos only (PROJECT/SLUG).") +@click.option("--branch", default=None, help="Source branch (default: repo default branch).") +@click.option("--migration-branch", default="ci/github-actions-migration", + help="Migration branch name to include in manifest.") +@click.option("--output", default="jenkins-manifest.yaml", help="Output manifest file path.") +@click.pass_context +def jenkins_manifest(ctx, repo, branch, migration_branch, output): + """Generate a manifest of repos with Jenkinsfiles for remote AI agents. + + Scans migrated repos for Jenkinsfiles and writes a YAML manifest + listing clone URLs, branch names, and file paths. Transfer this + file to the machine where the AI agent runs. + """ + config = ctx.obj["config"] + from .jenkins_prep import generate_manifest + only_repos = set(repo) if repo else None + found = generate_manifest( + config, only_repos=only_repos, branch=branch, + migration_branch_name=migration_branch, output_path=output, + ) + click.echo(f"\nManifest written to {output}: {found} repos with Jenkinsfiles") + + if __name__ == "__main__": cli() diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py index 60b74db..90bc5c5 100644 --- a/bb2gh/github_client.py +++ b/bb2gh/github_client.py @@ -102,6 +102,30 @@ def set_default_branch(self, repo_name, branch, org_name=None): repo.edit(default_branch=branch) logger.info("Set default branch for %s to %s", repo_name, branch) + def get_default_branch(self, repo_name, org_name=None): + """Get the default branch name for a repository.""" + repo = self.get_repo(repo_name, org_name) + return repo.default_branch + + def create_branch(self, repo_name, branch_name, from_branch=None, org_name=None): + """Create a branch on a GitHub repository. + + If from_branch is None, branches from the default branch. + Returns the branch name. If it already exists, returns it. + """ + repo = self.get_repo(repo_name, org_name) + source = from_branch or repo.default_branch + sha = repo.get_branch(source).commit.sha + try: + repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=sha) + logger.info("Created branch %s on %s from %s", branch_name, repo_name, source) + except GithubException as e: + if e.status == 422: + logger.info("Branch %s already exists on %s", branch_name, repo_name) + else: + raise + return branch_name + def get_clone_url(self, repo_name, org_name=None, ssh_url=None): """Get the clone URL for a repo. diff --git a/bb2gh/jenkins_prep.py b/bb2gh/jenkins_prep.py new file mode 100644 index 0000000..79ab0a9 --- /dev/null +++ b/bb2gh/jenkins_prep.py @@ -0,0 +1,353 @@ +"""Prepare workspaces for Jenkins-to-GitHub-Actions conversion.""" + +import json +import logging +import os +import re +import subprocess +from datetime import datetime, timezone + +import yaml + +from .github_client import GithubClient +from .state import State + +logger = logging.getLogger(__name__) + + +def _run_git(args, cwd=None, quiet=False): + cmd = ["git"] + args + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + if not quiet: + logger.error("git %s failed: %s", args[0], result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def find_jenkins_files(all_paths): + """Filter a list of file paths for Jenkins-related files. + + Returns (jenkins_files, dependency_files) tuple. + """ + jenkins_files = [] + deps = [] + jenkinsfile_dirs = set() + + for path in all_paths: + basename = os.path.basename(path).lower() + + if basename == "jenkinsfile" or basename.startswith("jenkinsfile.") or basename.endswith(".jenkinsfile"): + jenkins_files.append(path) + jenkinsfile_dirs.add(os.path.dirname(path)) + continue + + parts = path.split("/") + if parts[0] in ("vars", "resources"): + deps.append(path) + continue + if parts[0] == "src" and path.lower().endswith((".groovy", ".java")): + deps.append(path) + continue + + for path in all_paths: + if path in jenkins_files or path in deps: + continue + if path.lower().endswith(".groovy") and os.path.dirname(path) in jenkinsfile_dirs: + deps.append(path) + + return jenkins_files, deps + + +def parse_jenkinsfile_refs(content): + """Extract file references from Jenkinsfile content.""" + refs = set() + for pattern in [ + r"""load\s+['"]([^'"]+)['"]""", + r"""readFile\s*\(\s*['"]([^'"]+)['"]""", + r"""evaluate\s*\(\s*readFile\s*\(\s*['"]([^'"]+)['"]""", + ]: + for match in re.finditer(pattern, content): + refs.add(match.group(1)) + return refs + + +def _resolve_dependencies(bare_path, jenkins_files, ref="HEAD"): + """Read Jenkinsfiles from a repo and find referenced files.""" + extra_deps = set() + for jf in jenkins_files: + try: + content = _run_git(["show", f"{ref}:{jf}"], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + continue + refs = parse_jenkinsfile_refs(content) + extra_deps.update(refs) + return extra_deps + + +def _get_all_tree_paths(repo_path, ref="HEAD"): + """List all file paths in a git tree.""" + output = _run_git(["ls-tree", "-r", "--name-only", ref], cwd=repo_path) + return [p for p in output.splitlines() if p.strip()] + + +def create_workspace(config, state, gh, project_key, repo_slug, + source_branch, migration_branch, dry_run=False): + """Create a Jenkins workspace for a single repo.""" + gh_org, gh_repo_name = state.get_github_target(project_key, repo_slug) + if not gh_org: + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + + repo_key = f"{project_key}/{repo_slug}" + workspace_base = os.path.join(config.work_dir, "jenkins-workspaces") + workspace_path = os.path.join(workspace_base, f"{gh_org}__{gh_repo_name}") + + # Discover Jenkins files from the bare clone (fast, no network) + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + logger.warning("Bare clone not found for %s, skipping", repo_key) + return None + + # Determine source branch + if not source_branch: + try: + source_branch = gh.get_default_branch(gh_repo_name, org_name=gh_org) + except Exception: + source_branch = "master" + + # Get file tree from bare clone + try: + ref = f"refs/heads/{source_branch}" + all_paths = _get_all_tree_paths(bare_path, ref=ref) + except subprocess.CalledProcessError: + all_paths = _get_all_tree_paths(bare_path) + + jenkins_files, dep_files = find_jenkins_files(all_paths) + + if not jenkins_files: + logger.info("No Jenkinsfiles found in %s, skipping", repo_key) + return None + + # Resolve inline dependencies from Jenkinsfile content + extra_refs = _resolve_dependencies(bare_path, jenkins_files, ref=ref) + validated_extras = [p for p in extra_refs if p in all_paths] + all_required = sorted(set(jenkins_files + dep_files + validated_extras)) + + if dry_run: + logger.info("[DRY RUN] Would prepare %s: %d Jenkinsfiles, %d dependencies", + repo_key, len(jenkins_files), len(all_required) - len(jenkins_files)) + for f in jenkins_files: + logger.info("[DRY RUN] Jenkinsfile: %s", f) + return None + + # Create migration branch on GitHub + try: + gh.create_branch(gh_repo_name, migration_branch, + from_branch=source_branch, org_name=gh_org) + except Exception: + logger.warning("Could not create migration branch %s on %s/%s", + migration_branch, gh_org, gh_repo_name) + + # Create workspace via sparse checkout + os.makedirs(workspace_base, exist_ok=True) + if os.path.exists(workspace_path): + import shutil + shutil.rmtree(workspace_path) + + clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, + ssh_url=config.gh_ssh_url or None) + + env_no_lfs = {**os.environ, "GIT_LFS_SKIP_SMUDGE": "1"} + subprocess.run( + ["git", "clone", "--no-checkout", "--depth=1", + f"--branch={source_branch}", clone_url, workspace_path], + capture_output=True, text=True, check=True, env=env_no_lfs, + ) + + # Configure sparse checkout + _run_git(["sparse-checkout", "init", "--no-cone"], cwd=workspace_path) + sparse_file = os.path.join(workspace_path, ".git", "info", "sparse-checkout") + with open(sparse_file, "w") as f: + for path in all_required: + f.write(path + "\n") + + _run_git(["checkout"], cwd=workspace_path) + + # Create migration branch locally + _run_git(["checkout", "-b", migration_branch], cwd=workspace_path) + + # Create .github/workflows directory + workflows_dir = os.path.join(workspace_path, ".github", "workflows") + os.makedirs(workflows_dir, exist_ok=True) + + # Write metadata + meta = { + "source": { + "bitbucket_project": project_key, + "bitbucket_repo": repo_slug, + "github_org": gh_org, + "github_repo": gh_repo_name, + "source_branch": source_branch, + "migration_branch": migration_branch, + }, + "jenkins_files": jenkins_files, + "dependencies": [f for f in all_required if f not in jenkins_files], + "prepared_at": datetime.now(timezone.utc).isoformat(), + } + with open(os.path.join(workspace_path, ".bb2gh-jenkins-meta.json"), "w") as f: + json.dump(meta, f, indent=2) + + # Write per-repo manifest + manifest = { + "repo": { + "github_org": gh_org, + "github_repo": gh_repo_name, + "clone_url": clone_url, + "source_branch": source_branch, + "migration_branch": migration_branch, + }, + "jenkins_files": jenkins_files, + "dependencies": [f for f in all_required if f not in jenkins_files], + "all_files": all_required, + } + with open(os.path.join(workspace_path, ".bb2gh-jenkins-manifest.yaml"), "w") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) + + # Track in state + state.mark_jenkins_prepared( + project_key, repo_slug, migration_branch, + workspace_path, jenkins_files, + ) + + logger.info("Prepared workspace for %s: %d Jenkinsfiles, %d total files -> %s", + repo_key, len(jenkins_files), len(all_required), workspace_path) + return workspace_path + + +def prepare_jenkins_workspaces(config, only_repos=None, branch=None, + migration_branch_name="ci/github-actions-migration", + dry_run=False): + """Prepare Jenkins workspaces for migrated repos.""" + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + state = State(config.work_dir) + + repos = state.get_migrated_repos() + if not repos: + logger.warning("No migrated repos found.") + return 0, 0, 0 + + prepared = 0 + skipped = 0 + failed = 0 + + for project_key, repo_slug in repos: + repo_key = f"{project_key}/{repo_slug}" + if only_repos and repo_key not in only_repos: + continue + + if state.is_jenkins_prepared(project_key, repo_slug): + logger.info("Skipping already prepared: %s", repo_key) + skipped += 1 + continue + + try: + result = create_workspace( + config, state, gh, project_key, repo_slug, + branch, migration_branch_name, dry_run, + ) + if result: + prepared += 1 + else: + skipped += 1 + except Exception: + logger.exception("Failed to prepare %s", repo_key) + failed += 1 + + if not dry_run and prepared > 0: + logger.warning( + "WARNING: The sync loop (bb2gh sync) will delete the migration " + "branch '%s' on its next cycle. Pause sync or merge your changes " + "before the next sync.", migration_branch_name, + ) + + return prepared, skipped, failed + + +def generate_manifest(config, only_repos=None, branch=None, + migration_branch_name="ci/github-actions-migration", + output_path="jenkins-manifest.yaml"): + """Generate a manifest file listing all repos with Jenkins files.""" + state = State(config.work_dir) + repos = state.get_migrated_repos() + if not repos: + logger.warning("No migrated repos found.") + return 0 + + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + manifest_repos = [] + found = 0 + + for project_key, repo_slug in repos: + repo_key = f"{project_key}/{repo_slug}" + if only_repos and repo_key not in only_repos: + continue + + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + continue + + gh_org, gh_repo_name = state.get_github_target(project_key, repo_slug) + if not gh_org: + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + + # Determine source branch + src_branch = branch + if not src_branch: + try: + src_branch = gh.get_default_branch(gh_repo_name, org_name=gh_org) + except Exception: + src_branch = "master" + + try: + ref = f"refs/heads/{src_branch}" + all_paths = _get_all_tree_paths(bare_path, ref=ref) + except subprocess.CalledProcessError: + all_paths = _get_all_tree_paths(bare_path) + + jenkins_files, dep_files = find_jenkins_files(all_paths) + if not jenkins_files: + continue + + extra_refs = _resolve_dependencies(bare_path, jenkins_files, ref=ref) + validated_extras = [p for p in extra_refs if p in all_paths] + all_deps = sorted(set(dep_files + validated_extras)) + + clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, + ssh_url=config.gh_ssh_url or None) + + manifest_repos.append({ + "github_org": gh_org, + "github_repo": gh_repo_name, + "clone_url": clone_url, + "source_branch": src_branch, + "jenkins_files": jenkins_files, + "dependencies": all_deps, + }) + found += 1 + logger.info("Found %d Jenkinsfiles in %s", len(jenkins_files), repo_key) + + manifest = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "migration_branch": migration_branch_name, + "repos": manifest_repos, + } + + with open(output_path, "w") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) + + logger.info("Manifest written to %s: %d repos with Jenkinsfiles", output_path, found) + return found diff --git a/bb2gh/state.py b/bb2gh/state.py index e628a39..f62027d 100644 --- a/bb2gh/state.py +++ b/bb2gh/state.py @@ -138,3 +138,24 @@ def is_pr_migrated(self, project_key, repo_slug, bb_pr_id): key = f"{project_key}/{repo_slug}" repo_state = self._data["repos"].get(key, {}) return str(bb_pr_id) in repo_state.get("pr_mappings", {}) + + def mark_jenkins_prepared(self, project_key, repo_slug, migration_branch, + workspace_path, jenkins_files): + """Record that a repo's Jenkins workspace has been prepared.""" + key = f"{project_key}/{repo_slug}" + if key not in self._data["repos"]: + return + self._data["repos"][key]["jenkins_prep"] = { + "status": "prepared", + "prepared_at": self._now(), + "migration_branch": migration_branch, + "workspace_path": workspace_path, + "jenkins_files": jenkins_files, + } + self._save() + + def is_jenkins_prepared(self, project_key, repo_slug): + """Check if a repo's Jenkins workspace has been prepared.""" + key = f"{project_key}/{repo_slug}" + entry = self._data["repos"].get(key, {}) + return entry.get("jenkins_prep", {}).get("status") == "prepared" diff --git a/tests/test_jenkins_prep.py b/tests/test_jenkins_prep.py new file mode 100644 index 0000000..9a521c5 --- /dev/null +++ b/tests/test_jenkins_prep.py @@ -0,0 +1,123 @@ +"""Tests for Jenkins workspace preparation.""" + +from unittest.mock import MagicMock + +import pytest + +from bb2gh.jenkins_prep import find_jenkins_files, parse_jenkinsfile_refs + + +class TestFindJenkinsFiles: + def test_finds_jenkinsfile(self): + paths = ["README.md", "Jenkinsfile", "src/main.py"] + jf, deps = find_jenkins_files(paths) + assert jf == ["Jenkinsfile"] + assert deps == [] + + def test_finds_jenkinsfile_case_insensitive(self): + paths = ["jenkinsfile", "JENKINSFILE"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + + def test_finds_jenkinsfile_with_suffix(self): + paths = ["Jenkinsfile.deploy", "Jenkinsfile.staging", "README.md"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + assert "Jenkinsfile.deploy" in jf + assert "Jenkinsfile.staging" in jf + + def test_finds_dot_jenkinsfile(self): + paths = ["pipelines/build.jenkinsfile", "deploy.jenkinsfile"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + + def test_finds_nested_jenkinsfile(self): + paths = ["ci/Jenkinsfile", "ci/pipelines/Jenkinsfile.nightly"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + + def test_finds_vars_directory(self): + paths = ["Jenkinsfile", "vars/myHelper.groovy", "vars/deploy.groovy"] + jf, deps = find_jenkins_files(paths) + assert jf == ["Jenkinsfile"] + assert "vars/myHelper.groovy" in deps + assert "vars/deploy.groovy" in deps + + def test_finds_src_groovy(self): + paths = ["Jenkinsfile", "src/org/company/Pipeline.groovy"] + jf, deps = find_jenkins_files(paths) + assert "src/org/company/Pipeline.groovy" in deps + + def test_finds_resources(self): + paths = ["Jenkinsfile", "resources/config.yaml"] + jf, deps = find_jenkins_files(paths) + assert "resources/config.yaml" in deps + + def test_finds_colocated_groovy(self): + paths = ["ci/Jenkinsfile", "ci/helpers.groovy", "other/utils.groovy"] + jf, deps = find_jenkins_files(paths) + assert "ci/helpers.groovy" in deps + assert "other/utils.groovy" not in deps + + def test_no_jenkinsfiles(self): + paths = ["README.md", "src/main.py", "Makefile"] + jf, deps = find_jenkins_files(paths) + assert jf == [] + assert deps == [] + + def test_empty_paths(self): + jf, deps = find_jenkins_files([]) + assert jf == [] + assert deps == [] + + +class TestParseJenkinsfileRefs: + def test_extracts_load_single_quotes(self): + content = "load 'scripts/deploy.groovy'" + refs = parse_jenkinsfile_refs(content) + assert "scripts/deploy.groovy" in refs + + def test_extracts_load_double_quotes(self): + content = 'load "scripts/deploy.groovy"' + refs = parse_jenkinsfile_refs(content) + assert "scripts/deploy.groovy" in refs + + def test_extracts_readfile(self): + content = "def config = readFile('config/settings.yaml')" + refs = parse_jenkinsfile_refs(content) + assert "config/settings.yaml" in refs + + def test_extracts_evaluate_readfile(self): + content = """evaluate(readFile('scripts/helper.groovy'))""" + refs = parse_jenkinsfile_refs(content) + assert "scripts/helper.groovy" in refs + + def test_multiple_references(self): + content = """ + load 'scripts/build.groovy' + def cfg = readFile('config.yaml') + load "scripts/deploy.groovy" + """ + refs = parse_jenkinsfile_refs(content) + assert len(refs) == 3 + assert "scripts/build.groovy" in refs + assert "config.yaml" in refs + assert "scripts/deploy.groovy" in refs + + def test_no_references(self): + content = """ + pipeline { + agent any + stages { + stage('Build') { + steps { sh 'make build' } + } + } + } + """ + refs = parse_jenkinsfile_refs(content) + assert refs == set() + + def test_empty_content(self): + refs = parse_jenkinsfile_refs("") + assert refs == set() From 2df32893ae7ef3dd425472ecfabe710314560e48 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Apr 2026 07:56:46 +0000 Subject: [PATCH 58/74] Skip empty repos gracefully in prepare-jenkins Repos with no HEAD (empty/broken) now skip with a warning instead of crashing. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/jenkins_prep.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bb2gh/jenkins_prep.py b/bb2gh/jenkins_prep.py index 79ab0a9..cc7a4f2 100644 --- a/bb2gh/jenkins_prep.py +++ b/bb2gh/jenkins_prep.py @@ -124,7 +124,11 @@ def create_workspace(config, state, gh, project_key, repo_slug, ref = f"refs/heads/{source_branch}" all_paths = _get_all_tree_paths(bare_path, ref=ref) except subprocess.CalledProcessError: - all_paths = _get_all_tree_paths(bare_path) + try: + all_paths = _get_all_tree_paths(bare_path) + except subprocess.CalledProcessError: + logger.warning("No branches found in %s (empty repo?), skipping", repo_key) + return None jenkins_files, dep_files = find_jenkins_files(all_paths) From 1667d0ddff9da6bd0fbfc7160239c365b6fe4fc2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 13 May 2026 16:45:05 +0000 Subject: [PATCH 59/74] Add project aliases for old URLs and protected branches for sync 1. project_aliases config maps old Bitbucket project keys to current keys, so .gitmodules with outdated URLs get remapped correctly. 2. sync.protected_branches prevents sync from deleting specified branches on GitHub (e.g., ci/github-actions-migration). When set, sync uses --all --force instead of --mirror, then prunes only unprotected branches that don't exist in Bitbucket. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/config.py | 6 ++++++ bb2gh/submodules.py | 7 ++++++- bb2gh/syncer.py | 39 +++++++++++++++++++++++++++++++++++++++ config.yaml.example | 9 +++++++++ tests/test_submodules.py | 1 + tests/test_syncer.py | 1 + 6 files changed, 62 insertions(+), 1 deletion(-) diff --git a/bb2gh/config.py b/bb2gh/config.py index 8a7c17c..c4974b0 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -40,6 +40,7 @@ def __init__(self, path="config.yaml"): self.sync_exclude_projects = set( p.upper() for p in sync.get("exclude_projects", []) ) + self.sync_protected_branches = sync.get("protected_branches", []) # User mapping (Bitbucket username -> GitHub username) self.user_mapping = raw.get("user_mapping", {}) @@ -55,6 +56,11 @@ def __init__(self, path="config.yaml"): # Repos that need branch-by-branch push (too large for --mirror's 2GB pack limit) self.push_by_branch = set(raw.get("push_by_branch", [])) + # Project key aliases (old_key -> current_key) for .gitmodules remapping + self.project_aliases = {} + for old, new in raw.get("project_aliases", {}).items(): + self.project_aliases[old.upper()] = new.upper() + # Repository mapping (Bitbucket project/repo -> GitHub org/repo) rm = raw.get("repo_mapping", {}) self._repo_mapping = rm diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 01a0432..b28328f 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -124,7 +124,12 @@ def remap_submodule_urls(content, config): project_key_raw, slug = parsed resolved = None - for pk in [project_key_raw.upper(), project_key_raw]: + # Try the raw key and its uppercase, plus any alias + candidates = [project_key_raw.upper(), project_key_raw] + alias = config.project_aliases.get(project_key_raw.upper()) + if alias: + candidates.insert(0, alias) + for pk in candidates: if config.bb_projects and pk not in config.bb_projects: continue if not config.should_migrate_repo(pk, slug): diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index cc31d37..6479b9a 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -113,6 +113,39 @@ def _sync_all(self): synced, skipped, failed, ) + def _prune_unprotected_branches(self, bare_path, project_key, repo_slug): + """Delete remote branches on GitHub that don't exist locally, except protected ones.""" + protected = set(self.config.sync_protected_branches) + + # Get local branches (from Bitbucket) + local_output = _run_git( + ["for-each-ref", "--format=%(refname:short)", "refs/heads/"], + cwd=bare_path, quiet=True, + ) + local_branches = set(l.strip() for l in local_output.splitlines() if l.strip()) + + # Get remote branches on GitHub + try: + remote_output = _run_git( + ["ls-remote", "--heads", "github"], + cwd=bare_path, quiet=True, + ) + except subprocess.CalledProcessError: + return + + for line in remote_output.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + ref = parts[1].replace("refs/heads/", "") + if ref in local_branches or ref in protected: + continue + try: + _run_git(["push", "github", "--delete", ref], cwd=bare_path, quiet=True) + logger.debug("Deleted remote branch %s (not in source, not protected)", ref) + except subprocess.CalledProcessError: + pass + def _sync_repo(self, project_key, repo_slug): """Sync a single repo: fetch from Bitbucket, push to GitHub only if changed.""" bare_path = os.path.join( @@ -176,6 +209,12 @@ def _sync_repo(self, project_key, repo_slug): repo_key = f"{project_key}/{repo_slug}" if repo_key in self.config.push_by_branch: _push_branch_by_branch(bare_path, project_key, repo_slug, delay=self.config.migrate_delay) + elif self.config.sync_protected_branches: + # Can't use --mirror (it deletes branches not in source). + # Push all branches + tags, then prune only unprotected branches. + _run_git(["push", "github", "--all", "--force"], cwd=bare_path) + _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) + self._prune_unprotected_branches(bare_path, project_key, repo_slug) else: _run_git(["push", "github", "--mirror"], cwd=bare_path) diff --git a/config.yaml.example b/config.yaml.example index 1b421b2..0d5f151 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -40,6 +40,10 @@ sync: migrate_delay_seconds: 2 # Max time (seconds) for LFS migration during sync (default: 60) sync_lfs_timeout_seconds: 60 + # Branches on GitHub to protect from sync deletion (e.g., CI migration branches) + # When set, sync uses --all --force instead of --mirror to preserve these branches + # protected_branches: + # - ci/github-actions-migration # Projects to exclude from continuous sync (still migrated, just not synced) # exclude_projects: # - UPSTREAM @@ -92,6 +96,11 @@ repo_mapping: # - UPSTREAM/linux # - BGAN_UT_RM/bganut-linux +# Optional: map old Bitbucket project keys to current keys. +# Used for .gitmodules remapping when URLs reference renamed projects. +# project_aliases: +# OLD_PROJECT_KEY: CURRENT_PROJECT_KEY + # Optional: auto-convert large files to Git LFS before pushing to GitHub. # GitHub rejects files >100MB. This rewrites history to store them as LFS objects. lfs: diff --git a/tests/test_submodules.py b/tests/test_submodules.py index e75b0f8..bb4ab59 100644 --- a/tests/test_submodules.py +++ b/tests/test_submodules.py @@ -19,6 +19,7 @@ def mock_config(): config.bb_verify_ssl = True config.gh_ssh_host = "gatehousesatcom.ghe.com" config.gh_ssh_url = "ssh://gatehousesatcom@gatehousesatcom.ghe.com" + config.project_aliases = {} config.should_migrate_repo = MagicMock(return_value=True) config.resolve_target = MagicMock( side_effect=lambda proj, slug: ("networks-ngsp", slug) diff --git a/tests/test_syncer.py b/tests/test_syncer.py index ccef373..a84d2d1 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -20,6 +20,7 @@ def mock_config(tmp_path): config.sync_exclude_projects = set() config.get_trim_since = MagicMock(return_value=None) config.push_by_branch = set() + config.sync_protected_branches = [] return config From a78241cc96a91343688095a1bc899f4b36cf2649 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 13 May 2026 16:52:35 +0000 Subject: [PATCH 60/74] Auto-resolve old Bitbucket project keys and add sync protected branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Auto-resolve renamed project keys via Bitbucket API — when a .gitmodules URL uses an old project key, the tool queries GET /rest/api/1.0/projects/{old_key} to get the current key. Results are cached per remap cycle. Manual project_aliases config still works as a fallback. 2. sync.protected_branches config prevents sync from deleting specified branches on GitHub (e.g., ci/github-actions-migration). When set, sync uses --all --force + selective pruning instead of --mirror. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/bitbucket_client.py | 14 ++++++++++++++ bb2gh/migrator.py | 4 +++- bb2gh/submodules.py | 29 ++++++++++++++++++++++------- bb2gh/syncer.py | 6 +++++- tests/test_syncer.py | 18 +++++++++++++----- 5 files changed, 57 insertions(+), 14 deletions(-) diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py index 4535368..1a02785 100644 --- a/bb2gh/bitbucket_client.py +++ b/bb2gh/bitbucket_client.py @@ -41,6 +41,20 @@ def list_projects(self): url = f"{self.api_url}/projects" return list(self._paginate(url)) + def resolve_project_key(self, project_key): + """Resolve a project key, following Bitbucket aliases for renamed projects. + + Returns the current/canonical project key, or None if not found. + """ + url = f"{self.api_url}/projects/{project_key}" + try: + resp = self.session.get(url) + if resp.status_code == 200: + return resp.json().get("key") + except Exception: + pass + return None + def list_repos(self, project_key): """List all repositories in a project.""" url = f"{self.api_url}/projects/{project_key}/repos" diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index d223e23..20e9390 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -433,7 +433,9 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam _trim_history(bare_path, trim_since) # 5. Remap submodule URLs from Bitbucket to GitHub - submodules_remapped = remap_submodules_in_bare_repo(bare_path, config) + submodules_remapped = remap_submodules_in_bare_repo( + bare_path, config, alias_resolver=bb.resolve_project_key, + ) has_submodules = submodules_remapped > 0 try: _run_git(["show", "HEAD:.gitmodules"], cwd=bare_path, quiet=True) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index b28328f..16cb837 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -92,17 +92,22 @@ def _is_already_github(url, gh_ssh_url, gh_ssh_host, gh_https_base): return False -def remap_submodule_urls(content, config): +def remap_submodule_urls(content, config, alias_resolver=None): """Replace Bitbucket submodule URLs in .gitmodules with GitHub URLs. If any Bitbucket URL cannot be resolved (project not migrated, repo excluded), the entire .gitmodules is left unchanged to avoid a mix of old and new URLs. + + Args: + alias_resolver: Optional callable(project_key) -> canonical_key. + Used to auto-resolve renamed Bitbucket projects. """ bb_hostnames = _build_bb_hostnames(config) gh_ssh_url = config.gh_ssh_url gh_ssh_host = config.gh_ssh_host gh_https_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") + alias_cache = dict(config.project_aliases) urls = _extract_submodule_urls(content) if not urls: @@ -124,9 +129,9 @@ def remap_submodule_urls(content, config): project_key_raw, slug = parsed resolved = None - # Try the raw key and its uppercase, plus any alias + # Try the raw key, its uppercase, cached alias, and auto-resolved alias candidates = [project_key_raw.upper(), project_key_raw] - alias = config.project_aliases.get(project_key_raw.upper()) + alias = alias_cache.get(project_key_raw.upper()) if alias: candidates.insert(0, alias) for pk in candidates: @@ -137,6 +142,16 @@ def remap_submodule_urls(content, config): resolved = config.resolve_target(pk, slug) break + # Auto-resolve via Bitbucket API if still unresolved + if not resolved and alias_resolver and project_key_raw.upper() not in alias_cache: + real_key = alias_resolver(project_key_raw) + if real_key: + alias_cache[project_key_raw.upper()] = real_key.upper() + pk = real_key.upper() + if not config.bb_projects or pk in config.bb_projects: + if config.should_migrate_repo(pk, slug): + resolved = config.resolve_target(pk, slug) + if not resolved: logger.warning( "Cannot remap submodule URL %s — project %s/%s not in migration scope. " @@ -166,7 +181,7 @@ def remap_submodule_urls(content, config): return new_content -def remap_submodules_in_bare_repo(bare_repo_path, config): +def remap_submodules_in_bare_repo(bare_repo_path, config, alias_resolver=None): """Rewrite .gitmodules in all branches of a bare repo. Uses git plumbing to create deterministic commits (fixed timestamp) @@ -188,7 +203,7 @@ def remap_submodules_in_bare_repo(bare_repo_path, config): remapped = 0 for ref in output.strip().splitlines(): - if _remap_branch(bare_repo_path, ref, config): + if _remap_branch(bare_repo_path, ref, config, alias_resolver): remapped += 1 if remapped: @@ -200,14 +215,14 @@ def remap_submodules_in_bare_repo(bare_repo_path, config): return remapped -def _remap_branch(bare_repo_path, ref, config): +def _remap_branch(bare_repo_path, ref, config, alias_resolver=None): """Remap .gitmodules on a single branch ref. Returns True if changed.""" try: content = _git(["show", f"{ref}:.gitmodules"], cwd=bare_repo_path) except subprocess.CalledProcessError: return False - new_content = remap_submodule_urls(content, config) + new_content = remap_submodule_urls(content, config, alias_resolver=alias_resolver) if new_content == content: return False diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 6479b9a..9f91131 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -6,6 +6,7 @@ import subprocess import time +from .bitbucket_client import BitbucketClient from .migrator import _migrate_lfs, _has_large_blobs, _trim_history, _push_branch_by_branch from .state import State from .submodules import remap_submodules_in_bare_repo @@ -54,6 +55,8 @@ class Syncer: def __init__(self, config): self.config = config self.state = State(config.work_dir) + self._bb = BitbucketClient(config.bb_base_url, config.bb_token, + verify_ssl=config.bb_verify_ssl) self._running = True signal.signal(signal.SIGTERM, self._handle_signal) @@ -193,7 +196,8 @@ def _sync_repo(self, project_key, repo_slug): _trim_history(bare_path, trim_since) # Remap submodule URLs from Bitbucket to GitHub - remap_submodules_in_bare_repo(bare_path, self.config) + remap_submodules_in_bare_repo(bare_path, self.config, + alias_resolver=self._bb.resolve_project_key) # LFS: only run if repo actually has large blobs (fast pre-check) has_lfs = False diff --git a/tests/test_syncer.py b/tests/test_syncer.py index a84d2d1..3be610f 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -21,13 +21,17 @@ def mock_config(tmp_path): config.get_trim_since = MagicMock(return_value=None) config.push_by_branch = set() config.sync_protected_branches = [] + config.bb_base_url = "https://bitbucket.example.com" + config.bb_token = "fake" + config.bb_verify_ssl = True return config class TestSyncer: + @patch("bb2gh.syncer.BitbucketClient") @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") - def test_sync_repo_with_changes(self, mock_git, MockState, mock_config, tmp_path): + def test_sync_repo_with_changes(self, mock_git, MockState, MockBB, mock_config, tmp_path): """Test syncing a repo when changes are detected (no prior snapshot).""" bare_path = tmp_path / "PROJ__my-repo.git" bare_path.mkdir() @@ -53,9 +57,10 @@ def side_effect(args, cwd=None, quiet=False): mock_git.assert_any_call(["push", "github", "--mirror"], cwd=str(bare_path)) state_instance.update_sync_time.assert_called_once_with("PROJ", "my-repo") + @patch("bb2gh.syncer.BitbucketClient") @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") - def test_sync_skips_when_no_changes(self, mock_git, MockState, mock_config, tmp_path): + def test_sync_skips_when_no_changes(self, mock_git, MockState, MockBB, mock_config, tmp_path): """Test that sync skips push when BB refs match stored snapshot.""" bare_path = tmp_path / "PROJ__my-repo.git" bare_path.mkdir() @@ -83,8 +88,9 @@ def side_effect(args, cwd=None, quiet=False): assert c[0][0] != ["push", "github", "--mirror"] state_instance.update_sync_time.assert_not_called() + @patch("bb2gh.syncer.BitbucketClient") @patch("bb2gh.syncer.State") - def test_no_migrated_repos(self, MockState, mock_config): + def test_no_migrated_repos(self, MockState, MockBB, mock_config): """Test sync when no repos are migrated yet.""" state_instance = MockState.return_value state_instance.get_migrated_repos.return_value = [] @@ -92,9 +98,10 @@ def test_no_migrated_repos(self, MockState, mock_config): syncer = Syncer(mock_config) syncer._sync_all() # Should not raise + @patch("bb2gh.syncer.BitbucketClient") @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") - def test_sync_handles_failure(self, mock_git, MockState, mock_config, tmp_path): + def test_sync_handles_failure(self, mock_git, MockState, MockBB, mock_config, tmp_path): """Test that sync continues if one repo fails.""" bare1 = tmp_path / "PROJ__repo1.git" bare1.mkdir() @@ -123,9 +130,10 @@ def side_effect(args, cwd=None, quiet=False): # repo1 failed on fetch, repo2 skipped (no changes) # Neither should have update_sync_time called + @patch("bb2gh.syncer.BitbucketClient") @patch("bb2gh.syncer.State") @patch("bb2gh.syncer._run_git") - def test_sync_logs_github_target(self, mock_git, MockState, mock_config, tmp_path): + def test_sync_logs_github_target(self, mock_git, MockState, MockBB, mock_config, tmp_path): """Test that sync uses the stored GitHub target for logging.""" bare_path = tmp_path / "INFRA__my-service.git" bare_path.mkdir() From fa6064d2f090badcee85c8d22ebaab37a0154b2e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 13 May 2026 17:06:24 +0000 Subject: [PATCH 61/74] Support glob patterns in sync.protected_branches Uses fnmatch for pattern matching, so ci/* matches ci/github-actions, ci/migration, etc. Exact names still work (e.g., main). https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 9f91131..e51ff2e 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -116,10 +116,13 @@ def _sync_all(self): synced, skipped, failed, ) + def _is_branch_protected(self, branch_name): + """Check if a branch matches any protected pattern (supports * glob).""" + from fnmatch import fnmatch + return any(fnmatch(branch_name, p) for p in self.config.sync_protected_branches) + def _prune_unprotected_branches(self, bare_path, project_key, repo_slug): """Delete remote branches on GitHub that don't exist locally, except protected ones.""" - protected = set(self.config.sync_protected_branches) - # Get local branches (from Bitbucket) local_output = _run_git( ["for-each-ref", "--format=%(refname:short)", "refs/heads/"], @@ -141,7 +144,7 @@ def _prune_unprotected_branches(self, bare_path, project_key, repo_slug): if len(parts) < 2: continue ref = parts[1].replace("refs/heads/", "") - if ref in local_branches or ref in protected: + if ref in local_branches or self._is_branch_protected(ref): continue try: _run_git(["push", "github", "--delete", ref], cwd=bare_path, quiet=True) From ed8efc7989742d5f2a22952f234c9a8a72e9224c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 13 May 2026 19:49:54 +0000 Subject: [PATCH 62/74] Fix alias resolver to handle repos moved between projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous resolver only checked project renames. Now queries GET /projects/{key}/repos/{slug} which returns the repo's CURRENT project key — handling repos that were moved from one project to another (e.g., 3rdparty_LIBCodegenix moved from SYS_BGANRAN to SYS_COM). Results are cached per remap cycle. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/bitbucket_client.py | 19 +++++++++++++++++++ bb2gh/migrator.py | 2 +- bb2gh/submodules.py | 34 ++++++++++++++++++++++------------ bb2gh/syncer.py | 2 +- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py index 1a02785..e8e9987 100644 --- a/bb2gh/bitbucket_client.py +++ b/bb2gh/bitbucket_client.py @@ -55,6 +55,25 @@ def resolve_project_key(self, project_key): pass return None + def resolve_repo_location(self, project_key, repo_slug): + """Resolve a repo's current project and slug, following moves/aliases. + + When a repo is moved from one project to another, Bitbucket keeps + the old URL alive. This method returns the current (project_key, slug). + """ + url = f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" + try: + resp = self.session.get(url) + if resp.status_code == 200: + data = resp.json() + real_project = data.get("project", {}).get("key") + real_slug = data.get("slug") + if real_project: + return real_project, real_slug or repo_slug + except Exception: + pass + return None, None + def list_repos(self, project_key): """List all repositories in a project.""" url = f"{self.api_url}/projects/{project_key}/repos" diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index 20e9390..efe38e2 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -434,7 +434,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # 5. Remap submodule URLs from Bitbucket to GitHub submodules_remapped = remap_submodules_in_bare_repo( - bare_path, config, alias_resolver=bb.resolve_project_key, + bare_path, config, alias_resolver=bb.resolve_repo_location, ) has_submodules = submodules_remapped > 0 try: diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 16cb837..9247b02 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -107,7 +107,10 @@ def remap_submodule_urls(content, config, alias_resolver=None): gh_ssh_url = config.gh_ssh_url gh_ssh_host = config.gh_ssh_host gh_https_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") - alias_cache = dict(config.project_aliases) + alias_cache = {} + # Pre-populate from manual project_aliases config + # These are project-level overrides, stored as PROJECT/slug -> (resolved_project, slug) + project_aliases = getattr(config, "project_aliases", {}) urls = _extract_submodule_urls(content) if not urls: @@ -129,11 +132,11 @@ def remap_submodule_urls(content, config, alias_resolver=None): project_key_raw, slug = parsed resolved = None - # Try the raw key, its uppercase, cached alias, and auto-resolved alias + # Try the raw key, its uppercase, and manual project alias candidates = [project_key_raw.upper(), project_key_raw] - alias = alias_cache.get(project_key_raw.upper()) - if alias: - candidates.insert(0, alias) + manual_alias = project_aliases.get(project_key_raw.upper()) + if manual_alias: + candidates.insert(0, manual_alias) for pk in candidates: if config.bb_projects and pk not in config.bb_projects: continue @@ -143,14 +146,21 @@ def remap_submodule_urls(content, config, alias_resolver=None): break # Auto-resolve via Bitbucket API if still unresolved - if not resolved and alias_resolver and project_key_raw.upper() not in alias_cache: - real_key = alias_resolver(project_key_raw) - if real_key: - alias_cache[project_key_raw.upper()] = real_key.upper() - pk = real_key.upper() + # Handles both renamed projects AND repos moved between projects + if not resolved and alias_resolver: + cache_key = f"{project_key_raw.upper()}/{slug}" + if cache_key not in alias_cache: + real_proj, real_slug = alias_resolver(project_key_raw, slug) + if real_proj: + alias_cache[cache_key] = (real_proj.upper(), real_slug or slug) + else: + alias_cache[cache_key] = None + cached = alias_cache.get(cache_key) + if cached: + pk, resolved_slug = cached if not config.bb_projects or pk in config.bb_projects: - if config.should_migrate_repo(pk, slug): - resolved = config.resolve_target(pk, slug) + if config.should_migrate_repo(pk, resolved_slug): + resolved = config.resolve_target(pk, resolved_slug) if not resolved: logger.warning( diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index e51ff2e..03fba88 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -200,7 +200,7 @@ def _sync_repo(self, project_key, repo_slug): # Remap submodule URLs from Bitbucket to GitHub remap_submodules_in_bare_repo(bare_path, self.config, - alias_resolver=self._bb.resolve_project_key) + alias_resolver=self._bb.resolve_repo_location) # LFS: only run if repo actually has large blobs (fast pre-check) has_lfs = False From 280ba1f4fbb48de7be606d914ff1700384446bb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 13 May 2026 19:54:59 +0000 Subject: [PATCH 63/74] Resolve repo location via API BEFORE config lookup Previously the API resolver was a fallback after config lookup succeeded with the old project key. Now resolves the actual repo location first, so moved repos (e.g., SYS_BGANRAN -> SYS_COM) get the correct GitHub org mapping. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/submodules.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 9247b02..6a3c0e4 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -132,35 +132,39 @@ def remap_submodule_urls(content, config, alias_resolver=None): project_key_raw, slug = parsed resolved = None - # Try the raw key, its uppercase, and manual project alias - candidates = [project_key_raw.upper(), project_key_raw] - manual_alias = project_aliases.get(project_key_raw.upper()) + + # Resolve the actual repo location (handles moved repos + renamed projects) + actual_pk = project_key_raw.upper() + actual_slug = slug + + # Check manual alias first + manual_alias = project_aliases.get(actual_pk) if manual_alias: - candidates.insert(0, manual_alias) - for pk in candidates: - if config.bb_projects and pk not in config.bb_projects: - continue - if not config.should_migrate_repo(pk, slug): - continue - resolved = config.resolve_target(pk, slug) - break + actual_pk = manual_alias - # Auto-resolve via Bitbucket API if still unresolved - # Handles both renamed projects AND repos moved between projects - if not resolved and alias_resolver: + # Auto-resolve via Bitbucket API (cached) + if alias_resolver: cache_key = f"{project_key_raw.upper()}/{slug}" if cache_key not in alias_cache: real_proj, real_slug = alias_resolver(project_key_raw, slug) - if real_proj: + if real_proj and (real_proj.upper() != project_key_raw.upper() or real_slug != slug): alias_cache[cache_key] = (real_proj.upper(), real_slug or slug) + logger.debug("Resolved %s/%s -> %s/%s via API", + project_key_raw, slug, real_proj, real_slug or slug) else: alias_cache[cache_key] = None cached = alias_cache.get(cache_key) if cached: - pk, resolved_slug = cached - if not config.bb_projects or pk in config.bb_projects: - if config.should_migrate_repo(pk, resolved_slug): - resolved = config.resolve_target(pk, resolved_slug) + actual_pk, actual_slug = cached + + # Now resolve using the actual (possibly redirected) project/slug + for pk in [actual_pk, project_key_raw.upper(), project_key_raw]: + if config.bb_projects and pk not in config.bb_projects: + continue + if not config.should_migrate_repo(pk, actual_slug): + continue + resolved = config.resolve_target(pk, actual_slug) + break if not resolved: logger.warning( From b0542b4322bd5340034f4a84ed3a2c37493e0835 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 13 May 2026 19:57:56 +0000 Subject: [PATCH 64/74] Share API alias cache across all branches in a repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the cache was recreated per branch, causing repeated API calls for the same project/slug combos. Now a single cache is shared across all branches within a repo remap — so if a repo has 700 branches with the same submodules, each unique combo is resolved once. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/submodules.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 6a3c0e4..7a864a5 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -92,7 +92,7 @@ def _is_already_github(url, gh_ssh_url, gh_ssh_host, gh_https_base): return False -def remap_submodule_urls(content, config, alias_resolver=None): +def remap_submodule_urls(content, config, alias_resolver=None, _alias_cache=None): """Replace Bitbucket submodule URLs in .gitmodules with GitHub URLs. If any Bitbucket URL cannot be resolved (project not migrated, repo @@ -100,16 +100,14 @@ def remap_submodule_urls(content, config, alias_resolver=None): of old and new URLs. Args: - alias_resolver: Optional callable(project_key) -> canonical_key. - Used to auto-resolve renamed Bitbucket projects. + alias_resolver: Optional callable(project_key, slug) -> (real_project, real_slug). + _alias_cache: Shared cache dict for API results across branches. """ bb_hostnames = _build_bb_hostnames(config) gh_ssh_url = config.gh_ssh_url gh_ssh_host = config.gh_ssh_host gh_https_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") - alias_cache = {} - # Pre-populate from manual project_aliases config - # These are project-level overrides, stored as PROJECT/slug -> (resolved_project, slug) + alias_cache = _alias_cache if _alias_cache is not None else {} project_aliases = getattr(config, "project_aliases", {}) urls = _extract_submodule_urls(content) @@ -142,7 +140,7 @@ def remap_submodule_urls(content, config, alias_resolver=None): if manual_alias: actual_pk = manual_alias - # Auto-resolve via Bitbucket API (cached) + # Auto-resolve via Bitbucket API (cached per project/slug combo) if alias_resolver: cache_key = f"{project_key_raw.upper()}/{slug}" if cache_key not in alias_cache: @@ -215,9 +213,12 @@ def remap_submodules_in_bare_repo(bare_repo_path, config, alias_resolver=None): if not output.strip(): return 0 + # Shared cache for API results across all branches in this repo + alias_cache = {} + remapped = 0 for ref in output.strip().splitlines(): - if _remap_branch(bare_repo_path, ref, config, alias_resolver): + if _remap_branch(bare_repo_path, ref, config, alias_resolver, alias_cache): remapped += 1 if remapped: @@ -229,14 +230,15 @@ def remap_submodules_in_bare_repo(bare_repo_path, config, alias_resolver=None): return remapped -def _remap_branch(bare_repo_path, ref, config, alias_resolver=None): +def _remap_branch(bare_repo_path, ref, config, alias_resolver=None, alias_cache=None): """Remap .gitmodules on a single branch ref. Returns True if changed.""" try: content = _git(["show", f"{ref}:.gitmodules"], cwd=bare_repo_path) except subprocess.CalledProcessError: return False - new_content = remap_submodule_urls(content, config, alias_resolver=alias_resolver) + new_content = remap_submodule_urls(content, config, alias_resolver=alias_resolver, + _alias_cache=alias_cache) if new_content == content: return False From 3cecdb411dfcf96ed6e9bc9aed71e3f9c724c883 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 13 May 2026 20:04:24 +0000 Subject: [PATCH 65/74] Use current date for submodule remap commits instead of year 2000 Fixed timestamp was deterministic but showed as 27 years old in git log. Now uses the current date/time, fixed for the entire run so all branches in one repo get the same timestamp. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/submodules.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py index 7a864a5..4f9ce96 100644 --- a/bb2gh/submodules.py +++ b/bb2gh/submodules.py @@ -4,17 +4,16 @@ import os import re import subprocess +from datetime import datetime, timezone from urllib.parse import urlparse logger = logging.getLogger(__name__) -_REMAP_ENV = { +_REMAP_IDENTITY = { "GIT_AUTHOR_NAME": "bb2gh", "GIT_AUTHOR_EMAIL": "bb2gh@migration", - "GIT_AUTHOR_DATE": "2000-01-01T00:00:00+00:00", "GIT_COMMITTER_NAME": "bb2gh", "GIT_COMMITTER_EMAIL": "bb2gh@migration", - "GIT_COMMITTER_DATE": "2000-01-01T00:00:00+00:00", } _COMMIT_MSG = "bb2gh: remap submodule URLs for GitHub migration" @@ -213,12 +212,15 @@ def remap_submodules_in_bare_repo(bare_repo_path, config, alias_resolver=None): if not output.strip(): return 0 + # Fixed timestamp for this run — deterministic within a cycle but a real date + run_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00") + # Shared cache for API results across all branches in this repo alias_cache = {} remapped = 0 for ref in output.strip().splitlines(): - if _remap_branch(bare_repo_path, ref, config, alias_resolver, alias_cache): + if _remap_branch(bare_repo_path, ref, config, alias_resolver, alias_cache, run_timestamp): remapped += 1 if remapped: @@ -230,7 +232,7 @@ def remap_submodules_in_bare_repo(bare_repo_path, config, alias_resolver=None): return remapped -def _remap_branch(bare_repo_path, ref, config, alias_resolver=None, alias_cache=None): +def _remap_branch(bare_repo_path, ref, config, alias_resolver=None, alias_cache=None, run_timestamp=None): """Remap .gitmodules on a single branch ref. Returns True if changed.""" try: content = _git(["show", f"{ref}:.gitmodules"], cwd=bare_repo_path) @@ -263,9 +265,14 @@ def _remap_branch(bare_repo_path, ref, config, alias_resolver=None, alias_cache= ) parent = _git(["rev-parse", ref], cwd=bare_repo_path) + commit_env = { + **_REMAP_IDENTITY, + "GIT_AUTHOR_DATE": run_timestamp or datetime.now(timezone.utc).isoformat(), + "GIT_COMMITTER_DATE": run_timestamp or datetime.now(timezone.utc).isoformat(), + } new_commit = _git( ["commit-tree", new_tree, "-p", parent, "-m", _COMMIT_MSG], - cwd=bare_repo_path, env_extra=_REMAP_ENV, + cwd=bare_repo_path, env_extra=commit_env, ) _git(["update-ref", ref, new_commit], cwd=bare_repo_path) From ec5fdd0271f84c450fde74f298246629196eb430 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 14 May 2026 03:51:54 +0000 Subject: [PATCH 66/74] Fix false change detection by cleaning hidden refs before snapshot Hidden refs (refs/pull/*, refs/merge-request/*) were included in the snapshot but cleaned before push. Next cycle's show-ref didn't include them, causing a mismatch every time. Now cleans before snapshotting. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 03fba88..c95abd6 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -171,7 +171,10 @@ def _sync_repo(self, project_key, repo_slug): "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"], cwd=bare_path) - # Compare Bitbucket's refs (post-fetch) against last sync snapshot + # Clean hidden refs before snapshot (so snapshot is consistent) + _clean_hidden_refs(bare_path) + + # Compare Bitbucket's refs (post-fetch, post-clean) against last sync snapshot try: bb_refs = _run_git(["show-ref"], cwd=bare_path, quiet=True) except subprocess.CalledProcessError: @@ -190,9 +193,6 @@ def _sync_repo(self, project_key, repo_slug): logger.info("Changes detected for %s/%s, pushing...", project_key, repo_slug) start = time.time() - # Clean hidden refs before pushing - _clean_hidden_refs(bare_path) - # Trim history if configured trim_since = self.config.get_trim_since(project_key, repo_slug) if trim_since: From 57d2dc108c2848dea59877276788555c9cc08c42 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 14 May 2026 03:56:34 +0000 Subject: [PATCH 67/74] Suppress show-ref error log in _clean_hidden_refs for empty repos https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index c95abd6..2d089b3 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -33,7 +33,7 @@ def _run_git(args, cwd=None, quiet=False): def _clean_hidden_refs(bare_repo_path): """Remove hidden refs that can't be pushed to GitHub.""" try: - output = _run_git(["show-ref"], cwd=bare_repo_path) + output = _run_git(["show-ref"], cwd=bare_repo_path, quiet=True) except subprocess.CalledProcessError: return From d606c497daf0d1f7266e106a981a7cece8c55e88 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 14 May 2026 04:08:26 +0000 Subject: [PATCH 68/74] Fix false change detection by comparing only heads+tags, not remotes show-ref includes refs/remotes/github/* which change every push cycle (remap commits have different hashes due to timestamps). Now uses show-ref --heads --tags to only compare Bitbucket's branches and tags, ignoring internal remote tracking refs. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 9 +++++---- tests/test_syncer.py | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 2d089b3..3df7bce 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -175,10 +175,11 @@ def _sync_repo(self, project_key, repo_slug): _clean_hidden_refs(bare_path) # Compare Bitbucket's refs (post-fetch, post-clean) against last sync snapshot + # Only compare heads and tags — ignore refs/remotes/* which change on every push try: - bb_refs = _run_git(["show-ref"], cwd=bare_path, quiet=True) + bb_heads = _run_git(["show-ref", "--heads", "--tags"], cwd=bare_path, quiet=True) except subprocess.CalledProcessError: - bb_refs = "" + bb_heads = "" refs_file = os.path.join(bare_path, "bb2gh_last_sync_refs") last_refs = "" @@ -186,7 +187,7 @@ def _sync_repo(self, project_key, repo_slug): with open(refs_file) as f: last_refs = f.read() - if bb_refs == last_refs: + if bb_heads == last_refs: logger.debug("No changes for %s/%s, skipping push", project_key, repo_slug) return False @@ -234,7 +235,7 @@ def _sync_repo(self, project_key, repo_slug): elapsed = time.time() - start # Store Bitbucket's refs so next cycle can detect real changes with open(refs_file, "w") as f: - f.write(bb_refs) + f.write(bb_heads) self.state.update_sync_time(project_key, repo_slug) logger.info( diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 3be610f..db455aa 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -41,7 +41,7 @@ def test_sync_repo_with_changes(self, mock_git, MockState, MockBB, mock_config, state_instance.get_github_target.return_value = ("my-org", "my-repo") def side_effect(args, cwd=None, quiet=False): - if args == ["show-ref"]: + if args == ["show-ref", "--heads", "--tags"]: return "abc123 refs/heads/master" return "" @@ -74,7 +74,7 @@ def test_sync_skips_when_no_changes(self, mock_git, MockState, MockBB, mock_conf state_instance.get_github_target.return_value = ("my-org", "my-repo") def side_effect(args, cwd=None, quiet=False): - if args == ["show-ref"]: + if args == ["show-ref", "--heads", "--tags"]: return "abc123 refs/heads/master" return "" @@ -118,7 +118,7 @@ def test_sync_handles_failure(self, mock_git, MockState, MockBB, mock_config, tm def side_effect(args, cwd=None, quiet=False): if "repo1" in str(cwd) and args[0] == "fetch": raise Exception("Network error") - if args == ["show-ref"]: + if args == ["show-ref", "--heads", "--tags"]: return "abc123 refs/heads/master" return "" @@ -144,7 +144,7 @@ def test_sync_logs_github_target(self, mock_git, MockState, MockBB, mock_config, state_instance.get_github_target.return_value = ("infra-team", "infra-my-service") def side_effect(args, cwd=None, quiet=False): - if args == ["show-ref"]: + if args == ["show-ref", "--heads", "--tags"]: return "aaa refs/heads/main" return "" From 6c3ccbbdae082b1b6cf9fec2ef62f0f607d2c1ff Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 14 May 2026 06:08:44 +0000 Subject: [PATCH 69/74] Add timeout to git lfs push --all in syncer The LFS push had no timeout and would hang indefinitely scanning large repos. Now respects sync_lfs_timeout_seconds for both the migration and the push steps. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/syncer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index 3df7bce..faa8b4e 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -228,7 +228,13 @@ def _sync_repo(self, project_key, repo_slug): if has_lfs: try: - _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) + cmd = ["git", "lfs", "push", "--all", "github"] + subprocess.run( + cmd, cwd=bare_path, capture_output=True, text=True, + check=True, timeout=self.config.sync_lfs_timeout, + ) + except subprocess.TimeoutExpired: + logger.warning("LFS push timed out for %s/%s", project_key, repo_slug) except subprocess.CalledProcessError: logger.warning("LFS push failed for %s/%s", project_key, repo_slug) From 4e6540ca35a650d8de2cbb68d43c77c1ae6de040 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 14 May 2026 06:12:04 +0000 Subject: [PATCH 70/74] Replace git lfs push --all with direct object-id pushes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --all scans every branch and tag history for LFS pointers, taking hours on repos with many refs even with only a few LFS files. Now enumerates actual LFS object files in lfs/objects/ and pushes each by OID directly — no scanning, instant for a handful of objects. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/migrator.py | 22 +++++++++++++++++++++- bb2gh/syncer.py | 12 +++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py index efe38e2..6998bec 100644 --- a/bb2gh/migrator.py +++ b/bb2gh/migrator.py @@ -348,6 +348,26 @@ def migrate_repos(config, only_repos=None): return total_migrated, total_skipped, total_failed +def _push_lfs_objects(bare_path): + """Push LFS objects by OID instead of scanning all refs with --all.""" + lfs_dir = os.path.join(bare_path, "lfs", "objects") + if not os.path.exists(lfs_dir): + return + oids = [] + for dirpath, _, filenames in os.walk(lfs_dir): + for f in filenames: + if len(f) == 64: + oids.append(f) + if not oids: + return + logger.info("Pushing %d LFS objects", len(oids)) + for oid in oids: + try: + _run_git(["lfs", "push", "github", "--object-id", oid], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + logger.warning("Failed to push LFS object %s", oid[:12]) + + def _push_branch_by_branch(bare_path, project_key, repo_slug, delay=2): """Push branches and tags individually when --mirror pack exceeds 2GB.""" branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], @@ -477,7 +497,7 @@ def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_nam # Push LFS objects separately — only if LFS actually converted files if has_lfs: try: - _run_git(["lfs", "push", "--all", "github"], cwd=bare_path) + _push_lfs_objects(bare_path) except subprocess.CalledProcessError: logger.warning("LFS push failed for %s/%s", gh_org, gh_repo_name) warnings.append("LFS push failed") diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py index faa8b4e..6f943d0 100644 --- a/bb2gh/syncer.py +++ b/bb2gh/syncer.py @@ -7,7 +7,7 @@ import time from .bitbucket_client import BitbucketClient -from .migrator import _migrate_lfs, _has_large_blobs, _trim_history, _push_branch_by_branch +from .migrator import _migrate_lfs, _has_large_blobs, _trim_history, _push_branch_by_branch, _push_lfs_objects from .state import State from .submodules import remap_submodules_in_bare_repo @@ -228,14 +228,8 @@ def _sync_repo(self, project_key, repo_slug): if has_lfs: try: - cmd = ["git", "lfs", "push", "--all", "github"] - subprocess.run( - cmd, cwd=bare_path, capture_output=True, text=True, - check=True, timeout=self.config.sync_lfs_timeout, - ) - except subprocess.TimeoutExpired: - logger.warning("LFS push timed out for %s/%s", project_key, repo_slug) - except subprocess.CalledProcessError: + _push_lfs_objects(bare_path) + except Exception: logger.warning("LFS push failed for %s/%s", project_key, repo_slug) elapsed = time.time() - start From 98956dce00013355b847348599759eeadea714ee Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 19:36:48 +0000 Subject: [PATCH 71/74] Add closed PR migration with branch recreation from commit SHAs --include-closed flag migrates merged/declined PRs by: 1. Finding the source branch's last commit SHA from Bitbucket PR data 2. If the commit exists in the repo, recreating the branch on GitHub 3. Creating a real PR with full discussion, then closing it 4. Fallback: if commit doesn't exist, creates a GitHub Issue instead Also adds --repo filter to migrate-prs command. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 15 ++- bb2gh/pr_migrator.py | 242 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 214 insertions(+), 43 deletions(-) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 3992f64..ea281dc 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -66,16 +66,23 @@ def sync(ctx): @cli.command("migrate-prs") +@click.option("--repo", multiple=True, help="Migrate PRs for specific repos only (PROJECT/SLUG).") +@click.option("--include-closed", is_flag=True, help="Also migrate merged/declined PRs.") @click.option("--dry-run", is_flag=True, help="Log what would be done without making changes.") @click.pass_context -def migrate_prs(ctx, dry_run): - """Migrate open pull requests from Bitbucket to GitHub. +def migrate_prs(ctx, repo, include_closed, dry_run): + """Migrate pull requests from Bitbucket to GitHub. Creates matching PRs on GitHub with title, description, comments, - and reviewer assignments. + and reviewer assignments. With --include-closed, also migrates + merged/declined PRs by recreating branches from commit SHAs + (falls back to GitHub Issues if the commit no longer exists). """ config = ctx.obj["config"] - migrated, skipped, failed = migrate_pull_requests(config, dry_run=dry_run) + only_repos = set(repo) if repo else None + migrated, skipped, failed = migrate_pull_requests( + config, dry_run=dry_run, include_closed=include_closed, only_repos=only_repos, + ) click.echo(f"\nPR migration complete: {migrated} migrated, {skipped} skipped, {failed} failed") if failed > 0: sys.exit(1) diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py index 42ff452..6b952a6 100644 --- a/bb2gh/pr_migrator.py +++ b/bb2gh/pr_migrator.py @@ -1,6 +1,8 @@ """Migrate pull requests from Bitbucket Server to GitHub Enterprise.""" import logging +import os +import subprocess from .bitbucket_client import BitbucketClient from .github_client import GithubClient @@ -9,7 +11,21 @@ logger = logging.getLogger(__name__) -def _format_pr_body(pr, config): +def _run_git(args, cwd=None, quiet=False): + cmd = ["git"] + args + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + if not quiet: + logger.error("git %s failed: %s", args[0], result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def _format_pr_body(pr, config, closed_state=None): """Format the GitHub PR body with migration metadata.""" bb_url = config.bb_base_url project = pr["toRef"]["repository"]["project"]["key"] @@ -18,7 +34,6 @@ def _format_pr_body(pr, config): author = pr["author"]["user"].get("displayName", pr["author"]["user"].get("name", "Unknown")) created = pr.get("createdDate", "") - # Build metadata header header = ( f"> **Migrated from Bitbucket**\n" f"> Source: [{project}/{repo} PR #{pr_id}]" @@ -27,6 +42,8 @@ def _format_pr_body(pr, config): ) if created: header += f"> Created: {created}\n" + if closed_state: + header += f"> Original status: **{closed_state}**\n" description = pr.get("description", "") or "" return f"{header}\n---\n\n{description}" @@ -59,12 +76,49 @@ def _map_reviewers(pr, config): return reviewers -def migrate_pull_requests(config, dry_run=False): - """Migrate open pull requests from Bitbucket to GitHub. +def _commit_exists(bare_path, sha): + """Check if a commit SHA exists in the bare repo.""" + try: + _run_git(["cat-file", "-t", sha], cwd=bare_path, quiet=True) + return True + except subprocess.CalledProcessError: + return False + + +def _create_temp_branch(bare_path, branch_name, sha): + """Create a branch pointing to a specific commit in the bare repo.""" + try: + _run_git(["branch", branch_name, sha], cwd=bare_path, quiet=True) + return True + except subprocess.CalledProcessError: + return False + + +def _push_temp_branch(bare_path, branch_name): + """Push a branch to the github remote.""" + try: + _run_git(["push", "github", f"{branch_name}:{branch_name}"], cwd=bare_path, quiet=True) + return True + except subprocess.CalledProcessError: + return False + + +def _delete_temp_branch(bare_path, branch_name): + """Delete a local branch from the bare repo.""" + try: + _run_git(["branch", "-D", branch_name], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + pass + + +def migrate_pull_requests(config, dry_run=False, include_closed=False, only_repos=None): + """Migrate pull requests from Bitbucket to GitHub. Args: config: Config object. dry_run: If True, log what would be done without making changes. + include_closed: If True, also migrate merged/declined PRs. + only_repos: Optional set of "PROJECT/SLUG" strings to filter repos. """ bb = BitbucketClient(config.bb_base_url, config.bb_token, verify_ssl=config.bb_verify_ssl) gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) @@ -80,23 +134,37 @@ def migrate_pull_requests(config, dry_run=False): total_failed = 0 for project_key, repo_slug in migrated_repos: - # Look up the GitHub target for this repo + repo_key = f"{project_key}/{repo_slug}" + if only_repos and repo_key not in only_repos: + continue + gh_org, gh_repo_name = state.get_github_target(project_key, repo_slug) if not gh_org or not gh_repo_name: - # Fallback: resolve from config (for repos migrated before mapping was added) gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) logger.info("Processing PRs for %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) + # Fetch open PRs try: - open_prs = bb.list_pull_requests(project_key, repo_slug, state="OPEN") + prs = bb.list_pull_requests(project_key, repo_slug, state="OPEN") except Exception: logger.exception("Failed to list PRs for %s/%s", project_key, repo_slug) continue - for pr in open_prs: + # Fetch closed PRs if requested + if include_closed: + try: + merged = bb.list_pull_requests(project_key, repo_slug, state="MERGED") + declined = bb.list_pull_requests(project_key, repo_slug, state="DECLINED") + prs.extend(merged) + prs.extend(declined) + except Exception: + logger.exception("Failed to list closed PRs for %s/%s", project_key, repo_slug) + + for pr in prs: pr_id = pr["id"] title = pr["title"] + pr_state = pr.get("state", "OPEN") if state.is_pr_migrated(project_key, repo_slug, pr_id): logger.info("Skipping already migrated PR #%d: %s", pr_id, title) @@ -108,17 +176,23 @@ def migrate_pull_requests(config, dry_run=False): if dry_run: logger.info( - "[DRY RUN] Would migrate PR #%d: %s (%s -> %s) to %s/%s", - pr_id, title, head_branch, base_branch, gh_org, gh_repo_name, + "[DRY RUN] Would migrate PR #%d [%s]: %s (%s -> %s) to %s/%s", + pr_id, pr_state, title, head_branch, base_branch, gh_org, gh_repo_name, ) total_migrated += 1 continue try: - _migrate_single_pr( - config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr, - ) + if pr_state == "OPEN": + _migrate_open_pr( + config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, + ) + else: + _migrate_closed_pr( + config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, + ) total_migrated += 1 except Exception: logger.exception( @@ -133,48 +207,138 @@ def migrate_pull_requests(config, dry_run=False): return total_migrated, total_skipped, total_failed -def _migrate_single_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr): - """Migrate a single pull request.""" +def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr): + """Migrate an open pull request.""" pr_id = pr["id"] title = pr["title"] head_branch = pr["fromRef"]["displayId"] base_branch = pr["toRef"]["displayId"] - logger.info( - "Migrating PR #%d: %s (%s -> %s) to %s/%s", - pr_id, title, head_branch, base_branch, gh_org, gh_repo_name, - ) + logger.info("Migrating open PR #%d: %s (%s -> %s)", pr_id, title, head_branch, base_branch) - # Create PR on GitHub (in the correct org) body = _format_pr_body(pr, config) gh_pr = gh.create_pull_request( - repo_name=gh_repo_name, - title=title, + repo_name=gh_repo_name, title=title, body=body, + head=head_branch, base=base_branch, org_name=gh_org, + ) + + _migrate_pr_comments(bb, gh, config, project_key, repo_slug, gh_org, gh_repo_name, pr_id, gh_pr.number) + + reviewers = _map_reviewers(pr, config) + if reviewers: + gh.add_pr_reviewers(gh_repo_name, gh_pr.number, reviewers, org_name=gh_org) + + state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) + logger.info("Migrated open PR #%d -> GitHub PR #%d", pr_id, gh_pr.number) + + +def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr): + """Migrate a closed (merged/declined) PR by recreating the branch from the commit SHA.""" + pr_id = pr["id"] + title = pr["title"] + pr_state = pr.get("state", "UNKNOWN") + head_branch = pr["fromRef"]["displayId"] + base_branch = pr["toRef"]["displayId"] + head_sha = pr["fromRef"].get("latestCommit", "") + + logger.info("Migrating %s PR #%d: %s (%s -> %s)", pr_state, pr_id, title, head_branch, base_branch) + + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + + # Try to recreate the source branch from the commit SHA + temp_branch = f"migrated-pr/{pr_id}/{head_branch}" + branch_created = False + + if head_sha and os.path.exists(bare_path) and _commit_exists(bare_path, head_sha): + if _create_temp_branch(bare_path, temp_branch, head_sha): + if _push_temp_branch(bare_path, temp_branch): + branch_created = True + _delete_temp_branch(bare_path, temp_branch) + + if branch_created: + # Create a real PR on GitHub, then close it + body = _format_pr_body(pr, config, closed_state=pr_state) + try: + gh_pr = gh.create_pull_request( + repo_name=gh_repo_name, + title=f"[{pr_state}] {title}", + body=body, + head=temp_branch, + base=base_branch, + org_name=gh_org, + ) + + _migrate_pr_comments(bb, gh, config, project_key, repo_slug, + gh_org, gh_repo_name, pr_id, gh_pr.number) + + # Close the PR with a status comment + gh.add_pr_comment( + gh_repo_name, gh_pr.number, + f"This PR was **{pr_state.lower()}** on Bitbucket. " + f"Migrated for historical reference.", + org_name=gh_org, + ) + + # Close the PR + repo = gh.get_repo(gh_repo_name, org_name=gh_org) + gh_pull = repo.get_pull(gh_pr.number) + gh_pull.edit(state="closed") + + state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) + logger.info("Migrated %s PR #%d -> GitHub PR #%d (closed)", pr_state, pr_id, gh_pr.number) + return + + except Exception: + logger.warning("Could not create PR for %s PR #%d, falling back to issue", pr_state, pr_id) + + # Fallback: create as a GitHub Issue + _migrate_pr_as_issue(config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr) + + +def _migrate_pr_as_issue(config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr): + """Migrate a PR as a GitHub Issue (when branch can't be recreated).""" + pr_id = pr["id"] + title = pr["title"] + pr_state = pr.get("state", "UNKNOWN") + + body = _format_pr_body(pr, config, closed_state=pr_state) + body += f"\n\n---\n*Migrated as issue because the source branch could not be recreated.*" + + repo = gh.get_repo(gh_repo_name, org_name=gh_org) + + # Create issue + issue = repo.create_issue( + title=f"[Migrated {pr_state} PR #{pr_id}] {title}", body=body, - head=head_branch, - base=base_branch, - org_name=gh_org, + labels=["migrated-pr", pr_state.lower()], ) # Migrate comments activities = bb.get_pr_activities(project_key, repo_slug, pr_id) - comment_count = 0 for activity in activities: action = activity.get("action", "") if action == "COMMENTED" and "comment" in activity: comment_body = _format_comment(activity, config) - gh.add_pr_comment(gh_repo_name, gh_pr.number, comment_body, org_name=gh_org) - comment_count += 1 + issue.create_comment(comment_body) - # Assign reviewers (best effort) - reviewers = _map_reviewers(pr, config) - if reviewers: - gh.add_pr_reviewers(gh_repo_name, gh_pr.number, reviewers, org_name=gh_org) + # Close the issue + issue.edit(state="closed") - # Record mapping - state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) + state.record_pr_mapping(project_key, repo_slug, pr_id, issue.number) + logger.info("Migrated %s PR #%d -> GitHub Issue #%d (closed)", pr_state, pr_id, issue.number) - logger.info( - "Migrated PR #%d -> GitHub PR #%d on %s/%s (%d comments, %d reviewers)", - pr_id, gh_pr.number, gh_org, gh_repo_name, comment_count, len(reviewers), - ) + +def _migrate_pr_comments(bb, gh, config, project_key, repo_slug, + gh_org, gh_repo_name, pr_id, gh_pr_number): + """Migrate comments from a Bitbucket PR to a GitHub PR.""" + activities = bb.get_pr_activities(project_key, repo_slug, pr_id) + comment_count = 0 + for activity in activities: + action = activity.get("action", "") + if action == "COMMENTED" and "comment" in activity: + comment_body = _format_comment(activity, config) + gh.add_pr_comment(gh_repo_name, gh_pr_number, comment_body, org_name=gh_org) + comment_count += 1 + return comment_count From 4cc35dcbc8278df30dcb73c93d154acd75604275 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 19:53:06 +0000 Subject: [PATCH 72/74] Add merge/squash commit SHA fallback for closed PR migration When fromRef.latestCommit is garbage-collected (common after squash merges), the migrator now falls back to: 1. properties.mergeCommit from the Bitbucket PR object 2. The MERGED activity's commit field For squash/merge commits (which are already on the target branch), creates a temp base branch at the commit's parent so the GitHub PR shows the actual squash diff rather than "nothing to compare". Also adds BitbucketClient.get_pull_request() and get_merge_commit() methods, plus tests for the SHA resolution cascade. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/bitbucket_client.py | 26 +++++++++ bb2gh/pr_migrator.py | 73 +++++++++++++++++++++--- tests/test_pr_migrator.py | 114 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 7 deletions(-) diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py index e8e9987..91ede78 100644 --- a/bb2gh/bitbucket_client.py +++ b/bb2gh/bitbucket_client.py @@ -88,6 +88,32 @@ def list_pull_requests(self, project_key, repo_slug, state="OPEN"): url = f"{self.api_url}/projects/{project_key}/repos/{repo_slug}/pull-requests" return list(self._paginate(url, params={"state": state})) + def get_pull_request(self, project_key, repo_slug, pr_id): + """Get a single pull request with full details (including merge properties).""" + url = ( + f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" + f"/pull-requests/{pr_id}" + ) + resp = self.session.get(url) + resp.raise_for_status() + return resp.json() + + def get_merge_commit(self, project_key, repo_slug, pr_id): + """Extract the merge/squash commit SHA for a merged PR. + + Bitbucket Server stores this in properties.mergeCommit on the PR object. + Returns the SHA string, or None if not available. + """ + try: + pr = self.get_pull_request(project_key, repo_slug, pr_id) + merge_commit = pr.get("properties", {}).get("mergeCommit", {}) + sha = merge_commit.get("id") or merge_commit.get("displayId") + if sha: + return sha + except Exception: + logger.debug("Could not fetch merge commit from PR properties for PR #%d", pr_id) + return None + def get_pr_activities(self, project_key, repo_slug, pr_id): """Get activities (comments, approvals, etc.) for a pull request.""" url = ( diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py index 6b952a6..87613b1 100644 --- a/bb2gh/pr_migrator.py +++ b/bb2gh/pr_migrator.py @@ -232,6 +232,46 @@ def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_r logger.info("Migrated open PR #%d -> GitHub PR #%d", pr_id, gh_pr.number) +def _resolve_head_sha(bb, bare_path, project_key, repo_slug, pr): + """Find a usable commit SHA for recreating a closed PR's head branch. + + Tries in order: + 1. fromRef.latestCommit — the original branch tip (works for regular merges) + 2. merge/squash commit from PR properties — guaranteed to exist for merged PRs + 3. merge commit from PR activities — alternative source for the same info + + Returns (sha, is_merge_commit) or (None, False). + """ + pr_id = pr["id"] + head_sha = pr["fromRef"].get("latestCommit", "") + + # 1. Original source branch tip + if head_sha and _commit_exists(bare_path, head_sha): + logger.debug("PR #%d: using fromRef.latestCommit %s", pr_id, head_sha[:12]) + return head_sha, False + + # 2. Merge/squash commit from PR properties + merge_sha = bb.get_merge_commit(project_key, repo_slug, pr_id) + if merge_sha and _commit_exists(bare_path, merge_sha): + logger.debug("PR #%d: using merge commit %s from properties", pr_id, merge_sha[:12]) + return merge_sha, True + + # 3. Merge commit from activities + try: + activities = bb.get_pr_activities(project_key, repo_slug, pr_id) + for activity in activities: + if activity.get("action") == "MERGED": + commit = activity.get("commit", {}) + act_sha = commit.get("id") or commit.get("displayId") + if act_sha and _commit_exists(bare_path, act_sha): + logger.debug("PR #%d: using merge commit %s from activity", pr_id, act_sha[:12]) + return act_sha, True + except Exception: + logger.debug("PR #%d: could not fetch activities for merge commit", pr_id) + + return None, False + + def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr): """Migrate a closed (merged/declined) PR by recreating the branch from the commit SHA.""" pr_id = pr["id"] @@ -239,21 +279,40 @@ def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh pr_state = pr.get("state", "UNKNOWN") head_branch = pr["fromRef"]["displayId"] base_branch = pr["toRef"]["displayId"] - head_sha = pr["fromRef"].get("latestCommit", "") logger.info("Migrating %s PR #%d: %s (%s -> %s)", pr_state, pr_id, title, head_branch, base_branch) bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") - # Try to recreate the source branch from the commit SHA temp_branch = f"migrated-pr/{pr_id}/{head_branch}" branch_created = False + pr_base = base_branch + + if os.path.exists(bare_path): + head_sha, is_merge_commit = _resolve_head_sha( + bb, bare_path, project_key, repo_slug, pr, + ) - if head_sha and os.path.exists(bare_path) and _commit_exists(bare_path, head_sha): - if _create_temp_branch(bare_path, temp_branch, head_sha): - if _push_temp_branch(bare_path, temp_branch): - branch_created = True - _delete_temp_branch(bare_path, temp_branch) + if head_sha: + if is_merge_commit: + # The merge/squash commit is already on the target branch. + # To get a meaningful diff, target the PR at the commit's parent. + try: + parent_sha = _run_git( + ["rev-parse", f"{head_sha}^"], cwd=bare_path, quiet=True, + ) + # Create a temp base branch at the parent so the PR shows the squash diff + pr_base = f"migrated-pr/{pr_id}/base" + if _create_temp_branch(bare_path, pr_base, parent_sha): + _push_temp_branch(bare_path, pr_base) + _delete_temp_branch(bare_path, pr_base) + except subprocess.CalledProcessError: + logger.debug("PR #%d: could not resolve parent of merge commit", pr_id) + + if _create_temp_branch(bare_path, temp_branch, head_sha): + if _push_temp_branch(bare_path, temp_branch): + branch_created = True + _delete_temp_branch(bare_path, temp_branch) if branch_created: # Create a real PR on GitHub, then close it diff --git a/tests/test_pr_migrator.py b/tests/test_pr_migrator.py index f2eb067..eae8382 100644 --- a/tests/test_pr_migrator.py +++ b/tests/test_pr_migrator.py @@ -10,6 +10,7 @@ _format_pr_body, _format_comment, _map_reviewers, + _resolve_head_sha, ) @@ -239,3 +240,116 @@ def test_fallback_to_config_resolve(self, MockBB, MockGH, MockState, mock_config call_kwargs = MockGH.return_value.create_pull_request.call_args assert call_kwargs[1]["org_name"] == "fallback-org" assert call_kwargs[1]["repo_name"] == "fallback-repo" + + +class TestResolveHeadSha: + """Tests for _resolve_head_sha — the SHA resolution cascade for closed PRs.""" + + def _make_pr(self, head_sha="abc123"): + return { + "id": 10, + "fromRef": {"displayId": "feature/x", "latestCommit": head_sha}, + "toRef": {"displayId": "main"}, + } + + @patch("bb2gh.pr_migrator._commit_exists") + def test_uses_from_ref_when_commit_exists(self, mock_exists): + mock_exists.return_value = True + bb = MagicMock() + + sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("deadbeef")) + + assert sha == "deadbeef" + assert is_merge is False + bb.get_merge_commit.assert_not_called() + + @patch("bb2gh.pr_migrator._commit_exists") + def test_falls_back_to_merge_commit_property(self, mock_exists): + mock_exists.side_effect = lambda path, sha: sha == "squash111" + bb = MagicMock() + bb.get_merge_commit.return_value = "squash111" + + sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("gone")) + + assert sha == "squash111" + assert is_merge is True + + @patch("bb2gh.pr_migrator._commit_exists") + def test_falls_back_to_merge_activity(self, mock_exists): + mock_exists.side_effect = lambda path, sha: sha == "act222" + bb = MagicMock() + bb.get_merge_commit.return_value = None + bb.get_pr_activities.return_value = [ + {"action": "COMMENTED", "comment": {"text": "hi"}}, + {"action": "MERGED", "commit": {"id": "act222", "displayId": "act222"}}, + ] + + sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("gone")) + + assert sha == "act222" + assert is_merge is True + + @patch("bb2gh.pr_migrator._commit_exists") + def test_returns_none_when_nothing_found(self, mock_exists): + mock_exists.return_value = False + bb = MagicMock() + bb.get_merge_commit.return_value = None + bb.get_pr_activities.return_value = [] + + sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("gone")) + + assert sha is None + assert is_merge is False + + @patch("bb2gh.pr_migrator._commit_exists") + def test_handles_empty_latest_commit(self, mock_exists): + mock_exists.return_value = False + bb = MagicMock() + bb.get_merge_commit.return_value = None + bb.get_pr_activities.return_value = [] + + sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("")) + + assert sha is None + assert is_merge is False + + +class TestFormatPrBodyClosed: + def test_includes_closed_state(self, sample_pr, mock_config): + body = _format_pr_body(sample_pr, mock_config, closed_state="MERGED") + assert "MERGED" in body + assert "Original status" in body + + def test_no_status_for_open(self, sample_pr, mock_config): + body = _format_pr_body(sample_pr, mock_config) + assert "Original status" not in body + + +class TestMigrateClosedDryRun: + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_dry_run_includes_closed_prs(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + merged_pr = dict(sample_pr, id=99, state="MERGED", + title="Already merged") + merged_pr["fromRef"] = dict(sample_pr["fromRef"], latestCommit="aaa") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "OPEN": [sample_pr], + "MERGED": [merged_pr], + "DECLINED": [], + }[state] + + migrated, skipped, failed = migrate_pull_requests( + mock_config, dry_run=True, include_closed=True, + ) + + assert migrated == 2 + assert skipped == 0 + MockGH.return_value.create_pull_request.assert_not_called() From 01f5820ba01dc13d7fc62b5b7dad6f490124faaa Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Jul 2026 17:45:06 +0000 Subject: [PATCH 73/74] Add --closed-only flag and API rate-limit throttling for PR migration --closed-only skips open PRs entirely (for users who already migrated open PRs and now want to backfill merged/declined ones). Throttler class enforces: - Minimum spacing between GitHub API calls (--api-delay, default 0.5s) - Pause between full PR migrations (--pr-delay, default 3s) - Automatic retry with exponential backoff on 429 / secondary rate limits, honoring Retry-After header when present Defaults are configurable via a new pr_migration section in config.yaml: pr_migration: api_delay_seconds: 0.5 pr_delay_seconds: 3.0 max_retries: 5 https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/cli.py | 25 +++++- bb2gh/config.py | 7 ++ bb2gh/pr_migrator.py | 162 +++++++++++++++++++++++++++++--------- tests/test_pr_migrator.py | 107 +++++++++++++++++++++++++ 4 files changed, 259 insertions(+), 42 deletions(-) diff --git a/bb2gh/cli.py b/bb2gh/cli.py index ea281dc..2c477fd 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -67,21 +67,40 @@ def sync(ctx): @cli.command("migrate-prs") @click.option("--repo", multiple=True, help="Migrate PRs for specific repos only (PROJECT/SLUG).") -@click.option("--include-closed", is_flag=True, help="Also migrate merged/declined PRs.") +@click.option("--include-closed", is_flag=True, help="Also migrate merged/declined PRs (in addition to open).") +@click.option("--closed-only", is_flag=True, help="Migrate ONLY merged/declined PRs, skip open ones.") +@click.option("--api-delay", type=float, default=None, + help="Seconds between API calls (default from config, or 0.5).") +@click.option("--pr-delay", type=float, default=None, + help="Seconds between full PR migrations (default from config, or 3.0).") @click.option("--dry-run", is_flag=True, help="Log what would be done without making changes.") @click.pass_context -def migrate_prs(ctx, repo, include_closed, dry_run): +def migrate_prs(ctx, repo, include_closed, closed_only, api_delay, pr_delay, dry_run): """Migrate pull requests from Bitbucket to GitHub. Creates matching PRs on GitHub with title, description, comments, and reviewer assignments. With --include-closed, also migrates merged/declined PRs by recreating branches from commit SHAs (falls back to GitHub Issues if the commit no longer exists). + With --closed-only, skips open PRs entirely (useful if you've + already migrated the open ones). + + Rate limiting: --api-delay controls minimum spacing between GitHub + API calls; --pr-delay adds a pause between full PR migrations. + Both fall back to the pr_migration section of config.yaml. """ config = ctx.obj["config"] only_repos = set(repo) if repo else None + + if api_delay is not None: + config.pr_api_delay = api_delay + if pr_delay is not None: + config.pr_pr_delay = pr_delay + migrated, skipped, failed = migrate_pull_requests( - config, dry_run=dry_run, include_closed=include_closed, only_repos=only_repos, + config, dry_run=dry_run, + include_closed=include_closed, closed_only=closed_only, + only_repos=only_repos, ) click.echo(f"\nPR migration complete: {migrated} migrated, {skipped} skipped, {failed} failed") if failed > 0: diff --git a/bb2gh/config.py b/bb2gh/config.py index c4974b0..2de3516 100644 --- a/bb2gh/config.py +++ b/bb2gh/config.py @@ -42,6 +42,13 @@ def __init__(self, path="config.yaml"): ) self.sync_protected_branches = sync.get("protected_branches", []) + # PR migration settings + pr = raw.get("pr_migration", {}) + self.pr_api_delay = pr.get("api_delay_seconds", 0.5) + self.pr_pr_delay = pr.get("pr_delay_seconds", 3.0) + self.pr_retry_on_rate_limit = pr.get("retry_on_rate_limit", True) + self.pr_max_retries = pr.get("max_retries", 5) + # User mapping (Bitbucket username -> GitHub username) self.user_mapping = raw.get("user_mapping", {}) diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py index 87613b1..ae78091 100644 --- a/bb2gh/pr_migrator.py +++ b/bb2gh/pr_migrator.py @@ -2,7 +2,11 @@ import logging import os +import random import subprocess +import time + +from github import GithubException from .bitbucket_client import BitbucketClient from .github_client import GithubClient @@ -11,6 +15,67 @@ logger = logging.getLogger(__name__) +class Throttler: + """Rate-limits API calls with a fixed delay and retry-on-rate-limit backoff.""" + + def __init__(self, api_delay=0.5, pr_delay=3.0, max_retries=5): + self.api_delay = api_delay + self.pr_delay = pr_delay + self.max_retries = max_retries + self._last_call = 0.0 + + def wait_api(self): + """Sleep to enforce minimum spacing between API calls.""" + if self.api_delay <= 0: + return + elapsed = time.monotonic() - self._last_call + remaining = self.api_delay - elapsed + if remaining > 0: + time.sleep(remaining) + self._last_call = time.monotonic() + + def wait_between_prs(self): + """Sleep between full PR migrations.""" + if self.pr_delay > 0: + time.sleep(self.pr_delay) + + def call(self, fn, *args, **kwargs): + """Invoke fn(*args, **kwargs) with rate limiting and retry on 429/403.""" + for attempt in range(self.max_retries + 1): + self.wait_api() + try: + return fn(*args, **kwargs) + except GithubException as e: + if not self._is_rate_limited(e) or attempt >= self.max_retries: + raise + delay = self._retry_delay(e, attempt) + logger.warning( + "GitHub rate limit hit (status=%s), sleeping %.1fs (attempt %d/%d)", + e.status, delay, attempt + 1, self.max_retries, + ) + time.sleep(delay) + + @staticmethod + def _is_rate_limited(e): + if e.status == 429: + return True + if e.status == 403: + msg = str(e).lower() + return "rate limit" in msg or "abuse" in msg or "secondary" in msg + return False + + @staticmethod + def _retry_delay(e, attempt): + headers = getattr(e, "headers", {}) or {} + retry_after = headers.get("Retry-After") or headers.get("retry-after") + if retry_after: + try: + return float(retry_after) + except ValueError: + pass + return min(60.0, (2 ** attempt) + random.random()) + + def _run_git(args, cwd=None, quiet=False): cmd = ["git"] + args result = subprocess.run( @@ -111,15 +176,31 @@ def _delete_temp_branch(bare_path, branch_name): pass -def migrate_pull_requests(config, dry_run=False, include_closed=False, only_repos=None): +def migrate_pull_requests( + config, dry_run=False, include_closed=False, closed_only=False, + only_repos=None, throttler=None, +): """Migrate pull requests from Bitbucket to GitHub. Args: config: Config object. dry_run: If True, log what would be done without making changes. include_closed: If True, also migrate merged/declined PRs. + closed_only: If True, migrate ONLY merged/declined PRs (skip open ones). + Implies include_closed. only_repos: Optional set of "PROJECT/SLUG" strings to filter repos. + throttler: Optional Throttler instance. If None, built from config. """ + if closed_only: + include_closed = True + + if throttler is None: + throttler = Throttler( + api_delay=getattr(config, "pr_api_delay", 0.5), + pr_delay=getattr(config, "pr_pr_delay", 3.0), + max_retries=getattr(config, "pr_max_retries", 5), + ) + bb = BitbucketClient(config.bb_base_url, config.bb_token, verify_ssl=config.bb_verify_ssl) gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) state = State(config.work_dir) @@ -144,14 +225,15 @@ def migrate_pull_requests(config, dry_run=False, include_closed=False, only_repo logger.info("Processing PRs for %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) - # Fetch open PRs - try: - prs = bb.list_pull_requests(project_key, repo_slug, state="OPEN") - except Exception: - logger.exception("Failed to list PRs for %s/%s", project_key, repo_slug) - continue + prs = [] + + if not closed_only: + try: + prs = bb.list_pull_requests(project_key, repo_slug, state="OPEN") + except Exception: + logger.exception("Failed to list open PRs for %s/%s", project_key, repo_slug) + continue - # Fetch closed PRs if requested if include_closed: try: merged = bb.list_pull_requests(project_key, repo_slug, state="MERGED") @@ -186,14 +268,15 @@ def migrate_pull_requests(config, dry_run=False, include_closed=False, only_repo if pr_state == "OPEN": _migrate_open_pr( config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr, + gh_org, gh_repo_name, pr, throttler, ) else: _migrate_closed_pr( config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr, + gh_org, gh_repo_name, pr, throttler, ) total_migrated += 1 + throttler.wait_between_prs() except Exception: logger.exception( "Failed to migrate PR #%d in %s/%s", pr_id, project_key, repo_slug @@ -207,7 +290,8 @@ def migrate_pull_requests(config, dry_run=False, include_closed=False, only_repo return total_migrated, total_skipped, total_failed -def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr): +def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, throttler): """Migrate an open pull request.""" pr_id = pr["id"] title = pr["title"] @@ -217,16 +301,20 @@ def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_r logger.info("Migrating open PR #%d: %s (%s -> %s)", pr_id, title, head_branch, base_branch) body = _format_pr_body(pr, config) - gh_pr = gh.create_pull_request( + gh_pr = throttler.call( + gh.create_pull_request, repo_name=gh_repo_name, title=title, body=body, head=head_branch, base=base_branch, org_name=gh_org, ) - _migrate_pr_comments(bb, gh, config, project_key, repo_slug, gh_org, gh_repo_name, pr_id, gh_pr.number) + _migrate_pr_comments(bb, gh, config, project_key, repo_slug, + gh_org, gh_repo_name, pr_id, gh_pr.number, throttler) reviewers = _map_reviewers(pr, config) if reviewers: - gh.add_pr_reviewers(gh_repo_name, gh_pr.number, reviewers, org_name=gh_org) + throttler.call( + gh.add_pr_reviewers, gh_repo_name, gh_pr.number, reviewers, org_name=gh_org, + ) state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) logger.info("Migrated open PR #%d -> GitHub PR #%d", pr_id, gh_pr.number) @@ -272,7 +360,8 @@ def _resolve_head_sha(bb, bare_path, project_key, repo_slug, pr): return None, False -def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr): +def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, throttler): """Migrate a closed (merged/declined) PR by recreating the branch from the commit SHA.""" pr_id = pr["id"] title = pr["title"] @@ -295,13 +384,10 @@ def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh if head_sha: if is_merge_commit: - # The merge/squash commit is already on the target branch. - # To get a meaningful diff, target the PR at the commit's parent. try: parent_sha = _run_git( ["rev-parse", f"{head_sha}^"], cwd=bare_path, quiet=True, ) - # Create a temp base branch at the parent so the PR shows the squash diff pr_base = f"migrated-pr/{pr_id}/base" if _create_temp_branch(bare_path, pr_base, parent_sha): _push_temp_branch(bare_path, pr_base) @@ -315,10 +401,10 @@ def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh _delete_temp_branch(bare_path, temp_branch) if branch_created: - # Create a real PR on GitHub, then close it body = _format_pr_body(pr, config, closed_state=pr_state) try: - gh_pr = gh.create_pull_request( + gh_pr = throttler.call( + gh.create_pull_request, repo_name=gh_repo_name, title=f"[{pr_state}] {title}", body=body, @@ -328,20 +414,19 @@ def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh ) _migrate_pr_comments(bb, gh, config, project_key, repo_slug, - gh_org, gh_repo_name, pr_id, gh_pr.number) + gh_org, gh_repo_name, pr_id, gh_pr.number, throttler) - # Close the PR with a status comment - gh.add_pr_comment( + throttler.call( + gh.add_pr_comment, gh_repo_name, gh_pr.number, f"This PR was **{pr_state.lower()}** on Bitbucket. " f"Migrated for historical reference.", org_name=gh_org, ) - # Close the PR - repo = gh.get_repo(gh_repo_name, org_name=gh_org) - gh_pull = repo.get_pull(gh_pr.number) - gh_pull.edit(state="closed") + repo = throttler.call(gh.get_repo, gh_repo_name, org_name=gh_org) + gh_pull = throttler.call(repo.get_pull, gh_pr.number) + throttler.call(gh_pull.edit, state="closed") state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) logger.info("Migrated %s PR #%d -> GitHub PR #%d (closed)", pr_state, pr_id, gh_pr.number) @@ -350,13 +435,12 @@ def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh except Exception: logger.warning("Could not create PR for %s PR #%d, falling back to issue", pr_state, pr_id) - # Fallback: create as a GitHub Issue _migrate_pr_as_issue(config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr) + gh_org, gh_repo_name, pr, throttler) def _migrate_pr_as_issue(config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr): + gh_org, gh_repo_name, pr, throttler): """Migrate a PR as a GitHub Issue (when branch can't be recreated).""" pr_id = pr["id"] title = pr["title"] @@ -365,32 +449,30 @@ def _migrate_pr_as_issue(config, bb, gh, state, project_key, repo_slug, body = _format_pr_body(pr, config, closed_state=pr_state) body += f"\n\n---\n*Migrated as issue because the source branch could not be recreated.*" - repo = gh.get_repo(gh_repo_name, org_name=gh_org) + repo = throttler.call(gh.get_repo, gh_repo_name, org_name=gh_org) - # Create issue - issue = repo.create_issue( + issue = throttler.call( + repo.create_issue, title=f"[Migrated {pr_state} PR #{pr_id}] {title}", body=body, labels=["migrated-pr", pr_state.lower()], ) - # Migrate comments activities = bb.get_pr_activities(project_key, repo_slug, pr_id) for activity in activities: action = activity.get("action", "") if action == "COMMENTED" and "comment" in activity: comment_body = _format_comment(activity, config) - issue.create_comment(comment_body) + throttler.call(issue.create_comment, comment_body) - # Close the issue - issue.edit(state="closed") + throttler.call(issue.edit, state="closed") state.record_pr_mapping(project_key, repo_slug, pr_id, issue.number) logger.info("Migrated %s PR #%d -> GitHub Issue #%d (closed)", pr_state, pr_id, issue.number) def _migrate_pr_comments(bb, gh, config, project_key, repo_slug, - gh_org, gh_repo_name, pr_id, gh_pr_number): + gh_org, gh_repo_name, pr_id, gh_pr_number, throttler): """Migrate comments from a Bitbucket PR to a GitHub PR.""" activities = bb.get_pr_activities(project_key, repo_slug, pr_id) comment_count = 0 @@ -398,6 +480,8 @@ def _migrate_pr_comments(bb, gh, config, project_key, repo_slug, action = activity.get("action", "") if action == "COMMENTED" and "comment" in activity: comment_body = _format_comment(activity, config) - gh.add_pr_comment(gh_repo_name, gh_pr_number, comment_body, org_name=gh_org) + throttler.call( + gh.add_pr_comment, gh_repo_name, gh_pr_number, comment_body, org_name=gh_org, + ) comment_count += 1 return comment_count diff --git a/tests/test_pr_migrator.py b/tests/test_pr_migrator.py index eae8382..e236da3 100644 --- a/tests/test_pr_migrator.py +++ b/tests/test_pr_migrator.py @@ -7,6 +7,7 @@ from bb2gh.config import Config from bb2gh.pr_migrator import ( migrate_pull_requests, + Throttler, _format_pr_body, _format_comment, _map_reviewers, @@ -14,6 +15,17 @@ ) +@pytest.fixture(autouse=True) +def fast_throttler(monkeypatch): + """Ensure Throttler defaults to zero-delay for all tests.""" + original_init = Throttler.__init__ + + def zero_init(self, api_delay=0.5, pr_delay=3.0, max_retries=5): + original_init(self, api_delay=0, pr_delay=0, max_retries=max_retries) + + monkeypatch.setattr(Throttler, "__init__", zero_init) + + @pytest.fixture def mock_config(): config = MagicMock(spec=Config) @@ -353,3 +365,98 @@ def test_dry_run_includes_closed_prs(self, MockBB, MockGH, MockState, mock_confi assert migrated == 2 assert skipped == 0 MockGH.return_value.create_pull_request.assert_not_called() + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_closed_only_skips_open(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + merged_pr = dict(sample_pr, id=99, state="MERGED", title="Old merged PR") + merged_pr["fromRef"] = dict(sample_pr["fromRef"], latestCommit="aaa") + + bb_instance = MockBB.return_value + calls = [] + def list_prs(proj, repo, state): + calls.append(state) + return {"MERGED": [merged_pr], "DECLINED": []}.get(state, []) + bb_instance.list_pull_requests.side_effect = list_prs + + migrated, _, _ = migrate_pull_requests( + mock_config, dry_run=True, closed_only=True, + ) + + # Only one PR (the merged one), OPEN was never queried + assert migrated == 1 + assert "OPEN" not in calls + assert "MERGED" in calls + assert "DECLINED" in calls + + +class TestThrottler: + def test_call_retries_on_rate_limit(self, monkeypatch): + from bb2gh.pr_migrator import Throttler + import github + + throttler = Throttler(api_delay=0, pr_delay=0, max_retries=3) + + # Fake rate-limit exception on first two calls, success on third + attempts = {"n": 0} + def flaky(): + attempts["n"] += 1 + if attempts["n"] < 3: + exc = github.GithubException(429, {"message": "too fast"}, {"Retry-After": "0"}) + raise exc + return "ok" + + result = throttler.call(flaky) + assert result == "ok" + assert attempts["n"] == 3 + + def test_call_raises_after_max_retries(self): + from bb2gh.pr_migrator import Throttler + import github + + throttler = Throttler(api_delay=0, pr_delay=0, max_retries=2) + + def always_fail(): + raise github.GithubException(429, {"message": "no"}, {"Retry-After": "0"}) + + with pytest.raises(github.GithubException): + throttler.call(always_fail) + + def test_call_reraises_non_rate_limit(self): + from bb2gh.pr_migrator import Throttler + import github + + throttler = Throttler(api_delay=0, pr_delay=0, max_retries=3) + + def not_found(): + raise github.GithubException(404, {"message": "gone"}, {}) + + with pytest.raises(github.GithubException) as exc_info: + throttler.call(not_found) + assert exc_info.value.status == 404 + + def test_wait_api_spacing(self, monkeypatch): + from bb2gh.pr_migrator import Throttler + + sleeps = [] + monkeypatch.setattr("bb2gh.pr_migrator.time.sleep", lambda s: sleeps.append(s)) + + # First call returns 100.1 (elapsed check), second returns 100.5 (record) + times = iter([100.1, 100.5]) + monkeypatch.setattr( + "bb2gh.pr_migrator.time.monotonic", lambda: next(times), + ) + + throttler = Throttler(api_delay=0.5, pr_delay=0, max_retries=0) + throttler.api_delay = 0.5 # override autouse zeroing + throttler._last_call = 100.0 # simulate a previous call at t=100.0 + throttler.wait_api() + + # elapsed = 100.1 - 100.0 = 0.1, so remaining = 0.5 - 0.1 = 0.4 + assert sleeps and abs(sleeps[0] - 0.4) < 0.01 From b3d5c2ebb5b6751f20df4ef3eda45a9b0cec8140 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 20:30:00 +0000 Subject: [PATCH 74/74] Migrate all closed PRs as closed GitHub Issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simpler and more useful than the branch-recreation approach: every closed PR (merged or declined) becomes a closed GitHub Issue with all its comments preserved. Labeled `migrated-pr` + `merged`/`declined` for filtering. Everything is searchable in one place — the issue tracker — instead of split between PRs and issues. Removes: - Branch recreation code (git subprocess, temp branches, push logic) - SHA resolution cascade (_resolve_head_sha, get_merge_commit) - BitbucketClient.get_pull_request, get_merge_commit - All dependency on the bare clone for closed PR migration Also fixes chronological comment ordering — comments were sorted by activity.createdDate but Bitbucket puts it on comment.createdDate. https://claude.ai/code/session_01HK99UAxTgAiWTNYAfSoVvP --- bb2gh/bitbucket_client.py | 26 ---- bb2gh/cli.py | 13 +- bb2gh/pr_migrator.py | 273 ++++++++------------------------------ tests/test_pr_migrator.py | 190 ++++++++++++++++++-------- 4 files changed, 202 insertions(+), 300 deletions(-) diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py index 91ede78..e8e9987 100644 --- a/bb2gh/bitbucket_client.py +++ b/bb2gh/bitbucket_client.py @@ -88,32 +88,6 @@ def list_pull_requests(self, project_key, repo_slug, state="OPEN"): url = f"{self.api_url}/projects/{project_key}/repos/{repo_slug}/pull-requests" return list(self._paginate(url, params={"state": state})) - def get_pull_request(self, project_key, repo_slug, pr_id): - """Get a single pull request with full details (including merge properties).""" - url = ( - f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" - f"/pull-requests/{pr_id}" - ) - resp = self.session.get(url) - resp.raise_for_status() - return resp.json() - - def get_merge_commit(self, project_key, repo_slug, pr_id): - """Extract the merge/squash commit SHA for a merged PR. - - Bitbucket Server stores this in properties.mergeCommit on the PR object. - Returns the SHA string, or None if not available. - """ - try: - pr = self.get_pull_request(project_key, repo_slug, pr_id) - merge_commit = pr.get("properties", {}).get("mergeCommit", {}) - sha = merge_commit.get("id") or merge_commit.get("displayId") - if sha: - return sha - except Exception: - logger.debug("Could not fetch merge commit from PR properties for PR #%d", pr_id) - return None - def get_pr_activities(self, project_key, repo_slug, pr_id): """Get activities (comments, approvals, etc.) for a pull request.""" url = ( diff --git a/bb2gh/cli.py b/bb2gh/cli.py index 2c477fd..d970b70 100644 --- a/bb2gh/cli.py +++ b/bb2gh/cli.py @@ -78,12 +78,13 @@ def sync(ctx): def migrate_prs(ctx, repo, include_closed, closed_only, api_delay, pr_delay, dry_run): """Migrate pull requests from Bitbucket to GitHub. - Creates matching PRs on GitHub with title, description, comments, - and reviewer assignments. With --include-closed, also migrates - merged/declined PRs by recreating branches from commit SHAs - (falls back to GitHub Issues if the commit no longer exists). - With --closed-only, skips open PRs entirely (useful if you've - already migrated the open ones). + Open PRs become GitHub PRs (with title, description, comments, and + reviewers). Closed PRs (MERGED/DECLINED) become closed GitHub Issues + with all their comments, labeled `migrated-pr` + `merged`/`declined`, + so everything is searchable in one place. + + --include-closed migrates open PRs plus closed ones as issues. + --closed-only skips open PRs entirely (use after migrating open ones). Rate limiting: --api-delay controls minimum spacing between GitHub API calls; --pr-delay adds a pause between full PR migrations. diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py index ae78091..76cdda9 100644 --- a/bb2gh/pr_migrator.py +++ b/bb2gh/pr_migrator.py @@ -1,9 +1,13 @@ -"""Migrate pull requests from Bitbucket Server to GitHub Enterprise.""" +"""Migrate pull requests from Bitbucket Server to GitHub Enterprise. + +Open PRs are migrated as GitHub PRs (they have a live branch). +Closed PRs (MERGED, DECLINED) are migrated as closed GitHub Issues with +all their comments preserved. Everything lives in one place — the issue +tracker — so PR history is searchable alongside code review discussions. +""" import logging -import os import random -import subprocess import time from github import GithubException @@ -25,7 +29,6 @@ def __init__(self, api_delay=0.5, pr_delay=3.0, max_retries=5): self._last_call = 0.0 def wait_api(self): - """Sleep to enforce minimum spacing between API calls.""" if self.api_delay <= 0: return elapsed = time.monotonic() - self._last_call @@ -35,12 +38,10 @@ def wait_api(self): self._last_call = time.monotonic() def wait_between_prs(self): - """Sleep between full PR migrations.""" if self.pr_delay > 0: time.sleep(self.pr_delay) def call(self, fn, *args, **kwargs): - """Invoke fn(*args, **kwargs) with rate limiting and retry on 429/403.""" for attempt in range(self.max_retries + 1): self.wait_api() try: @@ -76,34 +77,23 @@ def _retry_delay(e, attempt): return min(60.0, (2 ** attempt) + random.random()) -def _run_git(args, cwd=None, quiet=False): - cmd = ["git"] + args - result = subprocess.run( - cmd, cwd=cwd, capture_output=True, text=True, check=False - ) - if result.returncode != 0: - if not quiet: - logger.error("git %s failed: %s", args[0], result.stderr.strip()) - raise subprocess.CalledProcessError( - result.returncode, cmd, result.stdout, result.stderr - ) - return result.stdout.strip() - - def _format_pr_body(pr, config, closed_state=None): - """Format the GitHub PR body with migration metadata.""" + """Format the GitHub PR/issue body with migration metadata.""" bb_url = config.bb_base_url project = pr["toRef"]["repository"]["project"]["key"] repo = pr["toRef"]["repository"]["slug"] pr_id = pr["id"] author = pr["author"]["user"].get("displayName", pr["author"]["user"].get("name", "Unknown")) created = pr.get("createdDate", "") + head_branch = pr["fromRef"]["displayId"] + base_branch = pr["toRef"]["displayId"] header = ( f"> **Migrated from Bitbucket**\n" f"> Source: [{project}/{repo} PR #{pr_id}]" f"({bb_url}/projects/{project}/repos/{repo}/pull-requests/{pr_id})\n" f"> Original author: **{author}**\n" + f"> Branch: `{head_branch}` → `{base_branch}`\n" ) if created: header += f"> Created: {created}\n" @@ -115,7 +105,7 @@ def _format_pr_body(pr, config, closed_state=None): def _format_comment(activity, config): - """Format a Bitbucket comment as a GitHub PR comment.""" + """Format a Bitbucket comment for GitHub.""" comment = activity.get("comment", {}) user = comment.get("author", {}) display_name = user.get("displayName", user.get("name", "Unknown")) @@ -141,39 +131,11 @@ def _map_reviewers(pr, config): return reviewers -def _commit_exists(bare_path, sha): - """Check if a commit SHA exists in the bare repo.""" - try: - _run_git(["cat-file", "-t", sha], cwd=bare_path, quiet=True) - return True - except subprocess.CalledProcessError: - return False - - -def _create_temp_branch(bare_path, branch_name, sha): - """Create a branch pointing to a specific commit in the bare repo.""" - try: - _run_git(["branch", branch_name, sha], cwd=bare_path, quiet=True) - return True - except subprocess.CalledProcessError: - return False - - -def _push_temp_branch(bare_path, branch_name): - """Push a branch to the github remote.""" - try: - _run_git(["push", "github", f"{branch_name}:{branch_name}"], cwd=bare_path, quiet=True) - return True - except subprocess.CalledProcessError: - return False - - -def _delete_temp_branch(bare_path, branch_name): - """Delete a local branch from the bare repo.""" - try: - _run_git(["branch", "-D", branch_name], cwd=bare_path, quiet=True) - except subprocess.CalledProcessError: - pass +def _iter_comment_activities(activities): + """Yield COMMENTED activities in chronological order (oldest first).""" + comments = [a for a in activities if a.get("action") == "COMMENTED" and "comment" in a] + comments.sort(key=lambda a: a.get("comment", {}).get("createdDate") or a.get("createdDate", 0)) + return comments def migrate_pull_requests( @@ -185,11 +147,10 @@ def migrate_pull_requests( Args: config: Config object. dry_run: If True, log what would be done without making changes. - include_closed: If True, also migrate merged/declined PRs. - closed_only: If True, migrate ONLY merged/declined PRs (skip open ones). - Implies include_closed. + include_closed: If True, also migrate merged/declined PRs (as issues). + closed_only: If True, migrate ONLY merged/declined PRs. Implies include_closed. only_repos: Optional set of "PROJECT/SLUG" strings to filter repos. - throttler: Optional Throttler instance. If None, built from config. + throttler: Optional Throttler instance. Defaults are read from config. """ if closed_only: include_closed = True @@ -253,13 +214,11 @@ def migrate_pull_requests( total_skipped += 1 continue - head_branch = pr["fromRef"]["displayId"] - base_branch = pr["toRef"]["displayId"] - if dry_run: + target = "PR" if pr_state == "OPEN" else "closed Issue" logger.info( - "[DRY RUN] Would migrate PR #%d [%s]: %s (%s -> %s) to %s/%s", - pr_id, pr_state, title, head_branch, base_branch, gh_org, gh_repo_name, + "[DRY RUN] Would migrate PR #%d [%s] as %s: %s -> %s/%s", + pr_id, pr_state, target, title, gh_org, gh_repo_name, ) total_migrated += 1 continue @@ -271,7 +230,7 @@ def migrate_pull_requests( gh_org, gh_repo_name, pr, throttler, ) else: - _migrate_closed_pr( + _migrate_closed_pr_as_issue( config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr, throttler, ) @@ -292,7 +251,7 @@ def migrate_pull_requests( def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, gh_org, gh_repo_name, pr, throttler): - """Migrate an open pull request.""" + """Migrate an open PR as a GitHub PR.""" pr_id = pr["id"] title = pr["title"] head_branch = pr["fromRef"]["displayId"] @@ -307,8 +266,12 @@ def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, head=head_branch, base=base_branch, org_name=gh_org, ) - _migrate_pr_comments(bb, gh, config, project_key, repo_slug, - gh_org, gh_repo_name, pr_id, gh_pr.number, throttler) + activities = bb.get_pr_activities(project_key, repo_slug, pr_id) + for activity in _iter_comment_activities(activities): + comment_body = _format_comment(activity, config) + throttler.call( + gh.add_pr_comment, gh_repo_name, gh_pr.number, comment_body, org_name=gh_org, + ) reviewers = _map_reviewers(pr, config) if reviewers: @@ -320,168 +283,50 @@ def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, logger.info("Migrated open PR #%d -> GitHub PR #%d", pr_id, gh_pr.number) -def _resolve_head_sha(bb, bare_path, project_key, repo_slug, pr): - """Find a usable commit SHA for recreating a closed PR's head branch. - - Tries in order: - 1. fromRef.latestCommit — the original branch tip (works for regular merges) - 2. merge/squash commit from PR properties — guaranteed to exist for merged PRs - 3. merge commit from PR activities — alternative source for the same info +def _migrate_closed_pr_as_issue(config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, throttler): + """Migrate a closed (merged/declined) PR as a closed GitHub Issue. - Returns (sha, is_merge_commit) or (None, False). + Everything lives in one place — the issue tracker — so PR history is + searchable alongside other issues. No branch recreation, no repo access + needed, works even when the source branch has been GC'd. """ pr_id = pr["id"] - head_sha = pr["fromRef"].get("latestCommit", "") - - # 1. Original source branch tip - if head_sha and _commit_exists(bare_path, head_sha): - logger.debug("PR #%d: using fromRef.latestCommit %s", pr_id, head_sha[:12]) - return head_sha, False - - # 2. Merge/squash commit from PR properties - merge_sha = bb.get_merge_commit(project_key, repo_slug, pr_id) - if merge_sha and _commit_exists(bare_path, merge_sha): - logger.debug("PR #%d: using merge commit %s from properties", pr_id, merge_sha[:12]) - return merge_sha, True - - # 3. Merge commit from activities - try: - activities = bb.get_pr_activities(project_key, repo_slug, pr_id) - for activity in activities: - if activity.get("action") == "MERGED": - commit = activity.get("commit", {}) - act_sha = commit.get("id") or commit.get("displayId") - if act_sha and _commit_exists(bare_path, act_sha): - logger.debug("PR #%d: using merge commit %s from activity", pr_id, act_sha[:12]) - return act_sha, True - except Exception: - logger.debug("PR #%d: could not fetch activities for merge commit", pr_id) - - return None, False - - -def _migrate_closed_pr(config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr, throttler): - """Migrate a closed (merged/declined) PR by recreating the branch from the commit SHA.""" - pr_id = pr["id"] title = pr["title"] pr_state = pr.get("state", "UNKNOWN") - head_branch = pr["fromRef"]["displayId"] - base_branch = pr["toRef"]["displayId"] - logger.info("Migrating %s PR #%d: %s (%s -> %s)", pr_state, pr_id, title, head_branch, base_branch) + logger.info("Migrating %s PR #%d as issue: %s", pr_state, pr_id, title) - bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + body = _format_pr_body(pr, config, closed_state=pr_state) - temp_branch = f"migrated-pr/{pr_id}/{head_branch}" - branch_created = False - pr_base = base_branch + repo = throttler.call(gh.get_repo, gh_repo_name, org_name=gh_org) - if os.path.exists(bare_path): - head_sha, is_merge_commit = _resolve_head_sha( - bb, bare_path, project_key, repo_slug, pr, + labels = ["migrated-pr", pr_state.lower()] + try: + issue = throttler.call( + repo.create_issue, + title=f"[{pr_state} PR #{pr_id}] {title}", + body=body, + labels=labels, ) - - if head_sha: - if is_merge_commit: - try: - parent_sha = _run_git( - ["rev-parse", f"{head_sha}^"], cwd=bare_path, quiet=True, - ) - pr_base = f"migrated-pr/{pr_id}/base" - if _create_temp_branch(bare_path, pr_base, parent_sha): - _push_temp_branch(bare_path, pr_base) - _delete_temp_branch(bare_path, pr_base) - except subprocess.CalledProcessError: - logger.debug("PR #%d: could not resolve parent of merge commit", pr_id) - - if _create_temp_branch(bare_path, temp_branch, head_sha): - if _push_temp_branch(bare_path, temp_branch): - branch_created = True - _delete_temp_branch(bare_path, temp_branch) - - if branch_created: - body = _format_pr_body(pr, config, closed_state=pr_state) - try: - gh_pr = throttler.call( - gh.create_pull_request, - repo_name=gh_repo_name, - title=f"[{pr_state}] {title}", + except GithubException as e: + # Labels don't exist yet — create without labels + if e.status in (404, 422): + logger.debug("Labels not found on %s, creating issue without labels", gh_repo_name) + issue = throttler.call( + repo.create_issue, + title=f"[{pr_state} PR #{pr_id}] {title}", body=body, - head=temp_branch, - base=base_branch, - org_name=gh_org, ) - - _migrate_pr_comments(bb, gh, config, project_key, repo_slug, - gh_org, gh_repo_name, pr_id, gh_pr.number, throttler) - - throttler.call( - gh.add_pr_comment, - gh_repo_name, gh_pr.number, - f"This PR was **{pr_state.lower()}** on Bitbucket. " - f"Migrated for historical reference.", - org_name=gh_org, - ) - - repo = throttler.call(gh.get_repo, gh_repo_name, org_name=gh_org) - gh_pull = throttler.call(repo.get_pull, gh_pr.number) - throttler.call(gh_pull.edit, state="closed") - - state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) - logger.info("Migrated %s PR #%d -> GitHub PR #%d (closed)", pr_state, pr_id, gh_pr.number) - return - - except Exception: - logger.warning("Could not create PR for %s PR #%d, falling back to issue", pr_state, pr_id) - - _migrate_pr_as_issue(config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr, throttler) - - -def _migrate_pr_as_issue(config, bb, gh, state, project_key, repo_slug, - gh_org, gh_repo_name, pr, throttler): - """Migrate a PR as a GitHub Issue (when branch can't be recreated).""" - pr_id = pr["id"] - title = pr["title"] - pr_state = pr.get("state", "UNKNOWN") - - body = _format_pr_body(pr, config, closed_state=pr_state) - body += f"\n\n---\n*Migrated as issue because the source branch could not be recreated.*" - - repo = throttler.call(gh.get_repo, gh_repo_name, org_name=gh_org) - - issue = throttler.call( - repo.create_issue, - title=f"[Migrated {pr_state} PR #{pr_id}] {title}", - body=body, - labels=["migrated-pr", pr_state.lower()], - ) + else: + raise activities = bb.get_pr_activities(project_key, repo_slug, pr_id) - for activity in activities: - action = activity.get("action", "") - if action == "COMMENTED" and "comment" in activity: - comment_body = _format_comment(activity, config) - throttler.call(issue.create_comment, comment_body) + for activity in _iter_comment_activities(activities): + comment_body = _format_comment(activity, config) + throttler.call(issue.create_comment, comment_body) throttler.call(issue.edit, state="closed") state.record_pr_mapping(project_key, repo_slug, pr_id, issue.number) logger.info("Migrated %s PR #%d -> GitHub Issue #%d (closed)", pr_state, pr_id, issue.number) - - -def _migrate_pr_comments(bb, gh, config, project_key, repo_slug, - gh_org, gh_repo_name, pr_id, gh_pr_number, throttler): - """Migrate comments from a Bitbucket PR to a GitHub PR.""" - activities = bb.get_pr_activities(project_key, repo_slug, pr_id) - comment_count = 0 - for activity in activities: - action = activity.get("action", "") - if action == "COMMENTED" and "comment" in activity: - comment_body = _format_comment(activity, config) - throttler.call( - gh.add_pr_comment, gh_repo_name, gh_pr_number, comment_body, org_name=gh_org, - ) - comment_count += 1 - return comment_count diff --git a/tests/test_pr_migrator.py b/tests/test_pr_migrator.py index e236da3..393b982 100644 --- a/tests/test_pr_migrator.py +++ b/tests/test_pr_migrator.py @@ -11,7 +11,7 @@ _format_pr_body, _format_comment, _map_reviewers, - _resolve_head_sha, + _iter_comment_activities, ) @@ -254,76 +254,158 @@ def test_fallback_to_config_resolve(self, MockBB, MockGH, MockState, mock_config assert call_kwargs[1]["repo_name"] == "fallback-repo" -class TestResolveHeadSha: - """Tests for _resolve_head_sha — the SHA resolution cascade for closed PRs.""" +class TestIterCommentActivities: + def test_filters_and_sorts_chronologically(self): + activities = [ + {"action": "COMMENTED", "comment": {"text": "second", "createdDate": 200}}, + {"action": "APPROVED"}, + {"action": "COMMENTED", "comment": {"text": "first", "createdDate": 100}}, + {"action": "MERGED"}, + {"action": "COMMENTED", "comment": {"text": "third", "createdDate": 300}}, + ] - def _make_pr(self, head_sha="abc123"): - return { - "id": 10, - "fromRef": {"displayId": "feature/x", "latestCommit": head_sha}, - "toRef": {"displayId": "main"}, - } + result = list(_iter_comment_activities(activities)) - @patch("bb2gh.pr_migrator._commit_exists") - def test_uses_from_ref_when_commit_exists(self, mock_exists): - mock_exists.return_value = True - bb = MagicMock() + assert len(result) == 3 + assert result[0]["comment"]["text"] == "first" + assert result[1]["comment"]["text"] == "second" + assert result[2]["comment"]["text"] == "third" - sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("deadbeef")) + def test_returns_empty_when_no_comments(self): + activities = [{"action": "APPROVED"}, {"action": "MERGED"}] + assert list(_iter_comment_activities(activities)) == [] - assert sha == "deadbeef" - assert is_merge is False - bb.get_merge_commit.assert_not_called() - @patch("bb2gh.pr_migrator._commit_exists") - def test_falls_back_to_merge_commit_property(self, mock_exists): - mock_exists.side_effect = lambda path, sha: sha == "squash111" - bb = MagicMock() - bb.get_merge_commit.return_value = "squash111" +class TestMigrateClosedAsIssue: + """Closed PRs are migrated as closed GitHub Issues (no branch recreation).""" - sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("gone")) + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_closed_pr_becomes_closed_issue(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False - assert sha == "squash111" - assert is_merge is True + merged_pr = dict(sample_pr, id=99, state="MERGED", title="Old merged PR") - @patch("bb2gh.pr_migrator._commit_exists") - def test_falls_back_to_merge_activity(self, mock_exists): - mock_exists.side_effect = lambda path, sha: sha == "act222" - bb = MagicMock() - bb.get_merge_commit.return_value = None - bb.get_pr_activities.return_value = [ - {"action": "COMMENTED", "comment": {"text": "hi"}}, - {"action": "MERGED", "commit": {"id": "act222", "displayId": "act222"}}, + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "MERGED": [merged_pr], "DECLINED": [], + }.get(state, []) + bb_instance.get_pr_activities.return_value = [ + { + "action": "COMMENTED", + "comment": { + "author": {"name": "reviewer", "displayName": "Reviewer"}, + "text": "Great work", + "createdDate": 1711234567000, + }, + }, ] - sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("gone")) + mock_issue = MagicMock() + mock_issue.number = 42 + mock_repo = MagicMock() + mock_repo.create_issue.return_value = mock_issue + MockGH.return_value.get_repo.return_value = mock_repo - assert sha == "act222" - assert is_merge is True + migrated, _, failed = migrate_pull_requests( + mock_config, dry_run=False, closed_only=True, + ) - @patch("bb2gh.pr_migrator._commit_exists") - def test_returns_none_when_nothing_found(self, mock_exists): - mock_exists.return_value = False - bb = MagicMock() - bb.get_merge_commit.return_value = None - bb.get_pr_activities.return_value = [] + assert migrated == 1 + assert failed == 0 - sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("gone")) + # No PR created — it's an issue + MockGH.return_value.create_pull_request.assert_not_called() - assert sha is None - assert is_merge is False + # Issue was created with the right title prefix + mock_repo.create_issue.assert_called_once() + issue_kwargs = mock_repo.create_issue.call_args.kwargs + assert "MERGED PR #99" in issue_kwargs["title"] + assert "Old merged PR" in issue_kwargs["title"] + assert "migrated-pr" in issue_kwargs["labels"] + assert "merged" in issue_kwargs["labels"] - @patch("bb2gh.pr_migrator._commit_exists") - def test_handles_empty_latest_commit(self, mock_exists): - mock_exists.return_value = False - bb = MagicMock() - bb.get_merge_commit.return_value = None - bb.get_pr_activities.return_value = [] + # Comment was added on the issue + mock_issue.create_comment.assert_called_once() - sha, is_merge = _resolve_head_sha(bb, "/repo.git", "PROJ", "repo", self._make_pr("")) + # Issue was closed + mock_issue.edit.assert_called_once_with(state="closed") - assert sha is None - assert is_merge is False + # State recorded the mapping + state_instance.record_pr_mapping.assert_called_once_with( + "PROJ", "my-repo", 99, 42, + ) + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_declined_pr_labeled_declined(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + declined_pr = dict(sample_pr, id=7, state="DECLINED", title="Won't fix") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "MERGED": [], "DECLINED": [declined_pr], + }.get(state, []) + bb_instance.get_pr_activities.return_value = [] + + mock_issue = MagicMock() + mock_issue.number = 8 + mock_repo = MagicMock() + mock_repo.create_issue.return_value = mock_issue + MockGH.return_value.get_repo.return_value = mock_repo + + migrate_pull_requests(mock_config, closed_only=True) + + issue_kwargs = mock_repo.create_issue.call_args.kwargs + assert "declined" in issue_kwargs["labels"] + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_falls_back_when_labels_missing(self, MockBB, MockGH, MockState, mock_config, sample_pr): + """If the labels don't exist on the repo yet, create the issue anyway.""" + from github import GithubException + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + merged_pr = dict(sample_pr, id=11, state="MERGED", title="Merged x") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "MERGED": [merged_pr], "DECLINED": [], + }.get(state, []) + bb_instance.get_pr_activities.return_value = [] + + mock_issue = MagicMock() + mock_issue.number = 12 + mock_repo = MagicMock() + # First call (with labels) fails 422; second call (no labels) succeeds + mock_repo.create_issue.side_effect = [ + GithubException(422, {"message": "label not found"}, {}), + mock_issue, + ] + MockGH.return_value.get_repo.return_value = mock_repo + + migrated, _, failed = migrate_pull_requests(mock_config, closed_only=True) + + assert migrated == 1 + assert failed == 0 + assert mock_repo.create_issue.call_count == 2 + # Second call had no labels kwarg + second_call = mock_repo.create_issue.call_args_list[1] + assert "labels" not in second_call.kwargs class TestFormatPrBodyClosed: