diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..043ae1b --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.pytest_cache/ +config.yaml +state.json +*.egg diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..84501e3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + openssh-client \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +RUN pip install --no-cache-dir -e . + +# SSH config for non-interactive git clone +RUN mkdir -p /root/.ssh && \ + echo "StrictHostKeyChecking no" >> /root/.ssh/config + +ENTRYPOINT ["bb2gh"] +CMD ["--help"] diff --git a/README.md b/README.md index e4afa9e..47ae7f1 100644 --- a/README.md +++ b/README.md @@ -1 +1,394 @@ -# bitbucket--github \ No newline at end of file +# bb2gh — Bitbucket Server to GitHub Enterprise Migration + +A Python CLI tool for migrating repositories and pull requests from self-hosted Bitbucket Server (Data Center) to GitHub Enterprise, with continuous sync support for a smooth transition. + +## Features + +- **Bulk migration** — Clone all repos from Bitbucket and push to GitHub with full history, branches, and tags +- **Continuous sync** — Keep repos in lockstep by fetching from Bitbucket and pushing to GitHub on a configurable interval +- **PR migration** — Recreate open Bitbucket PRs on GitHub with title, description, comments, and reviewers +- **Project-to-org mapping** — Route Bitbucket projects to different GitHub orgs with flexible repo naming +- **Idempotent** — Safe to re-run; skips already-migrated repos and PRs +- **Docker support** — Run as a one-shot command or a long-lived sync service + +## Prerequisites + +| Requirement | Details | +|---|---| +| **Python** | 3.9+ | +| **Git** | Installed and on `PATH` | +| **Bitbucket Server** | A service account with **Project READ** access on each project you want to migrate | +| **GitHub Enterprise** | A personal access token with `repo` + `admin:org` scopes | +| **SSH key** | The machine running bb2gh needs SSH access to Bitbucket for `git clone` | + +### Bitbucket Server Access + +You do **not** need admin access to Bitbucket. Request a service account from your Bitbucket admin with: + +- **Project READ** on every project you want to migrate + +That single permission covers: +- Cloning repos via SSH +- Listing repos via REST API +- Reading pull requests, comments, and reviewer info + +### GitHub Enterprise Access + +You need a personal access token (PAT) with these scopes: + +- `repo` — full control of private repositories +- `admin:org` — needed to create repos in organizations + +If you are migrating to multiple GitHub orgs, the token must have access to all of them. + +--- + +## Getting Started + +### Step 1: Clone this repo + +```bash +git clone +cd bitbucket--github +``` + +### Step 2: Install dependencies + +```bash +pip install -r requirements.txt +pip install -e . +``` + +Verify the install: + +```bash +bb2gh --help +``` + +### Step 3: Set up SSH access to Bitbucket + +The tool clones repos from Bitbucket via SSH. Make sure the machine running bb2gh can reach your Bitbucket Server over SSH: + +```bash +# Test connectivity (use your actual Bitbucket SSH host and port) +ssh -T git@bitbucket.mycompany.com -p 7999 +``` + +If you are using a non-default SSH key, configure it in `~/.ssh/config`: + +``` +Host bitbucket.mycompany.com + IdentityFile ~/.ssh/bb_migration_key + Port 7999 +``` + +### Step 4: Create your config file + +```bash +cp config.yaml.example config.yaml +``` + +Edit `config.yaml` with your actual values. At minimum, fill in: + +```yaml +bitbucket: + base_url: "https://bitbucket.mycompany.com" + token: "YOUR_BITBUCKET_TOKEN" + ssh_url: "ssh://git@bitbucket.mycompany.com:7999" + # Optional: limit to specific projects (omit to migrate all accessible projects) + projects: + - PROJ1 + - PROJ2 + +github: + base_url: "https://github.mycompany.com/api/v3" + token: "YOUR_GITHUB_TOKEN" + org: "my-org" +``` + +Alternatively, set tokens via environment variables instead of putting them in the file: + +```bash +export BB_TOKEN="your-bitbucket-token" +export GH_TOKEN="your-github-token" +``` + +### Step 5: Configure project-to-org mapping (optional) + +If your Bitbucket projects should land in different GitHub organizations, or you want to control repo naming, add the `repo_mapping` section: + +```yaml +repo_mapping: + # Default naming template for GitHub repos + # Available variables: {project}, {project_lower}, {slug} + name_template: "{project_lower}-{slug}" + + # Per-project overrides + projects: + INFRA: + github_org: "infra-team" # INFRA repos → infra-team org + repos: + legacy-monolith: + github_name: "infra-monolith" # Explicit rename for one repo + PLATFORM: + github_org: "platform-eng" # PLATFORM repos → platform-eng org +``` + +**How mapping resolution works:** + +| What | Resolution order | +|---|---| +| **GitHub org** | Per-project `github_org` → default `github.org` | +| **Repo name** | Per-repo `github_name` → per-project `name_template` → global `name_template` → `{slug}` | + +**Examples** (given the config above): + +| Bitbucket | GitHub | +|---|---| +| `INFRA/my-service` | `infra-team/infra-my-service` | +| `INFRA/legacy-monolith` | `infra-team/infra-monolith` (explicit override) | +| `PLATFORM/api-gateway` | `platform-eng/platform-api-gateway` | +| `OTHER/some-tool` | `my-org/other-some-tool` (uses defaults) | + +If you omit `repo_mapping` entirely, all repos go to `github.org` with their original Bitbucket slug as the name. + +#### Migrate only a subset of repos from a project + +If you only want specific repos from a project, use `include_repos` (allowlist) or `exclude_repos` (denylist): + +```yaml +repo_mapping: + projects: + INFRA: + github_org: "infra-team" + # Only these repos from INFRA are migrated; everything else is skipped + include_repos: + - my-service + - my-api + PLATFORM: + github_org: "platform-eng" + # Migrate all repos EXCEPT these + exclude_repos: + - deprecated-tool + - archived-spike +``` + +**Resolution:** +- If `include_repos` is set, only those repos migrate (acts as an allowlist). +- Otherwise, `exclude_repos` skips the listed repos. +- If both are set, `include_repos` wins and `exclude_repos` is ignored. +- Projects with no filter migrate all repos (the default). + +### Step 6: Configure user mapping (optional) + +Map Bitbucket usernames to GitHub usernames for PR reviewer assignments and author attribution: + +```yaml +user_mapping: + bb_jsmith: "gh-john-smith" + bb_jdoe: "gh-jane-doe" +``` + +Unmapped users fall through with their Bitbucket username as-is. + +### Step 7: Run the migration + +Run these commands in order: + +```bash +# 1. Bulk migrate all repos (creates GitHub repos, clones, pushes) +bb2gh --config config.yaml migrate + +# 2. Start continuous sync (keeps repos in lockstep during transition) +bb2gh --config config.yaml sync + +# 3. In a separate terminal, migrate open PRs +# Always dry-run first to review what will be created: +bb2gh --config config.yaml migrate-prs --dry-run +bb2gh --config config.yaml migrate-prs +``` + +Use `-v` for debug logging on any command: + +```bash +bb2gh --config config.yaml -v migrate +``` + +### Step 8: Cut over + +Once your team is ready to switch to GitHub: + +1. Stop the sync process (`Ctrl+C` or `docker compose down`) +2. Set Bitbucket repos to read-only (ask your Bitbucket admin) +3. Update CI/CD pipelines to point to GitHub +4. Notify your team to use GitHub going forward + +--- + +## Docker Usage + +### Build + +```bash +docker compose build +``` + +### Run bulk migration + +```bash +mkdir -p config && cp config.yaml config/ + +docker compose --profile migrate run migrate +``` + +### Run continuous sync as a background service + +```bash +docker compose up -d sync +``` + +Check logs: + +```bash +docker compose logs -f sync +``` + +### Run PR migration + +```bash +docker compose --profile migrate-prs run migrate-prs +``` + +### Environment variables + +Pass tokens via environment instead of the config file: + +```bash +BB_TOKEN=xxx GH_TOKEN=yyy docker compose up -d sync +``` + +### SSH keys + +By default, Docker Compose mounts `~/.ssh` into the container. Override with: + +```bash +SSH_KEY_PATH=/path/to/keys docker compose --profile migrate run migrate +``` + +--- + +## CLI Reference + +``` +bb2gh [OPTIONS] COMMAND + +Options: + --config PATH Path to config file (default: config.yaml) + -v, --verbose Enable debug logging + +Commands: + migrate Bulk migrate repos from Bitbucket to GitHub + sync Continuously sync repos (runs until interrupted) + migrate-prs Migrate open pull requests from Bitbucket to GitHub +``` + +### migrate + +Clones each Bitbucket repo as a bare mirror and pushes to GitHub. Creates the target GitHub repo if it does not exist. Skips repos that have already been migrated (tracked in `state.json`). + +### sync + +Runs a loop that fetches from Bitbucket (`origin`) and pushes to GitHub (`github` remote) for every migrated repo. The interval is configured via `sync.interval_seconds` (default: 60s). Handles `SIGTERM`/`SIGINT` for graceful shutdown. + +### migrate-prs + +Reads open pull requests from Bitbucket and creates matching PRs on GitHub. Each migrated PR includes: +- A metadata header with the original author, creation date, and a link back to the Bitbucket PR +- All general comments (attributed to the original commenter) +- Reviewer assignments (mapped via `user_mapping`) + +Use `--dry-run` to preview without creating anything. + +--- + +## How It Works + +``` +Bitbucket Server GitHub Enterprise +┌──────────────┐ bb2gh migrate ┌──────────────┐ +│ PROJ/repo-a ├──── git clone ──────►│ org/repo-a │ +│ PROJ/repo-b ├──── --bare ────────►│ org/repo-b │ +│ INFRA/svc ├──── + push mirror ──►│ infra/svc │ +└──────┬───────┘ └──────▲───────┘ + │ bb2gh sync │ + └──── fetch origin ── push github ─────┘ + (every 60s) +``` + +1. **`migrate`** — For each Bitbucket repo: resolves the target GitHub org/name from the mapping config, creates the GitHub repo, bare-clones via SSH, cleans hidden refs (`refs/pull/*`), and pushes `--mirror`. + +2. **`sync`** — Loops on a configurable interval: `git fetch origin --prune` then `git push github --mirror` for each migrated repo. The `--mirror` push ensures GitHub is an exact replica (all branches, tags, force-pushes). During the transition, Bitbucket is the source of truth. + +3. **`migrate-prs`** — For each open PR: looks up the correct GitHub org/repo from state, creates a GitHub PR with metadata header, migrates comments, and assigns reviewers. + +### State tracking + +Migration progress is stored in `state.json` (inside `sync.work_dir`). This file tracks: +- Which repos have been migrated and their GitHub org/repo mapping +- Last sync timestamp per repo +- Bitbucket PR ID → GitHub PR number mappings + +This makes every operation idempotent — re-running any command skips already-completed work. + +--- + +## PR Migration Notes + +- PRs are created under the service account — the original author is attributed in the PR body +- Inline/file-level Bitbucket comments are migrated as regular PR comments +- Only **open** PRs are migrated (merged/declined PRs are preserved in git history) +- Reviewer assignments use `user_mapping`; unmapped usernames pass through as-is + +--- + +## Configuration Reference + +| Setting | Required | Default | Description | +|---|---|---|---| +| `bitbucket.base_url` | Yes | — | Bitbucket Server URL (no trailing slash) | +| `bitbucket.token` | Yes* | `$BB_TOKEN` | Personal access token for REST API | +| `bitbucket.ssh_url` | Yes | — | SSH base URL for git clone (e.g. `ssh://git@host:7999`) | +| `bitbucket.projects` | No | all | List of project keys to migrate | +| `github.base_url` | Yes | — | GitHub Enterprise API URL | +| `github.token` | Yes* | `$GH_TOKEN` | PAT with `repo` + `admin:org` scopes | +| `github.org` | Yes | — | Default target GitHub organization | +| `sync.interval_seconds` | No | `60` | Seconds between sync cycles | +| `sync.work_dir` | No | `/data/mirror` | Directory for bare repo clones and state | +| `repo_mapping.name_template` | No | `{slug}` | Template for GitHub repo names | +| `repo_mapping.projects..github_org` | No | `github.org` | Override target org per project | +| `repo_mapping.projects..name_template` | No | global template | Override naming per project | +| `repo_mapping.projects..include_repos` | No | — | Allowlist: only listed repos migrate | +| `repo_mapping.projects..exclude_repos` | No | `[]` | Denylist: listed repos are skipped | +| `repo_mapping.projects..repos..github_name` | No | template | Explicit repo name override | +| `user_mapping` | No | `{}` | Bitbucket → GitHub username map | + +\* Can be set via environment variable instead. + +--- + +## Testing + +```bash +pip install pytest +python -m pytest tests/ -v +``` + +## Troubleshooting + +| Problem | Solution | +|---|---| +| `git clone` fails with permission denied | Verify SSH key is configured and the Bitbucket service account has Project READ | +| `422` error when creating GitHub repo | Repo already exists (this is handled automatically) — or the token lacks `repo` scope | +| PR migration fails with `404` | The source or target branch was deleted; the PR cannot be recreated | +| Sync takes too long for many repos | Increase `sync.interval_seconds` or reduce the project list | +| `state.json` is corrupted | Delete it and re-run `migrate` (it will skip repos that already exist on GitHub) | diff --git a/bb2gh/__init__.py b/bb2gh/__init__.py new file mode 100644 index 0000000..7cc069d --- /dev/null +++ b/bb2gh/__init__.py @@ -0,0 +1 @@ +"""Bitbucket Server to GitHub Enterprise migration tool.""" diff --git a/bb2gh/bitbucket_client.py b/bb2gh/bitbucket_client.py new file mode 100644 index 0000000..e8e9987 --- /dev/null +++ b/bb2gh/bitbucket_client.py @@ -0,0 +1,119 @@ +"""Bitbucket Server REST API client.""" + +import logging +import requests + +logger = logging.getLogger(__name__) + + +class BitbucketClient: + """Client for Bitbucket Server (Data Center) REST API v1.0.""" + + def __init__(self, base_url, token, verify_ssl=True): + self.base_url = base_url.rstrip("/") + self.api_url = f"{self.base_url}/rest/api/1.0" + self.session = requests.Session() + self.session.verify = verify_ssl + if not verify_ssl: + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + if token: + self.session.headers["Authorization"] = f"Bearer {token}" + + def _paginate(self, url, params=None): + """Iterate through all pages of a Bitbucket paginated endpoint.""" + params = dict(params or {}) + params.setdefault("limit", 25) + + while True: + resp = self.session.get(url, params=params) + resp.raise_for_status() + data = resp.json() + + yield from data.get("values", []) + + if data.get("isLastPage", True): + break + params["start"] = data["nextPageStart"] + + def list_projects(self): + """List all projects on the Bitbucket instance.""" + url = f"{self.api_url}/projects" + return list(self._paginate(url)) + + def resolve_project_key(self, project_key): + """Resolve a project key, following Bitbucket aliases for renamed projects. + + Returns the current/canonical project key, or None if not found. + """ + url = f"{self.api_url}/projects/{project_key}" + try: + resp = self.session.get(url) + if resp.status_code == 200: + return resp.json().get("key") + except Exception: + pass + return None + + def resolve_repo_location(self, project_key, repo_slug): + """Resolve a repo's current project and slug, following moves/aliases. + + When a repo is moved from one project to another, Bitbucket keeps + the old URL alive. This method returns the current (project_key, slug). + """ + url = f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" + try: + resp = self.session.get(url) + if resp.status_code == 200: + data = resp.json() + real_project = data.get("project", {}).get("key") + real_slug = data.get("slug") + if real_project: + return real_project, real_slug or repo_slug + except Exception: + pass + return None, None + + def list_repos(self, project_key): + """List all repositories in a project.""" + url = f"{self.api_url}/projects/{project_key}/repos" + return list(self._paginate(url)) + + def list_pull_requests(self, project_key, repo_slug, state="OPEN"): + """List pull requests for a repository. + + Args: + state: OPEN, DECLINED, MERGED, or ALL + """ + url = f"{self.api_url}/projects/{project_key}/repos/{repo_slug}/pull-requests" + return list(self._paginate(url, params={"state": state})) + + def get_pr_activities(self, project_key, repo_slug, pr_id): + """Get activities (comments, approvals, etc.) for a pull request.""" + url = ( + f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" + f"/pull-requests/{pr_id}/activities" + ) + return list(self._paginate(url)) + + def get_pr_diff(self, project_key, repo_slug, pr_id): + """Get the diff for a pull request.""" + url = ( + f"{self.api_url}/projects/{project_key}/repos/{repo_slug}" + f"/pull-requests/{pr_id}/diff" + ) + resp = self.session.get(url) + resp.raise_for_status() + return resp.json() + + def get_repo_clone_url(self, repo, protocol="ssh"): + """Extract clone URL from a repo object. + + Args: + repo: Repo dict from the Bitbucket API. + protocol: 'ssh' or 'http'. + """ + for link in repo.get("links", {}).get("clone", []): + if link.get("name") == protocol: + return link["href"] + return None diff --git a/bb2gh/cli.py b/bb2gh/cli.py new file mode 100644 index 0000000..d970b70 --- /dev/null +++ b/bb2gh/cli.py @@ -0,0 +1,461 @@ +"""CLI entry point for bb2gh migration tool.""" + +import logging +import sys + +import click + +from .config import Config +from .migrator import migrate_repos +from .pr_migrator import migrate_pull_requests +from .syncer import Syncer + + +def _setup_logging(verbose): + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + +@click.group() +@click.option("--config", "config_path", default="config.yaml", help="Path to config file.") +@click.option("-v", "--verbose", is_flag=True, help="Enable debug logging.") +@click.pass_context +def cli(ctx, config_path, verbose): + """bb2gh - Bitbucket Server to GitHub Enterprise migration tool.""" + _setup_logging(verbose) + ctx.ensure_object(dict) + try: + ctx.obj["config"] = Config(config_path) + except Exception as e: + click.echo(f"Error loading config: {e}", err=True) + sys.exit(1) + + +@cli.command() +@click.option("--repo", multiple=True, help="Migrate specific repos only (PROJECT/SLUG, can repeat).") +@click.pass_context +def migrate(ctx, repo): + """Bulk migrate all repositories from Bitbucket to GitHub. + + Clones repos via SSH, creates them on GitHub, and pushes all + branches, tags, and history. + """ + config = ctx.obj["config"] + only_repos = set(repo) if repo else None + migrated, skipped, failed = migrate_repos(config, only_repos=only_repos) + click.echo(f"\nMigration complete: {migrated} migrated, {skipped} skipped, {failed} failed") + if failed > 0: + sys.exit(1) + + +@cli.command() +@click.pass_context +def sync(ctx): + """Continuously sync repos from Bitbucket to GitHub. + + Fetches changes from Bitbucket and pushes to GitHub every N seconds + (configured via sync.interval_seconds). Runs until interrupted. + """ + config = ctx.obj["config"] + syncer = Syncer(config) + syncer.run() + + +@cli.command("migrate-prs") +@click.option("--repo", multiple=True, help="Migrate PRs for specific repos only (PROJECT/SLUG).") +@click.option("--include-closed", is_flag=True, help="Also migrate merged/declined PRs (in addition to open).") +@click.option("--closed-only", is_flag=True, help="Migrate ONLY merged/declined PRs, skip open ones.") +@click.option("--api-delay", type=float, default=None, + help="Seconds between API calls (default from config, or 0.5).") +@click.option("--pr-delay", type=float, default=None, + help="Seconds between full PR migrations (default from config, or 3.0).") +@click.option("--dry-run", is_flag=True, help="Log what would be done without making changes.") +@click.pass_context +def migrate_prs(ctx, repo, include_closed, closed_only, api_delay, pr_delay, dry_run): + """Migrate pull requests from Bitbucket to GitHub. + + Open PRs become GitHub PRs (with title, description, comments, and + reviewers). Closed PRs (MERGED/DECLINED) become closed GitHub Issues + with all their comments, labeled `migrated-pr` + `merged`/`declined`, + so everything is searchable in one place. + + --include-closed migrates open PRs plus closed ones as issues. + --closed-only skips open PRs entirely (use after migrating open ones). + + Rate limiting: --api-delay controls minimum spacing between GitHub + API calls; --pr-delay adds a pause between full PR migrations. + Both fall back to the pr_migration section of config.yaml. + """ + config = ctx.obj["config"] + only_repos = set(repo) if repo else None + + if api_delay is not None: + config.pr_api_delay = api_delay + if pr_delay is not None: + config.pr_pr_delay = pr_delay + + migrated, skipped, failed = migrate_pull_requests( + config, dry_run=dry_run, + include_closed=include_closed, closed_only=closed_only, + only_repos=only_repos, + ) + click.echo(f"\nPR migration complete: {migrated} migrated, {skipped} skipped, {failed} failed") + if failed > 0: + sys.exit(1) + + +@cli.command("reset") +@click.option("--project", multiple=True, help="Reset all repos in this project (can repeat).") +@click.option("--repo", multiple=True, help="Reset a specific PROJECT/SLUG (can repeat).") +@click.option("--all", "reset_all", is_flag=True, help="Reset ALL migrated repos.") +@click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") +@click.pass_context +def reset(ctx, project, repo, reset_all, dry_run): + """Reset repos in state.json so they get re-migrated. + + Examples: + bb2gh reset --project UPSTREAM + bb2gh reset --repo UPSTREAM/embeddedsw --repo UPSTREAM/git + bb2gh reset --all + """ + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + repos = state.get_all_repos() + if not repos: + click.echo("No repos found in state.") + return + + projects_set = set(p.upper() for p in project) + repos_set = set(repo) + + reset_count = 0 + for project_key, repo_slug in repos: + should_reset = False + if reset_all: + should_reset = True + elif project_key in projects_set or project_key.upper() in projects_set: + should_reset = True + elif f"{project_key}/{repo_slug}" in repos_set: + should_reset = True + + if not should_reset: + continue + + if dry_run: + gh_org, gh_repo = state.get_github_target(project_key, repo_slug) + click.echo(f"[DRY RUN] Would reset: {project_key}/{repo_slug} (was -> {gh_org}/{gh_repo})") + else: + state.reset_repo(project_key, repo_slug) + click.echo(f"Reset: {project_key}/{repo_slug}") + reset_count += 1 + + if reset_count == 0: + click.echo("No matching repos found in state.") + else: + click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos.") + if not dry_run: + click.echo("Run 'bb2gh migrate' to re-migrate them.") + + +@cli.command("reset-submodules") +@click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") +@click.pass_context +def reset_submodules(ctx, dry_run): + """Reset migrated repos that have .gitmodules so they get re-migrated. + + Scans bare clones for repos containing .gitmodules, removes them from + state.json, so the next 'bb2gh migrate' run re-processes them (with + submodule URL remapping). + """ + import os + import subprocess + + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + repos = state.get_all_repos() + if not repos: + click.echo("No repos found in state.") + return + + reset_count = 0 + for project_key, repo_slug in repos: + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + continue + + result = subprocess.run( + ["git", "show", "HEAD:.gitmodules"], + cwd=bare_path, capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + continue + + if dry_run: + click.echo(f"[DRY RUN] Would reset: {project_key}/{repo_slug}") + else: + state.reset_repo(project_key, repo_slug) + click.echo(f"Reset: {project_key}/{repo_slug}") + reset_count += 1 + + click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos with submodules.") + if not dry_run and reset_count > 0: + click.echo("Run 'bb2gh migrate' to re-migrate them.") + + +@cli.command("reset-lfs") +@click.option("--dry-run", is_flag=True, help="Show what would be reset without changing state.") +@click.pass_context +def reset_lfs(ctx, dry_run): + """Reset migrated repos that were LFS-migrated so they get re-processed. + + Finds repos where LFS migration previously ran (have lfs/ directory + or .gitattributes with LFS patterns), removes them from state.json + so the next 'bb2gh migrate' re-fetches from Bitbucket and re-runs + LFS with the current threshold. + """ + import os + import subprocess + + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + repos = state.get_all_repos() + if not repos: + click.echo("No repos found in state.") + return + + reset_count = 0 + for project_key, repo_slug in repos: + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + continue + + # Check for LFS indicators + has_lfs_dir = os.path.exists(os.path.join(bare_path, "lfs", "objects")) + + has_lfs_attrs = False + result = subprocess.run( + ["git", "show", "HEAD:.gitattributes"], + cwd=bare_path, capture_output=True, text=True, check=False, + ) + if result.returncode == 0 and "filter=lfs" in result.stdout: + has_lfs_attrs = True + + if not has_lfs_dir and not has_lfs_attrs: + continue + + if dry_run: + click.echo(f"[DRY RUN] Would reset: {project_key}/{repo_slug}") + else: + state.reset_repo(project_key, repo_slug) + click.echo(f"Reset: {project_key}/{repo_slug}") + reset_count += 1 + + click.echo(f"\n{'Would reset' if dry_run else 'Reset'} {reset_count} repos with previous LFS migration.") + if not dry_run and reset_count > 0: + click.echo("Run 'bb2gh migrate' to re-migrate them with the current LFS threshold.") + + +@cli.command() +@click.option("--format", "fmt", type=click.Choice(["text", "csv"]), default="text", + help="Output format.") +@click.option("--output", "output_file", default=None, help="Write report to file.") +@click.pass_context +def report(ctx, fmt, output_file): + """Generate a migration report from state.json. + + Shows migrated repos, failed repos, submodule status, LFS status, + and any warnings. + """ + import json + + config = ctx.obj["config"] + from .state import State + state = State(config.work_dir) + + data = state._data.get("repos", {}) + if not data: + click.echo("No migration data found.") + return + + migrated = [] + failed = [] + with_submodules = [] + submodules_not_remapped = [] + with_lfs = [] + with_warnings = [] + + for key, entry in sorted(data.items()): + status = entry.get("status", "unknown") + if status == "migrated": + migrated.append(entry) + elif status == "failed": + failed.append(entry) + + if entry.get("has_submodules"): + with_submodules.append(entry) + if not entry.get("submodules_remapped"): + submodules_not_remapped.append(entry) + if entry.get("has_lfs"): + with_lfs.append(entry) + if entry.get("warnings"): + with_warnings.append(entry) + + lines = [] + + if fmt == "text": + lines.append("=" * 70) + lines.append("MIGRATION REPORT") + lines.append("=" * 70) + lines.append("") + lines.append(f"Total repos in state: {len(data)}") + lines.append(f" Migrated: {len(migrated)}") + lines.append(f" Failed: {len(failed)}") + lines.append(f" With submodules: {len(with_submodules)}") + lines.append(f" Remapped: {len(with_submodules) - len(submodules_not_remapped)}") + lines.append(f" Not remapped: {len(submodules_not_remapped)}") + lines.append(f" With LFS: {len(with_lfs)}") + lines.append(f" With warnings: {len(with_warnings)}") + + if failed: + lines.append("") + lines.append("-" * 70) + lines.append("FAILED REPOS") + lines.append("-" * 70) + for entry in failed: + lines.append(f" {entry['project_key']}/{entry['repo_slug']}") + lines.append(f" Target: {entry.get('gh_org', '?')}/{entry.get('gh_repo_name', '?')}") + lines.append(f" Error: {entry.get('error', 'unknown')[:200]}") + + if submodules_not_remapped: + lines.append("") + lines.append("-" * 70) + lines.append("SUBMODULES NOT REMAPPED") + lines.append("-" * 70) + for entry in submodules_not_remapped: + lines.append(f" {entry['project_key']}/{entry['repo_slug']} -> {entry.get('gh_org')}/{entry.get('gh_repo_name')}") + + if with_lfs: + lines.append("") + lines.append("-" * 70) + lines.append("REPOS WITH LFS") + lines.append("-" * 70) + for entry in with_lfs: + lines.append(f" {entry['project_key']}/{entry['repo_slug']} -> {entry.get('gh_org')}/{entry.get('gh_repo_name')}") + + if with_warnings: + lines.append("") + lines.append("-" * 70) + lines.append("WARNINGS") + lines.append("-" * 70) + for entry in with_warnings: + for w in entry.get("warnings", []): + lines.append(f" {entry['project_key']}/{entry['repo_slug']}: {w}") + + if migrated: + lines.append("") + lines.append("-" * 70) + lines.append("ALL MIGRATED REPOS") + lines.append("-" * 70) + for entry in migrated: + flags = [] + if entry.get("has_submodules"): + flags.append("submodules") + if entry.get("has_lfs"): + flags.append("LFS") + if entry.get("warnings"): + flags.append("warnings") + flag_str = f" [{', '.join(flags)}]" if flags else "" + lines.append( + f" {entry['project_key']}/{entry['repo_slug']} " + f"-> {entry.get('gh_org')}/{entry.get('gh_repo_name')}{flag_str}" + ) + + elif fmt == "csv": + lines.append("status,bb_project,bb_repo,gh_org,gh_repo,has_submodules,submodules_remapped,has_lfs,warnings,error") + for key, entry in sorted(data.items()): + warnings_str = "; ".join(entry.get("warnings", [])) + error_str = entry.get("error", "").replace(",", " ")[:200] + lines.append( + f"{entry.get('status', 'unknown')}," + f"{entry.get('project_key', '')}," + f"{entry.get('repo_slug', '')}," + f"{entry.get('gh_org', '')}," + f"{entry.get('gh_repo_name', '')}," + f"{entry.get('has_submodules', False)}," + f"{entry.get('submodules_remapped', False)}," + f"{entry.get('has_lfs', False)}," + f"\"{warnings_str}\"," + f"\"{error_str}\"" + ) + + output = "\n".join(lines) + + if output_file: + with open(output_file, "w") as f: + f.write(output + "\n") + click.echo(f"Report written to {output_file}") + else: + click.echo(output) + + +@cli.command("prepare-jenkins") +@click.option("--repo", multiple=True, help="Prepare specific repos only (PROJECT/SLUG).") +@click.option("--branch", default=None, help="Source branch (default: repo default branch).") +@click.option("--migration-branch", default="ci/github-actions-migration", + help="Branch name to create on GitHub for the migration.") +@click.option("--dry-run", is_flag=True, help="Show what would be done without making changes.") +@click.pass_context +def prepare_jenkins(ctx, repo, branch, migration_branch, dry_run): + """Prepare workspaces for Jenkins-to-GitHub-Actions conversion. + + Creates clean workspace directories containing only Jenkinsfiles + and their dependencies, with a new migration branch on GitHub. + """ + config = ctx.obj["config"] + from .jenkins_prep import prepare_jenkins_workspaces + only_repos = set(repo) if repo else None + prepared, skipped, failed = prepare_jenkins_workspaces( + config, only_repos=only_repos, branch=branch, + migration_branch_name=migration_branch, dry_run=dry_run, + ) + click.echo(f"\nJenkins prep: {prepared} prepared, {skipped} skipped, {failed} failed") + if failed > 0: + sys.exit(1) + + +@cli.command("jenkins-manifest") +@click.option("--repo", multiple=True, help="Include specific repos only (PROJECT/SLUG).") +@click.option("--branch", default=None, help="Source branch (default: repo default branch).") +@click.option("--migration-branch", default="ci/github-actions-migration", + help="Migration branch name to include in manifest.") +@click.option("--output", default="jenkins-manifest.yaml", help="Output manifest file path.") +@click.pass_context +def jenkins_manifest(ctx, repo, branch, migration_branch, output): + """Generate a manifest of repos with Jenkinsfiles for remote AI agents. + + Scans migrated repos for Jenkinsfiles and writes a YAML manifest + listing clone URLs, branch names, and file paths. Transfer this + file to the machine where the AI agent runs. + """ + config = ctx.obj["config"] + from .jenkins_prep import generate_manifest + only_repos = set(repo) if repo else None + found = generate_manifest( + config, only_repos=only_repos, branch=branch, + migration_branch_name=migration_branch, output_path=output, + ) + click.echo(f"\nManifest written to {output}: {found} repos with Jenkinsfiles") + + +if __name__ == "__main__": + cli() diff --git a/bb2gh/config.py b/bb2gh/config.py new file mode 100644 index 0000000..2de3516 --- /dev/null +++ b/bb2gh/config.py @@ -0,0 +1,181 @@ +"""Configuration loading and validation.""" + +import os +import yaml + + +class Config: + """Loads and validates migration configuration from a YAML file.""" + + def __init__(self, path="config.yaml"): + with open(path) as f: + raw = yaml.safe_load(f) + + self._validate(raw) + self._raw = raw + + # Bitbucket settings + bb = raw["bitbucket"] + self.bb_base_url = bb["base_url"].rstrip("/") + self.bb_token = bb.get("token") or os.environ.get("BB_TOKEN", "") + self.bb_ssh_url = bb["ssh_url"].rstrip("/") + self.bb_ssh_hostnames = bb.get("ssh_hostnames", []) + self.bb_projects = bb.get("projects") # None means all projects + self.bb_verify_ssl = bb.get("verify_ssl", True) + + # GitHub settings + gh = raw["github"] + self.gh_base_url = gh["base_url"].rstrip("/") + self.gh_token = gh.get("token") or os.environ.get("GH_TOKEN", "") + self.gh_org = gh["org"] + self.gh_ssh_host = gh.get("ssh_host", "") + self.gh_ssh_url = gh.get("ssh_url", "") + + # Sync settings + sync = raw.get("sync", {}) + self.sync_interval = sync.get("interval_seconds", 60) + self.work_dir = sync.get("work_dir", "/data/mirror") + self.migrate_delay = sync.get("migrate_delay_seconds", 2) + self.sync_lfs_timeout = sync.get("sync_lfs_timeout_seconds", 60) + self.sync_exclude_projects = set( + p.upper() for p in sync.get("exclude_projects", []) + ) + self.sync_protected_branches = sync.get("protected_branches", []) + + # PR migration settings + pr = raw.get("pr_migration", {}) + self.pr_api_delay = pr.get("api_delay_seconds", 0.5) + self.pr_pr_delay = pr.get("pr_delay_seconds", 3.0) + self.pr_retry_on_rate_limit = pr.get("retry_on_rate_limit", True) + self.pr_max_retries = pr.get("max_retries", 5) + + # User mapping (Bitbucket username -> GitHub username) + self.user_mapping = raw.get("user_mapping", {}) + + # LFS settings + lfs = raw.get("lfs", {}) + self.lfs_enabled = lfs.get("enabled", False) + self.lfs_threshold = lfs.get("threshold", "100mb") + + # History trimming (repo-specific) — rewrites history, changes commit hashes + self.trim_history = raw.get("trim_history", {}) + + # Repos that need branch-by-branch push (too large for --mirror's 2GB pack limit) + self.push_by_branch = set(raw.get("push_by_branch", [])) + + # Project key aliases (old_key -> current_key) for .gitmodules remapping + self.project_aliases = {} + for old, new in raw.get("project_aliases", {}).items(): + self.project_aliases[old.upper()] = new.upper() + + # Repository mapping (Bitbucket project/repo -> GitHub org/repo) + rm = raw.get("repo_mapping", {}) + self._repo_mapping = rm + self._name_template = rm.get("name_template", "{slug}") + self._project_mappings = rm.get("projects", {}) + + def resolve_target(self, project_key, repo_slug): + """Resolve a Bitbucket project/repo to a GitHub org and repo name. + + Lookup order for org: + 1. Per-repo github_org in repo_mapping.projects..repos..github_org + 2. Per-project github_org in repo_mapping.projects..github_org + 3. Global github.org + + Lookup order for repo name: + 1. Per-repo github_name in repo_mapping.projects..repos..github_name + 2. Per-project name_template + 3. Global name_template (default: "{slug}") + + Returns: + (github_org, github_repo_name) tuple + """ + project_conf = self._project_mappings.get(project_key, {}) + + # Resolve org + gh_org = project_conf.get("github_org", self.gh_org) + + # Resolve repo name and org: check explicit per-repo override first + repos_conf = project_conf.get("repos", {}) + if repo_slug in repos_conf: + repo_conf = repos_conf[repo_slug] + gh_org = repo_conf.get("github_org", gh_org) + gh_repo = repo_conf.get("github_name", repo_slug) + else: + # Use per-project template, falling back to global template + template = project_conf.get("name_template", self._name_template) + gh_repo = template.format( + project=project_key, + project_lower=project_key.lower(), + slug=repo_slug, + ) + + return gh_org, gh_repo + + def get_trim_since(self, project_key, repo_slug): + """Get the --shallow-since date for a repo, or None if no trimming. + + Config format: + trim_history: + UPSTREAM/linux: "2y" + UPSTREAM/git: "1y" + + Supports: Ny (years), Nm (months), Nd (days). + Returns an ISO date string or None. + """ + from datetime import datetime, timedelta + + key = f"{project_key}/{repo_slug}" + period = self.trim_history.get(key) + if not period: + return None + + period = period.strip().lower() + if period.endswith("y"): + delta = timedelta(days=int(period[:-1]) * 365) + elif period.endswith("m"): + delta = timedelta(days=int(period[:-1]) * 30) + elif period.endswith("d"): + delta = timedelta(days=int(period[:-1])) + else: + return None + + since = datetime.now() - delta + return since.strftime("%Y-%m-%d") + + def should_migrate_repo(self, project_key, repo_slug): + """Check if a repo should be migrated based on include/exclude lists. + + Resolution: + - If `include_repos` is set for the project, the repo is migrated only + if it is in that list (allowlist). + - Otherwise, the repo is migrated unless it is in `exclude_repos` + (denylist). + - Projects with no repo filter config migrate all repos. + + Returns: + True if the repo should be migrated. + """ + project_conf = self._project_mappings.get(project_key, {}) + include = project_conf.get("include_repos") + exclude = project_conf.get("exclude_repos", []) + + if include is not None: + return repo_slug in include + return repo_slug not in exclude + + @staticmethod + def _validate(raw): + for section in ("bitbucket", "github"): + if section not in raw: + raise ValueError(f"Missing required config section: {section}") + + bb = raw["bitbucket"] + for key in ("base_url", "ssh_url"): + if key not in bb: + raise ValueError(f"Missing required bitbucket config: {key}") + + gh = raw["github"] + for key in ("base_url", "org"): + if key not in gh: + raise ValueError(f"Missing required github config: {key}") diff --git a/bb2gh/github_client.py b/bb2gh/github_client.py new file mode 100644 index 0000000..90bc5c5 --- /dev/null +++ b/bb2gh/github_client.py @@ -0,0 +1,141 @@ +"""GitHub Enterprise API client wrapper.""" + +import logging +from github import Github, GithubException + +logger = logging.getLogger(__name__) + + +class GithubClient: + """Wrapper around PyGithub for GitHub Enterprise operations. + + Supports multiple GitHub organizations. Each method accepts an + org_name parameter to target the correct org. + """ + + def __init__(self, base_url, token, default_org): + self.gh = Github(base_url=base_url, login_or_token=token) + self._token = token + self.default_org = default_org + self._org_cache = {} + + def _get_org(self, org_name=None): + """Get a GitHub organization object, with caching.""" + org_name = org_name or self.default_org + if org_name not in self._org_cache: + self._org_cache[org_name] = self.gh.get_organization(org_name) + return self._org_cache[org_name] + + def create_repo(self, name, description="", private=True, org_name=None): + """Create a repository in the organization. + + Returns the repo object. If the repo already exists, returns the existing one. + """ + org = self._get_org(org_name) + actual_org = org_name or self.default_org + try: + repo = org.create_repo( + name=name, + description=description, + private=private, + auto_init=False, + ) + logger.info("Created GitHub repo: %s/%s", actual_org, name) + return repo + except GithubException as e: + if e.status == 422: # Already exists + logger.info("GitHub repo already exists: %s/%s", actual_org, name) + return org.get_repo(name) + raise + + def get_repo(self, name, org_name=None): + """Get an existing repository.""" + org = self._get_org(org_name) + return org.get_repo(name) + + def create_pull_request(self, repo_name, title, body, head, base, org_name=None): + """Create a pull request on a GitHub repository. + + Args: + repo_name: Repository name. + title: PR title. + body: PR body/description (markdown). + head: Source branch name. + base: Target branch name. + org_name: Target GitHub org (defaults to default_org). + + Returns the created PR object. + """ + repo = self.get_repo(repo_name, org_name) + pr = repo.create_pull(title=title, body=body, head=head, base=base) + logger.info("Created PR #%d on %s: %s", pr.number, repo_name, title) + return pr + + def add_pr_comment(self, repo_name, pr_number, body, org_name=None): + """Add a comment to a pull request.""" + repo = self.get_repo(repo_name, org_name) + pr = repo.get_pull(pr_number) + comment = pr.create_issue_comment(body) + return comment + + def add_pr_reviewers(self, repo_name, pr_number, reviewers, org_name=None): + """Request reviewers on a pull request. + + Args: + reviewers: List of GitHub usernames. + """ + if not reviewers: + return + repo = self.get_repo(repo_name, org_name) + pr = repo.get_pull(pr_number) + try: + pr.create_review_request(reviewers=reviewers) + logger.info("Added reviewers to PR #%d: %s", pr_number, reviewers) + except GithubException as e: + logger.warning( + "Failed to add reviewers to PR #%d: %s", pr_number, e + ) + + def set_default_branch(self, repo_name, branch, org_name=None): + """Set the default branch for a repository.""" + repo = self.get_repo(repo_name, org_name) + repo.edit(default_branch=branch) + logger.info("Set default branch for %s to %s", repo_name, branch) + + def get_default_branch(self, repo_name, org_name=None): + """Get the default branch name for a repository.""" + repo = self.get_repo(repo_name, org_name) + return repo.default_branch + + def create_branch(self, repo_name, branch_name, from_branch=None, org_name=None): + """Create a branch on a GitHub repository. + + If from_branch is None, branches from the default branch. + Returns the branch name. If it already exists, returns it. + """ + repo = self.get_repo(repo_name, org_name) + source = from_branch or repo.default_branch + sha = repo.get_branch(source).commit.sha + try: + repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=sha) + logger.info("Created branch %s on %s from %s", branch_name, repo_name, source) + except GithubException as e: + if e.status == 422: + logger.info("Branch %s already exists on %s", branch_name, repo_name) + else: + raise + return branch_name + + def get_clone_url(self, repo_name, org_name=None, ssh_url=None): + """Get the clone URL for a repo. + + If ssh_url is provided (e.g. "ssh://gatehousesatcom@host"), builds + an SSH URL. Otherwise returns HTTPS with the token embedded. + """ + org = org_name or self.default_org + if ssh_url: + return f"{ssh_url.rstrip('/')}/{org}/{repo_name}.git" + repo = self.get_repo(repo_name, org_name) + url = repo.clone_url + url = url.replace("https://", f"https://x-access-token:{self._token}@", 1) + return url diff --git a/bb2gh/jenkins_prep.py b/bb2gh/jenkins_prep.py new file mode 100644 index 0000000..cc7a4f2 --- /dev/null +++ b/bb2gh/jenkins_prep.py @@ -0,0 +1,357 @@ +"""Prepare workspaces for Jenkins-to-GitHub-Actions conversion.""" + +import json +import logging +import os +import re +import subprocess +from datetime import datetime, timezone + +import yaml + +from .github_client import GithubClient +from .state import State + +logger = logging.getLogger(__name__) + + +def _run_git(args, cwd=None, quiet=False): + cmd = ["git"] + args + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + if not quiet: + logger.error("git %s failed: %s", args[0], result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def find_jenkins_files(all_paths): + """Filter a list of file paths for Jenkins-related files. + + Returns (jenkins_files, dependency_files) tuple. + """ + jenkins_files = [] + deps = [] + jenkinsfile_dirs = set() + + for path in all_paths: + basename = os.path.basename(path).lower() + + if basename == "jenkinsfile" or basename.startswith("jenkinsfile.") or basename.endswith(".jenkinsfile"): + jenkins_files.append(path) + jenkinsfile_dirs.add(os.path.dirname(path)) + continue + + parts = path.split("/") + if parts[0] in ("vars", "resources"): + deps.append(path) + continue + if parts[0] == "src" and path.lower().endswith((".groovy", ".java")): + deps.append(path) + continue + + for path in all_paths: + if path in jenkins_files or path in deps: + continue + if path.lower().endswith(".groovy") and os.path.dirname(path) in jenkinsfile_dirs: + deps.append(path) + + return jenkins_files, deps + + +def parse_jenkinsfile_refs(content): + """Extract file references from Jenkinsfile content.""" + refs = set() + for pattern in [ + r"""load\s+['"]([^'"]+)['"]""", + r"""readFile\s*\(\s*['"]([^'"]+)['"]""", + r"""evaluate\s*\(\s*readFile\s*\(\s*['"]([^'"]+)['"]""", + ]: + for match in re.finditer(pattern, content): + refs.add(match.group(1)) + return refs + + +def _resolve_dependencies(bare_path, jenkins_files, ref="HEAD"): + """Read Jenkinsfiles from a repo and find referenced files.""" + extra_deps = set() + for jf in jenkins_files: + try: + content = _run_git(["show", f"{ref}:{jf}"], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + continue + refs = parse_jenkinsfile_refs(content) + extra_deps.update(refs) + return extra_deps + + +def _get_all_tree_paths(repo_path, ref="HEAD"): + """List all file paths in a git tree.""" + output = _run_git(["ls-tree", "-r", "--name-only", ref], cwd=repo_path) + return [p for p in output.splitlines() if p.strip()] + + +def create_workspace(config, state, gh, project_key, repo_slug, + source_branch, migration_branch, dry_run=False): + """Create a Jenkins workspace for a single repo.""" + gh_org, gh_repo_name = state.get_github_target(project_key, repo_slug) + if not gh_org: + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + + repo_key = f"{project_key}/{repo_slug}" + workspace_base = os.path.join(config.work_dir, "jenkins-workspaces") + workspace_path = os.path.join(workspace_base, f"{gh_org}__{gh_repo_name}") + + # Discover Jenkins files from the bare clone (fast, no network) + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + logger.warning("Bare clone not found for %s, skipping", repo_key) + return None + + # Determine source branch + if not source_branch: + try: + source_branch = gh.get_default_branch(gh_repo_name, org_name=gh_org) + except Exception: + source_branch = "master" + + # Get file tree from bare clone + try: + ref = f"refs/heads/{source_branch}" + all_paths = _get_all_tree_paths(bare_path, ref=ref) + except subprocess.CalledProcessError: + try: + all_paths = _get_all_tree_paths(bare_path) + except subprocess.CalledProcessError: + logger.warning("No branches found in %s (empty repo?), skipping", repo_key) + return None + + jenkins_files, dep_files = find_jenkins_files(all_paths) + + if not jenkins_files: + logger.info("No Jenkinsfiles found in %s, skipping", repo_key) + return None + + # Resolve inline dependencies from Jenkinsfile content + extra_refs = _resolve_dependencies(bare_path, jenkins_files, ref=ref) + validated_extras = [p for p in extra_refs if p in all_paths] + all_required = sorted(set(jenkins_files + dep_files + validated_extras)) + + if dry_run: + logger.info("[DRY RUN] Would prepare %s: %d Jenkinsfiles, %d dependencies", + repo_key, len(jenkins_files), len(all_required) - len(jenkins_files)) + for f in jenkins_files: + logger.info("[DRY RUN] Jenkinsfile: %s", f) + return None + + # Create migration branch on GitHub + try: + gh.create_branch(gh_repo_name, migration_branch, + from_branch=source_branch, org_name=gh_org) + except Exception: + logger.warning("Could not create migration branch %s on %s/%s", + migration_branch, gh_org, gh_repo_name) + + # Create workspace via sparse checkout + os.makedirs(workspace_base, exist_ok=True) + if os.path.exists(workspace_path): + import shutil + shutil.rmtree(workspace_path) + + clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, + ssh_url=config.gh_ssh_url or None) + + env_no_lfs = {**os.environ, "GIT_LFS_SKIP_SMUDGE": "1"} + subprocess.run( + ["git", "clone", "--no-checkout", "--depth=1", + f"--branch={source_branch}", clone_url, workspace_path], + capture_output=True, text=True, check=True, env=env_no_lfs, + ) + + # Configure sparse checkout + _run_git(["sparse-checkout", "init", "--no-cone"], cwd=workspace_path) + sparse_file = os.path.join(workspace_path, ".git", "info", "sparse-checkout") + with open(sparse_file, "w") as f: + for path in all_required: + f.write(path + "\n") + + _run_git(["checkout"], cwd=workspace_path) + + # Create migration branch locally + _run_git(["checkout", "-b", migration_branch], cwd=workspace_path) + + # Create .github/workflows directory + workflows_dir = os.path.join(workspace_path, ".github", "workflows") + os.makedirs(workflows_dir, exist_ok=True) + + # Write metadata + meta = { + "source": { + "bitbucket_project": project_key, + "bitbucket_repo": repo_slug, + "github_org": gh_org, + "github_repo": gh_repo_name, + "source_branch": source_branch, + "migration_branch": migration_branch, + }, + "jenkins_files": jenkins_files, + "dependencies": [f for f in all_required if f not in jenkins_files], + "prepared_at": datetime.now(timezone.utc).isoformat(), + } + with open(os.path.join(workspace_path, ".bb2gh-jenkins-meta.json"), "w") as f: + json.dump(meta, f, indent=2) + + # Write per-repo manifest + manifest = { + "repo": { + "github_org": gh_org, + "github_repo": gh_repo_name, + "clone_url": clone_url, + "source_branch": source_branch, + "migration_branch": migration_branch, + }, + "jenkins_files": jenkins_files, + "dependencies": [f for f in all_required if f not in jenkins_files], + "all_files": all_required, + } + with open(os.path.join(workspace_path, ".bb2gh-jenkins-manifest.yaml"), "w") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) + + # Track in state + state.mark_jenkins_prepared( + project_key, repo_slug, migration_branch, + workspace_path, jenkins_files, + ) + + logger.info("Prepared workspace for %s: %d Jenkinsfiles, %d total files -> %s", + repo_key, len(jenkins_files), len(all_required), workspace_path) + return workspace_path + + +def prepare_jenkins_workspaces(config, only_repos=None, branch=None, + migration_branch_name="ci/github-actions-migration", + dry_run=False): + """Prepare Jenkins workspaces for migrated repos.""" + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + state = State(config.work_dir) + + repos = state.get_migrated_repos() + if not repos: + logger.warning("No migrated repos found.") + return 0, 0, 0 + + prepared = 0 + skipped = 0 + failed = 0 + + for project_key, repo_slug in repos: + repo_key = f"{project_key}/{repo_slug}" + if only_repos and repo_key not in only_repos: + continue + + if state.is_jenkins_prepared(project_key, repo_slug): + logger.info("Skipping already prepared: %s", repo_key) + skipped += 1 + continue + + try: + result = create_workspace( + config, state, gh, project_key, repo_slug, + branch, migration_branch_name, dry_run, + ) + if result: + prepared += 1 + else: + skipped += 1 + except Exception: + logger.exception("Failed to prepare %s", repo_key) + failed += 1 + + if not dry_run and prepared > 0: + logger.warning( + "WARNING: The sync loop (bb2gh sync) will delete the migration " + "branch '%s' on its next cycle. Pause sync or merge your changes " + "before the next sync.", migration_branch_name, + ) + + return prepared, skipped, failed + + +def generate_manifest(config, only_repos=None, branch=None, + migration_branch_name="ci/github-actions-migration", + output_path="jenkins-manifest.yaml"): + """Generate a manifest file listing all repos with Jenkins files.""" + state = State(config.work_dir) + repos = state.get_migrated_repos() + if not repos: + logger.warning("No migrated repos found.") + return 0 + + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + manifest_repos = [] + found = 0 + + for project_key, repo_slug in repos: + repo_key = f"{project_key}/{repo_slug}" + if only_repos and repo_key not in only_repos: + continue + + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + if not os.path.exists(bare_path): + continue + + gh_org, gh_repo_name = state.get_github_target(project_key, repo_slug) + if not gh_org: + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + + # Determine source branch + src_branch = branch + if not src_branch: + try: + src_branch = gh.get_default_branch(gh_repo_name, org_name=gh_org) + except Exception: + src_branch = "master" + + try: + ref = f"refs/heads/{src_branch}" + all_paths = _get_all_tree_paths(bare_path, ref=ref) + except subprocess.CalledProcessError: + all_paths = _get_all_tree_paths(bare_path) + + jenkins_files, dep_files = find_jenkins_files(all_paths) + if not jenkins_files: + continue + + extra_refs = _resolve_dependencies(bare_path, jenkins_files, ref=ref) + validated_extras = [p for p in extra_refs if p in all_paths] + all_deps = sorted(set(dep_files + validated_extras)) + + clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, + ssh_url=config.gh_ssh_url or None) + + manifest_repos.append({ + "github_org": gh_org, + "github_repo": gh_repo_name, + "clone_url": clone_url, + "source_branch": src_branch, + "jenkins_files": jenkins_files, + "dependencies": all_deps, + }) + found += 1 + logger.info("Found %d Jenkinsfiles in %s", len(jenkins_files), repo_key) + + manifest = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "migration_branch": migration_branch_name, + "repos": manifest_repos, + } + + with open(output_path, "w") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) + + logger.info("Manifest written to %s: %d repos with Jenkinsfiles", output_path, found) + return found diff --git a/bb2gh/migrator.py b/bb2gh/migrator.py new file mode 100644 index 0000000..6998bec --- /dev/null +++ b/bb2gh/migrator.py @@ -0,0 +1,520 @@ +"""Bulk migration of repositories from Bitbucket Server to GitHub Enterprise.""" + +import logging +import os +import subprocess +import time + +from .bitbucket_client import BitbucketClient +from .github_client import GithubClient +from .state import State +from .submodules import remap_submodules_in_bare_repo + +logger = logging.getLogger(__name__) + + +def _redact(text): + """Remove tokens/passwords from URLs in log output.""" + import re + return re.sub(r"(https?://)[^@/]+@", r"\1***@", text) + + +def _run_git(args, cwd=None, quiet=False): + """Run a git command and return stdout.""" + cmd = ["git"] + args + logger.debug("Running: %s", " ".join(cmd)) + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + if not quiet: + logger.error("git %s failed: %s", args[0], _redact(result.stderr.strip())) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def _clean_hidden_refs(bare_repo_path): + """Remove hidden refs (refs/pull/*, refs/merge-requests/*) that can't be pushed.""" + try: + output = _run_git(["show-ref"], cwd=bare_repo_path) + except subprocess.CalledProcessError: + return # No refs to clean + + for line in output.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + ref = parts[1] + if "/pull/" in ref or "/merge-request" in ref: + try: + _run_git(["update-ref", "-d", ref], cwd=bare_repo_path) + logger.debug("Deleted hidden ref: %s", ref) + except subprocess.CalledProcessError: + logger.warning("Failed to delete ref: %s", ref) + + +def _has_large_blobs(bare_path, threshold): + """Check if a bare repo has any reachable blobs above the threshold. + + Only checks blobs reachable from refs (what push --mirror would send). + Uses a shell pipeline for reliable streaming on large repos. + """ + import glob + + t = threshold.lower().strip() + if t.endswith("mb"): + threshold_bytes = int(t[:-2]) * 1024 * 1024 + elif t.endswith("gb"): + threshold_bytes = int(t[:-2]) * 1024 * 1024 * 1024 + elif t.endswith("kb"): + threshold_bytes = int(t[:-2]) * 1024 + else: + threshold_bytes = int(t) + + # Shell pipeline: list reachable objects, strip paths, check sizes, stop at first match + cmd = ( + "git rev-list --objects --all" + " | cut -d' ' -f1" + " | git cat-file --batch-check='%(objecttype) %(objectsize)'" + f" | awk '$1 == \"blob\" && $2 > {threshold_bytes} {{print; exit}}'" + ) + proc = subprocess.Popen( + cmd, shell=True, cwd=bare_path, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + ) + line = proc.stdout.readline() + proc.terminate() + proc.wait() + return len(line.strip()) > 0 + + +def _trim_history(bare_path, since_date): + """Trim repo history using git replace --graft + filter-branch. + + Finds the oldest commit after since_date on each branch, grafts it + as a root commit (no parents), then rewrites history permanently. + This creates a clean, pushable history without shallow boundaries. + Note: commit hashes will change. + """ + import shutil + import tempfile + + logger.info("Trimming history (keeping since %s) in %s", since_date, os.path.basename(bare_path)) + tmp_dir = tempfile.mkdtemp(suffix=".trim") + try: + work_path = os.path.join(tmp_dir, "work") + env_no_lfs = {"GIT_LFS_SKIP_SMUDGE": "1"} + cmd = ["git", "clone", bare_path, work_path] + subprocess.run(cmd, capture_output=True, text=True, check=True, + env={**os.environ, **env_no_lfs}) + + # Create local branches for all remotes + branches_output = _run_git(["branch", "-r"], cwd=work_path) + for line in branches_output.splitlines(): + branch = line.strip() + if "HEAD" in branch or not branch.startswith("origin/"): + continue + local_name = branch.replace("origin/", "", 1) + try: + _run_git(["branch", "--track", local_name, branch], + cwd=work_path, quiet=True) + except subprocess.CalledProcessError: + pass + + # Find graft points: oldest commit after cutoff per branch + graft_points = set() + local_branches = _run_git( + ["for-each-ref", "--format=%(refname)", "refs/heads/"], + cwd=work_path, quiet=True, + ) + for ref in local_branches.strip().splitlines(): + try: + commits = _run_git( + ["rev-list", f"--after={since_date}", "--reverse", ref], + cwd=work_path, quiet=True, + ) + except subprocess.CalledProcessError: + continue + lines = commits.strip().splitlines() + if lines: + graft_points.add(lines[0]) + + if not graft_points: + logger.info("No commits to trim in %s", os.path.basename(bare_path)) + return + + # Graft each point as a root commit + for sha in graft_points: + _run_git(["replace", "--graft", sha], cwd=work_path) + + # Rewrite history permanently + env_filter = {**os.environ, "FILTER_BRANCH_SQUELCH_WARNING": "1"} + subprocess.run( + ["git", "filter-branch", "--tag-name-filter", "cat", "--", "--all"], + cwd=work_path, capture_output=True, text=True, check=True, + env=env_filter, + ) + + # Fetch rewritten refs back into the bare repo + _run_git(["remote", "add", "trim-source", work_path], cwd=bare_path) + _run_git(["fetch", "trim-source", "--force", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) + _run_git(["remote", "remove", "trim-source"], cwd=bare_path) + + # Clean up old objects + _run_git(["reflog", "expire", "--expire=now", "--all"], cwd=bare_path, quiet=True) + _run_git(["gc", "--prune=now"], cwd=bare_path, quiet=True) + + logger.info("Trimmed history in %s: %d graft points", os.path.basename(bare_path), len(graft_points)) + + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +def _migrate_lfs(bare_path, threshold, timeout=None): + """Convert files above threshold to Git LFS in all branches. + + git lfs migrate import requires a working tree, so we clone the + bare repo to a temp directory, create local branches for all remotes, + run LFS migration, then fetch the rewritten refs back. + """ + import shutil + import tempfile + + if not _has_large_blobs(bare_path, threshold): + logger.info("LFS: no files above %s in %s, skipping", threshold, os.path.basename(bare_path)) + return False + + logger.info("LFS: large files detected, migrating (threshold: %s) in %s", threshold, bare_path) + tmp_dir = tempfile.mkdtemp(suffix=".lfs-migrate") + deadline = time.time() + timeout if timeout else None + try: + work_path = os.path.join(tmp_dir, "work") + # Skip LFS smudge during clone — repo may already have LFS pointers + # pointing to the original BB LFS server + env_no_lfs = { + "GIT_LFS_SKIP_SMUDGE": "1", + } + cmd = ["git", "clone", bare_path, work_path] + remaining = int(deadline - time.time()) if deadline else None + result = subprocess.run( + cmd, capture_output=True, text=True, check=False, + env={**os.environ, **env_no_lfs}, timeout=remaining, + ) + if result.returncode != 0: + logger.error("git clone failed: %s", result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + + # Create local branches for ALL remote branches so LFS rewrites them all + branches_output = _run_git(["branch", "-r"], cwd=work_path) + for line in branches_output.splitlines(): + branch = line.strip() + if "HEAD" in branch or not branch.startswith("origin/"): + continue + local_name = branch.replace("origin/", "", 1) + try: + _run_git(["branch", "--track", local_name, branch], cwd=work_path, quiet=True) + except subprocess.CalledProcessError: + pass # Already exists (default branch) + + _run_git(["lfs", "install"], cwd=work_path) + remaining = int(deadline - time.time()) if deadline else None + if remaining is not None and remaining <= 0: + raise subprocess.TimeoutExpired("git lfs migrate", timeout) + lfs_cmd = ["git", "lfs", "migrate", "import", "--everything", + f"--above={threshold}", "--yes"] + lfs_result = subprocess.run( + lfs_cmd, cwd=work_path, capture_output=True, text=True, + check=False, timeout=remaining, + ) + if lfs_result.returncode != 0: + stderr = lfs_result.stderr or "" + if "Could not checkout" in stderr and "Rewriting commits" in stderr: + logger.warning("LFS rewrite completed but checkout failed (harmless)") + else: + logger.error("git lfs migrate failed: %s", stderr.strip()) + raise subprocess.CalledProcessError( + lfs_result.returncode, lfs_cmd, lfs_result.stdout, stderr + ) + + # Fetch rewritten branches and tags back into the bare repo + _run_git(["remote", "add", "lfs-source", work_path], cwd=bare_path) + _run_git(["fetch", "lfs-source", "--force", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) + _run_git(["remote", "remove", "lfs-source"], cwd=bare_path) + + # Check if LFS actually tracked any files (not just empty dirs) + lfs_objects_dir = os.path.join(work_path, ".git", "lfs", "objects") + has_lfs_objects = False + if os.path.exists(lfs_objects_dir): + for dirpath, dirnames, filenames in os.walk(lfs_objects_dir): + if filenames: + has_lfs_objects = True + break + + # Copy LFS objects into the bare repo only if real objects exist + if has_lfs_objects: + lfs_dst = os.path.join(bare_path, "lfs") + if os.path.exists(lfs_dst): + shutil.rmtree(lfs_dst) + shutil.copytree(os.path.join(work_path, ".git", "lfs"), lfs_dst) + + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + if has_lfs_objects: + logger.info("LFS migration converted files in %s", bare_path) + else: + logger.info("LFS migration: no files above threshold in %s", bare_path) + return has_lfs_objects + + +def migrate_repos(config, only_repos=None): + """Run the full bulk migration. + + Args: + config: Config object. + only_repos: Optional set of "PROJECT/SLUG" strings to migrate. + If provided, only these repos are processed. + """ + bb = BitbucketClient(config.bb_base_url, config.bb_token, verify_ssl=config.bb_verify_ssl) + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + state = State(config.work_dir) + + os.makedirs(config.work_dir, exist_ok=True) + + if only_repos: + # Extract unique project keys from the repo list + projects = list({r.split("/")[0] for r in only_repos}) + else: + projects = config.bb_projects or [p["key"] for p in bb.list_projects()] + + total_migrated = 0 + total_skipped = 0 + total_failed = 0 + + for project_key in projects: + logger.info("Processing project: %s", project_key) + repos = bb.list_repos(project_key) + + for repo in repos: + repo_slug = repo["slug"] + repo_name = repo.get("name", repo_slug) + + if only_repos and f"{project_key}/{repo_slug}" not in only_repos: + continue + + if not config.should_migrate_repo(project_key, repo_slug): + logger.info( + "Skipping %s/%s (filtered out by include/exclude_repos)", + project_key, repo_slug, + ) + total_skipped += 1 + continue + + if state.is_migrated(project_key, repo_slug): + logger.info("Skipping already migrated: %s/%s", project_key, repo_slug) + total_skipped += 1 + continue + + try: + _migrate_single_repo( + config, bb, gh, state, project_key, repo_slug, repo_name, repo + ) + total_migrated += 1 + except Exception as e: + logger.exception("Failed to migrate %s/%s", project_key, repo_slug) + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + state.record_failure( + project_key, repo_slug, str(e), + gh_org=gh_org, gh_repo_name=gh_repo_name, + ) + total_failed += 1 + + # Throttle between repos to avoid SSH/API rate limits + import time + time.sleep(config.migrate_delay) + + logger.info( + "Migration complete: %d migrated, %d skipped, %d failed", + total_migrated, total_skipped, total_failed, + ) + return total_migrated, total_skipped, total_failed + + +def _push_lfs_objects(bare_path): + """Push LFS objects by OID instead of scanning all refs with --all.""" + lfs_dir = os.path.join(bare_path, "lfs", "objects") + if not os.path.exists(lfs_dir): + return + oids = [] + for dirpath, _, filenames in os.walk(lfs_dir): + for f in filenames: + if len(f) == 64: + oids.append(f) + if not oids: + return + logger.info("Pushing %d LFS objects", len(oids)) + for oid in oids: + try: + _run_git(["lfs", "push", "github", "--object-id", oid], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + logger.warning("Failed to push LFS object %s", oid[:12]) + + +def _push_branch_by_branch(bare_path, project_key, repo_slug, delay=2): + """Push branches and tags individually when --mirror pack exceeds 2GB.""" + branches = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/heads/"], + cwd=bare_path, quiet=True) + pushed = 0 + failed = 0 + for branch in branches.strip().splitlines(): + branch = branch.strip() + if not branch: + continue + try: + _run_git(["push", "github", f"{branch}:{branch}", "--force"], cwd=bare_path) + pushed += 1 + except subprocess.CalledProcessError: + logger.warning("Failed to push branch %s for %s/%s", branch, project_key, repo_slug) + failed += 1 + time.sleep(delay) + + tags = _run_git(["for-each-ref", "--format=%(refname:short)", "refs/tags/"], + cwd=bare_path, quiet=True) + tag_list = [t.strip() for t in tags.strip().splitlines() if t.strip()] + if tag_list: + try: + # Push all tags at once — they're lightweight (no large packs) + _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) + except subprocess.CalledProcessError: + # Fallback: push individually if batch fails + for tag in tag_list: + try: + _run_git(["push", "github", f"refs/tags/{tag}:refs/tags/{tag}", "--force"], + cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + pass + + logger.info("Branch-by-branch push: %d pushed, %d failed for %s/%s", + pushed, failed, project_key, repo_slug) + if failed > 0 and pushed == 0: + raise RuntimeError(f"All branch pushes failed for {project_key}/{repo_slug}") + + +def _migrate_single_repo(config, bb, gh, state, project_key, repo_slug, repo_name, repo): + """Migrate a single repository.""" + # Resolve target GitHub org and repo name + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + logger.info( + "Migrating %s/%s -> %s/%s ...", + project_key, repo_slug, gh_org, gh_repo_name, + ) + + # 1. Create repo on GitHub (in the resolved org) + import re + description = repo.get("description", "") or f"Migrated from Bitbucket: {project_key}/{repo_slug}" + description = re.sub(r"[\x00-\x1f\x7f]", " ", description).strip()[:350] + gh.create_repo(gh_repo_name, description=description, private=True, org_name=gh_org) + + # 2. Bare clone from Bitbucket + bare_path = os.path.join(config.work_dir, f"{project_key}__{repo_slug}.git") + trim_since = config.get_trim_since(project_key, repo_slug) + + if os.path.exists(bare_path): + # Already cloned, fetch latest + logger.info("Bare clone exists, fetching latest: %s", bare_path) + _run_git(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) + else: + clone_url = bb.get_repo_clone_url(repo, protocol="ssh") + if not clone_url: + # Fallback: construct SSH URL from config + clone_url = f"{config.bb_ssh_url}/{project_key.lower()}/{repo_slug}.git" + + logger.info("Cloning %s -> %s", clone_url, bare_path) + _run_git(["clone", "--bare", clone_url, bare_path]) + + # 3. Clean hidden refs + _clean_hidden_refs(bare_path) + + # Track migration details + warnings = [] + + # 4. Trim history if configured (must run before submodule remap) + if trim_since: + _trim_history(bare_path, trim_since) + + # 5. Remap submodule URLs from Bitbucket to GitHub + submodules_remapped = remap_submodules_in_bare_repo( + bare_path, config, alias_resolver=bb.resolve_repo_location, + ) + has_submodules = submodules_remapped > 0 + try: + _run_git(["show", "HEAD:.gitmodules"], cwd=bare_path, quiet=True) + has_submodules = True + if submodules_remapped == 0: + warnings.append("Has .gitmodules but submodule URLs could not be fully remapped") + except subprocess.CalledProcessError: + pass + + # 6. Migrate large files to LFS if enabled + has_lfs = False + if config.lfs_enabled: + has_lfs = _migrate_lfs(bare_path, config.lfs_threshold) + + # 7. Add GitHub remote and push + gh_clone_url = gh.get_clone_url(gh_repo_name, org_name=gh_org, ssh_url=config.gh_ssh_url or None) + + try: + _run_git(["remote", "remove", "github"], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + pass + + _run_git(["remote", "add", "github", gh_clone_url], cwd=bare_path) + + repo_key = f"{project_key}/{repo_slug}" + if repo_key in config.push_by_branch: + logger.info("Pushing branch-by-branch for %s (configured)", repo_key) + _push_branch_by_branch(bare_path, project_key, repo_slug, delay=config.migrate_delay) + else: + try: + _run_git(["push", "--mirror", "github"], cwd=bare_path) + except subprocess.CalledProcessError as e: + if "pack exceeds maximum allowed size" in (e.stderr or ""): + logger.warning("Pack too large for --mirror, pushing branch-by-branch for %s/%s", project_key, repo_slug) + _push_branch_by_branch(bare_path, project_key, repo_slug, delay=config.migrate_delay) + else: + raise + + # Push LFS objects separately — only if LFS actually converted files + if has_lfs: + try: + _push_lfs_objects(bare_path) + except subprocess.CalledProcessError: + logger.warning("LFS push failed for %s/%s", gh_org, gh_repo_name) + warnings.append("LFS push failed") + + # 7. Set default branch on GitHub to match Bitbucket's HEAD + try: + head_ref = _run_git(["symbolic-ref", "HEAD"], cwd=bare_path) + default_branch = head_ref.replace("refs/heads/", "") + gh.set_default_branch(gh_repo_name, default_branch, org_name=gh_org) + except Exception: + logger.warning("Could not set default branch for %s/%s", gh_org, gh_repo_name) + warnings.append("Could not set default branch") + + # 8. Record in state + state.mark_migrated( + project_key, repo_slug, gh_org=gh_org, gh_repo_name=gh_repo_name, + has_submodules=has_submodules, submodules_remapped=submodules_remapped > 0, + has_lfs=has_lfs, warnings=warnings, + ) + logger.info("Successfully migrated %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) diff --git a/bb2gh/pr_migrator.py b/bb2gh/pr_migrator.py new file mode 100644 index 0000000..76cdda9 --- /dev/null +++ b/bb2gh/pr_migrator.py @@ -0,0 +1,332 @@ +"""Migrate pull requests from Bitbucket Server to GitHub Enterprise. + +Open PRs are migrated as GitHub PRs (they have a live branch). +Closed PRs (MERGED, DECLINED) are migrated as closed GitHub Issues with +all their comments preserved. Everything lives in one place — the issue +tracker — so PR history is searchable alongside code review discussions. +""" + +import logging +import random +import time + +from github import GithubException + +from .bitbucket_client import BitbucketClient +from .github_client import GithubClient +from .state import State + +logger = logging.getLogger(__name__) + + +class Throttler: + """Rate-limits API calls with a fixed delay and retry-on-rate-limit backoff.""" + + def __init__(self, api_delay=0.5, pr_delay=3.0, max_retries=5): + self.api_delay = api_delay + self.pr_delay = pr_delay + self.max_retries = max_retries + self._last_call = 0.0 + + def wait_api(self): + if self.api_delay <= 0: + return + elapsed = time.monotonic() - self._last_call + remaining = self.api_delay - elapsed + if remaining > 0: + time.sleep(remaining) + self._last_call = time.monotonic() + + def wait_between_prs(self): + if self.pr_delay > 0: + time.sleep(self.pr_delay) + + def call(self, fn, *args, **kwargs): + for attempt in range(self.max_retries + 1): + self.wait_api() + try: + return fn(*args, **kwargs) + except GithubException as e: + if not self._is_rate_limited(e) or attempt >= self.max_retries: + raise + delay = self._retry_delay(e, attempt) + logger.warning( + "GitHub rate limit hit (status=%s), sleeping %.1fs (attempt %d/%d)", + e.status, delay, attempt + 1, self.max_retries, + ) + time.sleep(delay) + + @staticmethod + def _is_rate_limited(e): + if e.status == 429: + return True + if e.status == 403: + msg = str(e).lower() + return "rate limit" in msg or "abuse" in msg or "secondary" in msg + return False + + @staticmethod + def _retry_delay(e, attempt): + headers = getattr(e, "headers", {}) or {} + retry_after = headers.get("Retry-After") or headers.get("retry-after") + if retry_after: + try: + return float(retry_after) + except ValueError: + pass + return min(60.0, (2 ** attempt) + random.random()) + + +def _format_pr_body(pr, config, closed_state=None): + """Format the GitHub PR/issue body with migration metadata.""" + bb_url = config.bb_base_url + project = pr["toRef"]["repository"]["project"]["key"] + repo = pr["toRef"]["repository"]["slug"] + pr_id = pr["id"] + author = pr["author"]["user"].get("displayName", pr["author"]["user"].get("name", "Unknown")) + created = pr.get("createdDate", "") + head_branch = pr["fromRef"]["displayId"] + base_branch = pr["toRef"]["displayId"] + + header = ( + f"> **Migrated from Bitbucket**\n" + f"> Source: [{project}/{repo} PR #{pr_id}]" + f"({bb_url}/projects/{project}/repos/{repo}/pull-requests/{pr_id})\n" + f"> Original author: **{author}**\n" + f"> Branch: `{head_branch}` → `{base_branch}`\n" + ) + if created: + header += f"> Created: {created}\n" + if closed_state: + header += f"> Original status: **{closed_state}**\n" + + description = pr.get("description", "") or "" + return f"{header}\n---\n\n{description}" + + +def _format_comment(activity, config): + """Format a Bitbucket comment for GitHub.""" + comment = activity.get("comment", {}) + user = comment.get("author", {}) + display_name = user.get("displayName", user.get("name", "Unknown")) + text = comment.get("text", "") + created = comment.get("createdDate", "") + + header = f"**{display_name}** commented" + if created: + header += f" (originally at {created})" + header += ":" + + return f"{header}\n\n{text}" + + +def _map_reviewers(pr, config): + """Map Bitbucket reviewer usernames to GitHub usernames.""" + reviewers = [] + for reviewer in pr.get("reviewers", []): + bb_username = reviewer["user"].get("name", "") + gh_username = config.user_mapping.get(bb_username, bb_username) + if gh_username: + reviewers.append(gh_username) + return reviewers + + +def _iter_comment_activities(activities): + """Yield COMMENTED activities in chronological order (oldest first).""" + comments = [a for a in activities if a.get("action") == "COMMENTED" and "comment" in a] + comments.sort(key=lambda a: a.get("comment", {}).get("createdDate") or a.get("createdDate", 0)) + return comments + + +def migrate_pull_requests( + config, dry_run=False, include_closed=False, closed_only=False, + only_repos=None, throttler=None, +): + """Migrate pull requests from Bitbucket to GitHub. + + Args: + config: Config object. + dry_run: If True, log what would be done without making changes. + include_closed: If True, also migrate merged/declined PRs (as issues). + closed_only: If True, migrate ONLY merged/declined PRs. Implies include_closed. + only_repos: Optional set of "PROJECT/SLUG" strings to filter repos. + throttler: Optional Throttler instance. Defaults are read from config. + """ + if closed_only: + include_closed = True + + if throttler is None: + throttler = Throttler( + api_delay=getattr(config, "pr_api_delay", 0.5), + pr_delay=getattr(config, "pr_pr_delay", 3.0), + max_retries=getattr(config, "pr_max_retries", 5), + ) + + bb = BitbucketClient(config.bb_base_url, config.bb_token, verify_ssl=config.bb_verify_ssl) + gh = GithubClient(config.gh_base_url, config.gh_token, config.gh_org) + state = State(config.work_dir) + + migrated_repos = state.get_migrated_repos() + if not migrated_repos: + logger.warning("No migrated repos found. Run 'bb2gh migrate' first.") + return 0, 0, 0 + + total_migrated = 0 + total_skipped = 0 + total_failed = 0 + + for project_key, repo_slug in migrated_repos: + repo_key = f"{project_key}/{repo_slug}" + if only_repos and repo_key not in only_repos: + continue + + gh_org, gh_repo_name = state.get_github_target(project_key, repo_slug) + if not gh_org or not gh_repo_name: + gh_org, gh_repo_name = config.resolve_target(project_key, repo_slug) + + logger.info("Processing PRs for %s/%s -> %s/%s", project_key, repo_slug, gh_org, gh_repo_name) + + prs = [] + + if not closed_only: + try: + prs = bb.list_pull_requests(project_key, repo_slug, state="OPEN") + except Exception: + logger.exception("Failed to list open PRs for %s/%s", project_key, repo_slug) + continue + + if include_closed: + try: + merged = bb.list_pull_requests(project_key, repo_slug, state="MERGED") + declined = bb.list_pull_requests(project_key, repo_slug, state="DECLINED") + prs.extend(merged) + prs.extend(declined) + except Exception: + logger.exception("Failed to list closed PRs for %s/%s", project_key, repo_slug) + + for pr in prs: + pr_id = pr["id"] + title = pr["title"] + pr_state = pr.get("state", "OPEN") + + if state.is_pr_migrated(project_key, repo_slug, pr_id): + logger.info("Skipping already migrated PR #%d: %s", pr_id, title) + total_skipped += 1 + continue + + if dry_run: + target = "PR" if pr_state == "OPEN" else "closed Issue" + logger.info( + "[DRY RUN] Would migrate PR #%d [%s] as %s: %s -> %s/%s", + pr_id, pr_state, target, title, gh_org, gh_repo_name, + ) + total_migrated += 1 + continue + + try: + if pr_state == "OPEN": + _migrate_open_pr( + config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, throttler, + ) + else: + _migrate_closed_pr_as_issue( + config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, throttler, + ) + total_migrated += 1 + throttler.wait_between_prs() + except Exception: + logger.exception( + "Failed to migrate PR #%d in %s/%s", pr_id, project_key, repo_slug + ) + total_failed += 1 + + logger.info( + "PR migration complete: %d migrated, %d skipped, %d failed", + total_migrated, total_skipped, total_failed, + ) + return total_migrated, total_skipped, total_failed + + +def _migrate_open_pr(config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, throttler): + """Migrate an open PR as a GitHub PR.""" + pr_id = pr["id"] + title = pr["title"] + head_branch = pr["fromRef"]["displayId"] + base_branch = pr["toRef"]["displayId"] + + logger.info("Migrating open PR #%d: %s (%s -> %s)", pr_id, title, head_branch, base_branch) + + body = _format_pr_body(pr, config) + gh_pr = throttler.call( + gh.create_pull_request, + repo_name=gh_repo_name, title=title, body=body, + head=head_branch, base=base_branch, org_name=gh_org, + ) + + activities = bb.get_pr_activities(project_key, repo_slug, pr_id) + for activity in _iter_comment_activities(activities): + comment_body = _format_comment(activity, config) + throttler.call( + gh.add_pr_comment, gh_repo_name, gh_pr.number, comment_body, org_name=gh_org, + ) + + reviewers = _map_reviewers(pr, config) + if reviewers: + throttler.call( + gh.add_pr_reviewers, gh_repo_name, gh_pr.number, reviewers, org_name=gh_org, + ) + + state.record_pr_mapping(project_key, repo_slug, pr_id, gh_pr.number) + logger.info("Migrated open PR #%d -> GitHub PR #%d", pr_id, gh_pr.number) + + +def _migrate_closed_pr_as_issue(config, bb, gh, state, project_key, repo_slug, + gh_org, gh_repo_name, pr, throttler): + """Migrate a closed (merged/declined) PR as a closed GitHub Issue. + + Everything lives in one place — the issue tracker — so PR history is + searchable alongside other issues. No branch recreation, no repo access + needed, works even when the source branch has been GC'd. + """ + pr_id = pr["id"] + title = pr["title"] + pr_state = pr.get("state", "UNKNOWN") + + logger.info("Migrating %s PR #%d as issue: %s", pr_state, pr_id, title) + + body = _format_pr_body(pr, config, closed_state=pr_state) + + repo = throttler.call(gh.get_repo, gh_repo_name, org_name=gh_org) + + labels = ["migrated-pr", pr_state.lower()] + try: + issue = throttler.call( + repo.create_issue, + title=f"[{pr_state} PR #{pr_id}] {title}", + body=body, + labels=labels, + ) + except GithubException as e: + # Labels don't exist yet — create without labels + if e.status in (404, 422): + logger.debug("Labels not found on %s, creating issue without labels", gh_repo_name) + issue = throttler.call( + repo.create_issue, + title=f"[{pr_state} PR #{pr_id}] {title}", + body=body, + ) + else: + raise + + activities = bb.get_pr_activities(project_key, repo_slug, pr_id) + for activity in _iter_comment_activities(activities): + comment_body = _format_comment(activity, config) + throttler.call(issue.create_comment, comment_body) + + throttler.call(issue.edit, state="closed") + + state.record_pr_mapping(project_key, repo_slug, pr_id, issue.number) + logger.info("Migrated %s PR #%d -> GitHub Issue #%d (closed)", pr_state, pr_id, issue.number) diff --git a/bb2gh/state.py b/bb2gh/state.py new file mode 100644 index 0000000..f62027d --- /dev/null +++ b/bb2gh/state.py @@ -0,0 +1,161 @@ +"""State tracking for migration progress.""" + +import json +import logging +import os +from datetime import datetime, timezone + +logger = logging.getLogger(__name__) + +STATE_FILE = "state.json" + + +class State: + """Tracks migration state in a JSON file.""" + + def __init__(self, work_dir): + self.path = os.path.join(work_dir, STATE_FILE) + self._data = self._load() + + def _load(self): + if os.path.exists(self.path): + with open(self.path) as f: + return json.load(f) + return {"repos": {}} + + def _save(self): + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, "w") as f: + json.dump(self._data, f, indent=2) + + def _now(self): + return datetime.now(timezone.utc).isoformat() + + def mark_migrated(self, project_key, repo_slug, gh_org=None, gh_repo_name=None, + has_submodules=False, submodules_remapped=False, + has_lfs=False, warnings=None): + """Record that a repo has been migrated. + + Args: + project_key: Bitbucket project key. + repo_slug: Bitbucket repo slug. + gh_org: GitHub organization the repo was migrated to. + gh_repo_name: GitHub repository name. + has_submodules: Whether the repo has .gitmodules. + submodules_remapped: Whether submodule URLs were remapped. + has_lfs: Whether large files were converted to LFS. + warnings: List of warning strings. + """ + key = f"{project_key}/{repo_slug}" + self._data["repos"][key] = { + "project_key": project_key, + "repo_slug": repo_slug, + "gh_org": gh_org, + "gh_repo_name": gh_repo_name or repo_slug, + "status": "migrated", + "migrated_at": self._now(), + "last_sync": self._now(), + "has_submodules": has_submodules, + "submodules_remapped": submodules_remapped, + "has_lfs": has_lfs, + "warnings": warnings or [], + "pr_mappings": {}, + } + self._save() + logger.info("Marked %s as migrated -> %s/%s", key, gh_org, gh_repo_name) + + def record_failure(self, project_key, repo_slug, error_message, + gh_org=None, gh_repo_name=None): + """Record that a repo failed to migrate.""" + key = f"{project_key}/{repo_slug}" + self._data["repos"][key] = { + "project_key": project_key, + "repo_slug": repo_slug, + "gh_org": gh_org, + "gh_repo_name": gh_repo_name, + "status": "failed", + "failed_at": self._now(), + "error": error_message, + } + self._save() + + def update_sync_time(self, project_key, repo_slug): + """Update the last sync timestamp for a repo.""" + key = f"{project_key}/{repo_slug}" + if key in self._data["repos"]: + self._data["repos"][key]["last_sync"] = self._now() + self._save() + + def record_pr_mapping(self, project_key, repo_slug, bb_pr_id, gh_pr_number): + """Record the mapping between a Bitbucket PR and GitHub PR.""" + key = f"{project_key}/{repo_slug}" + if key in self._data["repos"]: + self._data["repos"][key]["pr_mappings"][str(bb_pr_id)] = gh_pr_number + self._save() + + def get_migrated_repos(self): + """Return list of (project_key, repo_slug) for all migrated repos.""" + result = [] + for entry in self._data["repos"].values(): + if entry.get("status") == "migrated": + result.append((entry["project_key"], entry["repo_slug"])) + return result + + def get_all_repos(self): + """Return list of (project_key, repo_slug) for all repos in state.""" + result = [] + for entry in self._data["repos"].values(): + result.append((entry["project_key"], entry["repo_slug"])) + return result + + def get_github_target(self, project_key, repo_slug): + """Get the GitHub org and repo name for a migrated repo. + + Returns: + (gh_org, gh_repo_name) tuple, or (None, None) if not found. + """ + key = f"{project_key}/{repo_slug}" + entry = self._data["repos"].get(key, {}) + return entry.get("gh_org"), entry.get("gh_repo_name") + + def reset_repo(self, project_key, repo_slug): + """Remove a repo from state so it will be re-migrated on next run.""" + key = f"{project_key}/{repo_slug}" + if key in self._data["repos"]: + del self._data["repos"][key] + self._save() + return True + return False + + def is_migrated(self, project_key, repo_slug): + """Check if a repo has been successfully migrated.""" + key = f"{project_key}/{repo_slug}" + entry = self._data["repos"].get(key, {}) + return entry.get("status") == "migrated" + + def is_pr_migrated(self, project_key, repo_slug, bb_pr_id): + """Check if a specific PR has already been migrated.""" + key = f"{project_key}/{repo_slug}" + repo_state = self._data["repos"].get(key, {}) + return str(bb_pr_id) in repo_state.get("pr_mappings", {}) + + def mark_jenkins_prepared(self, project_key, repo_slug, migration_branch, + workspace_path, jenkins_files): + """Record that a repo's Jenkins workspace has been prepared.""" + key = f"{project_key}/{repo_slug}" + if key not in self._data["repos"]: + return + self._data["repos"][key]["jenkins_prep"] = { + "status": "prepared", + "prepared_at": self._now(), + "migration_branch": migration_branch, + "workspace_path": workspace_path, + "jenkins_files": jenkins_files, + } + self._save() + + def is_jenkins_prepared(self, project_key, repo_slug): + """Check if a repo's Jenkins workspace has been prepared.""" + key = f"{project_key}/{repo_slug}" + entry = self._data["repos"].get(key, {}) + return entry.get("jenkins_prep", {}).get("status") == "prepared" diff --git a/bb2gh/submodules.py b/bb2gh/submodules.py new file mode 100644 index 0000000..4f9ce96 --- /dev/null +++ b/bb2gh/submodules.py @@ -0,0 +1,280 @@ +"""Submodule URL remapping for Bitbucket-to-GitHub migration.""" + +import logging +import os +import re +import subprocess +from datetime import datetime, timezone +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +_REMAP_IDENTITY = { + "GIT_AUTHOR_NAME": "bb2gh", + "GIT_AUTHOR_EMAIL": "bb2gh@migration", + "GIT_COMMITTER_NAME": "bb2gh", + "GIT_COMMITTER_EMAIL": "bb2gh@migration", +} + +_COMMIT_MSG = "bb2gh: remap submodule URLs for GitHub migration" + + +def _git(args, cwd, stdin_data=None, env_extra=None): + cmd = ["git"] + args + env = None + if env_extra: + env = dict(os.environ) + env.update(env_extra) + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False, + input=stdin_data, env=env, + ) + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def _build_bb_hostnames(config): + """Extract all known Bitbucket hostnames from config.""" + hostnames = set(config.bb_ssh_hostnames) + + # Extract hostname from ssh_url: ssh://git@host:port -> host + parsed = urlparse(config.bb_ssh_url) + if parsed.hostname: + hostnames.add(parsed.hostname) + + # Extract hostname from base_url: https://host -> host + parsed = urlparse(config.bb_base_url) + if parsed.hostname: + hostnames.add(parsed.hostname) + + return hostnames + + +def _extract_submodule_urls(content): + """Extract all url = ... values from .gitmodules content.""" + return re.findall(r"url\s*=\s*(.+)", content) + + +def _parse_bb_url(url, bb_hostnames): + """Parse a URL and check if it's a Bitbucket URL. + + Returns (project_key, slug) if it's a BB URL, None otherwise. + Handles: + - ssh://git@host:port/project/repo.git + - git@host:port/project/repo.git (shouldn't exist for BB but just in case) + - https://host/scm/project/repo.git + """ + # SSH format: ssh://git@hostname:port/project/repo.git + m = re.match(r"ssh://[^@]+@([^:/]+)[:/]\d*/([^/]+)/([^/]+?)\.git$", url) + if m and m.group(1) in bb_hostnames: + return m.group(2), m.group(3) + + # HTTP format: https://hostname/scm/project/repo.git + m = re.match(r"https?://([^/]+)/scm/([^/]+)/([^/]+?)\.git$", url) + if m and m.group(1) in bb_hostnames: + return m.group(2), m.group(3) + + return None + + +def _is_already_github(url, gh_ssh_url, gh_ssh_host, gh_https_base): + """Check if a URL already points to GitHub.""" + if gh_ssh_url and url.startswith(gh_ssh_url.rstrip("/")): + return True + if gh_ssh_host and (url.startswith(f"git@{gh_ssh_host}:") or url.startswith(f"ssh://git@{gh_ssh_host}/") or url.startswith(f"ssh://{gh_ssh_host}/")): + return True + if gh_https_base and gh_https_base in url: + return True + return False + + +def remap_submodule_urls(content, config, alias_resolver=None, _alias_cache=None): + """Replace Bitbucket submodule URLs in .gitmodules with GitHub URLs. + + If any Bitbucket URL cannot be resolved (project not migrated, repo + excluded), the entire .gitmodules is left unchanged to avoid a mix + of old and new URLs. + + Args: + alias_resolver: Optional callable(project_key, slug) -> (real_project, real_slug). + _alias_cache: Shared cache dict for API results across branches. + """ + bb_hostnames = _build_bb_hostnames(config) + gh_ssh_url = config.gh_ssh_url + gh_ssh_host = config.gh_ssh_host + gh_https_base = config.gh_base_url.replace("/api/v3", "").rstrip("/") + alias_cache = _alias_cache if _alias_cache is not None else {} + project_aliases = getattr(config, "project_aliases", {}) + + urls = _extract_submodule_urls(content) + if not urls: + return content + + # First pass: check ALL URLs can be resolved + replacements = {} + for url in urls: + url = url.strip() + + # Already points to GitHub — skip + if _is_already_github(url, gh_ssh_url, gh_ssh_host, gh_https_base): + continue + + parsed = _parse_bb_url(url, bb_hostnames) + if parsed is None: + # Not a Bitbucket URL we recognize — skip (external dependency) + continue + + project_key_raw, slug = parsed + resolved = None + + # Resolve the actual repo location (handles moved repos + renamed projects) + actual_pk = project_key_raw.upper() + actual_slug = slug + + # Check manual alias first + manual_alias = project_aliases.get(actual_pk) + if manual_alias: + actual_pk = manual_alias + + # Auto-resolve via Bitbucket API (cached per project/slug combo) + if alias_resolver: + cache_key = f"{project_key_raw.upper()}/{slug}" + if cache_key not in alias_cache: + real_proj, real_slug = alias_resolver(project_key_raw, slug) + if real_proj and (real_proj.upper() != project_key_raw.upper() or real_slug != slug): + alias_cache[cache_key] = (real_proj.upper(), real_slug or slug) + logger.debug("Resolved %s/%s -> %s/%s via API", + project_key_raw, slug, real_proj, real_slug or slug) + else: + alias_cache[cache_key] = None + cached = alias_cache.get(cache_key) + if cached: + actual_pk, actual_slug = cached + + # Now resolve using the actual (possibly redirected) project/slug + for pk in [actual_pk, project_key_raw.upper(), project_key_raw]: + if config.bb_projects and pk not in config.bb_projects: + continue + if not config.should_migrate_repo(pk, actual_slug): + continue + resolved = config.resolve_target(pk, actual_slug) + break + + if not resolved: + logger.warning( + "Cannot remap submodule URL %s — project %s/%s not in migration scope. " + "Skipping .gitmodules rewrite entirely.", + url, project_key_raw, slug, + ) + return content # Return unchanged + + gh_org, gh_repo = resolved + is_ssh = url.startswith("ssh://") + if is_ssh and gh_ssh_url: + new_url = f"{gh_ssh_url.rstrip('/')}/{gh_org}/{gh_repo}.git" + elif is_ssh and gh_ssh_host: + new_url = f"ssh://git@{gh_ssh_host}/{gh_org}/{gh_repo}.git" + else: + new_url = f"{gh_https_base}/{gh_org}/{gh_repo}.git" + replacements[url] = new_url + + if not replacements: + return content + + # Second pass: apply all replacements + new_content = content + for old_url, new_url in replacements.items(): + new_content = new_content.replace(old_url, new_url) + + return new_content + + +def remap_submodules_in_bare_repo(bare_repo_path, config, alias_resolver=None): + """Rewrite .gitmodules in all branches of a bare repo. + + Uses git plumbing to create deterministic commits (fixed timestamp) + so repeated runs produce identical hashes when nothing changed on + the source side — avoiding unnecessary force-pushes. + + Returns the number of branches remapped. + """ + try: + output = _git( + ["for-each-ref", "--format=%(refname)", "refs/heads/"], + cwd=bare_repo_path, + ) + except subprocess.CalledProcessError: + return 0 + + if not output.strip(): + return 0 + + # Fixed timestamp for this run — deterministic within a cycle but a real date + run_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00") + + # Shared cache for API results across all branches in this repo + alias_cache = {} + + remapped = 0 + for ref in output.strip().splitlines(): + if _remap_branch(bare_repo_path, ref, config, alias_resolver, alias_cache, run_timestamp): + remapped += 1 + + if remapped: + logger.info( + "Remapped submodule URLs on %d branch(es) in %s", + remapped, os.path.basename(bare_repo_path), + ) + + return remapped + + +def _remap_branch(bare_repo_path, ref, config, alias_resolver=None, alias_cache=None, run_timestamp=None): + """Remap .gitmodules on a single branch ref. Returns True if changed.""" + try: + content = _git(["show", f"{ref}:.gitmodules"], cwd=bare_repo_path) + except subprocess.CalledProcessError: + return False + + new_content = remap_submodule_urls(content, config, alias_resolver=alias_resolver, + _alias_cache=alias_cache) + if new_content == content: + return False + + blob_hash = _git( + ["hash-object", "-w", "--stdin"], + cwd=bare_repo_path, stdin_data=new_content, + ) + + tree_listing = _git(["ls-tree", ref], cwd=bare_repo_path) + new_lines = [] + for line in tree_listing.splitlines(): + if "\t.gitmodules" in line: + meta, _ = line.split("\t", 1) + parts = meta.split() + new_lines.append(f"{parts[0]} {parts[1]} {blob_hash}\t.gitmodules") + else: + new_lines.append(line) + + new_tree = _git( + ["mktree"], + cwd=bare_repo_path, stdin_data="\n".join(new_lines) + "\n", + ) + + parent = _git(["rev-parse", ref], cwd=bare_repo_path) + commit_env = { + **_REMAP_IDENTITY, + "GIT_AUTHOR_DATE": run_timestamp or datetime.now(timezone.utc).isoformat(), + "GIT_COMMITTER_DATE": run_timestamp or datetime.now(timezone.utc).isoformat(), + } + new_commit = _git( + ["commit-tree", new_tree, "-p", parent, "-m", _COMMIT_MSG], + cwd=bare_repo_path, env_extra=commit_env, + ) + + _git(["update-ref", ref, new_commit], cwd=bare_repo_path) + logger.debug("Remapped submodules on %s", ref) + return True diff --git a/bb2gh/syncer.py b/bb2gh/syncer.py new file mode 100644 index 0000000..6f943d0 --- /dev/null +++ b/bb2gh/syncer.py @@ -0,0 +1,245 @@ +"""Continuous sync from Bitbucket to GitHub.""" + +import logging +import os +import signal +import subprocess +import time + +from .bitbucket_client import BitbucketClient +from .migrator import _migrate_lfs, _has_large_blobs, _trim_history, _push_branch_by_branch, _push_lfs_objects +from .state import State +from .submodules import remap_submodules_in_bare_repo + +logger = logging.getLogger(__name__) + + +def _run_git(args, cwd=None, quiet=False): + """Run a git command and return stdout.""" + cmd = ["git"] + args + logger.debug("Running: %s", " ".join(cmd)) + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + if not quiet: + logger.error("git %s failed: %s", args[0], result.stderr.strip()) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + return result.stdout.strip() + + +def _clean_hidden_refs(bare_repo_path): + """Remove hidden refs that can't be pushed to GitHub.""" + try: + output = _run_git(["show-ref"], cwd=bare_repo_path, quiet=True) + except subprocess.CalledProcessError: + return + + for line in output.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + ref = parts[1] + if "/pull/" in ref or "/merge-request" in ref: + try: + _run_git(["update-ref", "-d", ref], cwd=bare_repo_path) + except subprocess.CalledProcessError: + pass + + +class Syncer: + """Continuously syncs migrated repos from Bitbucket to GitHub.""" + + def __init__(self, config): + self.config = config + self.state = State(config.work_dir) + self._bb = BitbucketClient(config.bb_base_url, config.bb_token, + verify_ssl=config.bb_verify_ssl) + self._running = True + + signal.signal(signal.SIGTERM, self._handle_signal) + signal.signal(signal.SIGINT, self._handle_signal) + + def _handle_signal(self, signum, frame): + logger.info("Received signal %d, shutting down gracefully...", signum) + self._running = False + + def run(self): + """Run the sync loop.""" + logger.info( + "Starting continuous sync (interval: %ds)", self.config.sync_interval + ) + + while self._running: + self._sync_all() + self._sleep(self.config.sync_interval) + + logger.info("Syncer stopped.") + + def _sleep(self, seconds): + """Interruptible sleep.""" + end = time.time() + seconds + while self._running and time.time() < end: + time.sleep(min(1, end - time.time())) + + def _sync_all(self): + """Sync all migrated repos.""" + repos = self.state.get_migrated_repos() + if not repos: + logger.warning("No migrated repos found. Run 'bb2gh migrate' first.") + return + + synced = 0 + skipped = 0 + failed = 0 + + for project_key, repo_slug in repos: + if not self._running: + break + if project_key.upper() in self.config.sync_exclude_projects: + continue + try: + changed = self._sync_repo(project_key, repo_slug) + if changed: + synced += 1 + time.sleep(self.config.migrate_delay) + else: + skipped += 1 + except Exception: + logger.exception("Failed to sync %s/%s", project_key, repo_slug) + failed += 1 + + logger.info( + "Sync cycle complete: %d synced, %d unchanged, %d failed", + synced, skipped, failed, + ) + + def _is_branch_protected(self, branch_name): + """Check if a branch matches any protected pattern (supports * glob).""" + from fnmatch import fnmatch + return any(fnmatch(branch_name, p) for p in self.config.sync_protected_branches) + + def _prune_unprotected_branches(self, bare_path, project_key, repo_slug): + """Delete remote branches on GitHub that don't exist locally, except protected ones.""" + # Get local branches (from Bitbucket) + local_output = _run_git( + ["for-each-ref", "--format=%(refname:short)", "refs/heads/"], + cwd=bare_path, quiet=True, + ) + local_branches = set(l.strip() for l in local_output.splitlines() if l.strip()) + + # Get remote branches on GitHub + try: + remote_output = _run_git( + ["ls-remote", "--heads", "github"], + cwd=bare_path, quiet=True, + ) + except subprocess.CalledProcessError: + return + + for line in remote_output.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + ref = parts[1].replace("refs/heads/", "") + if ref in local_branches or self._is_branch_protected(ref): + continue + try: + _run_git(["push", "github", "--delete", ref], cwd=bare_path, quiet=True) + logger.debug("Deleted remote branch %s (not in source, not protected)", ref) + except subprocess.CalledProcessError: + pass + + def _sync_repo(self, project_key, repo_slug): + """Sync a single repo: fetch from Bitbucket, push to GitHub only if changed.""" + bare_path = os.path.join( + self.config.work_dir, f"{project_key}__{repo_slug}.git" + ) + + if not os.path.exists(bare_path): + logger.error("Bare repo not found: %s", bare_path) + return + + # Look up the GitHub target from state (set during migration) + gh_org, gh_repo_name = self.state.get_github_target(project_key, repo_slug) + target_label = f"{gh_org}/{gh_repo_name}" if gh_org else "github" + + # Fetch from Bitbucket (origin) + _run_git(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=bare_path) + + # Clean hidden refs before snapshot (so snapshot is consistent) + _clean_hidden_refs(bare_path) + + # Compare Bitbucket's refs (post-fetch, post-clean) against last sync snapshot + # Only compare heads and tags — ignore refs/remotes/* which change on every push + try: + bb_heads = _run_git(["show-ref", "--heads", "--tags"], cwd=bare_path, quiet=True) + except subprocess.CalledProcessError: + bb_heads = "" + + refs_file = os.path.join(bare_path, "bb2gh_last_sync_refs") + last_refs = "" + if os.path.exists(refs_file): + with open(refs_file) as f: + last_refs = f.read() + + if bb_heads == last_refs: + logger.debug("No changes for %s/%s, skipping push", project_key, repo_slug) + return False + + logger.info("Changes detected for %s/%s, pushing...", project_key, repo_slug) + start = time.time() + + # Trim history if configured + trim_since = self.config.get_trim_since(project_key, repo_slug) + if trim_since: + _trim_history(bare_path, trim_since) + + # Remap submodule URLs from Bitbucket to GitHub + remap_submodules_in_bare_repo(bare_path, self.config, + alias_resolver=self._bb.resolve_repo_location) + + # LFS: only run if repo actually has large blobs (fast pre-check) + has_lfs = False + if self.config.lfs_enabled and _has_large_blobs(bare_path, self.config.lfs_threshold): + try: + has_lfs = _migrate_lfs(bare_path, self.config.lfs_threshold, timeout=self.config.sync_lfs_timeout) + except subprocess.TimeoutExpired: + logger.warning("LFS migration timed out for %s/%s, skipping LFS", project_key, repo_slug) + except Exception: + logger.warning("LFS migration failed for %s/%s, skipping LFS", project_key, repo_slug) + + # Push to GitHub + repo_key = f"{project_key}/{repo_slug}" + if repo_key in self.config.push_by_branch: + _push_branch_by_branch(bare_path, project_key, repo_slug, delay=self.config.migrate_delay) + elif self.config.sync_protected_branches: + # Can't use --mirror (it deletes branches not in source). + # Push all branches + tags, then prune only unprotected branches. + _run_git(["push", "github", "--all", "--force"], cwd=bare_path) + _run_git(["push", "github", "--tags", "--force"], cwd=bare_path) + self._prune_unprotected_branches(bare_path, project_key, repo_slug) + else: + _run_git(["push", "github", "--mirror"], cwd=bare_path) + + if has_lfs: + try: + _push_lfs_objects(bare_path) + except Exception: + logger.warning("LFS push failed for %s/%s", project_key, repo_slug) + + elapsed = time.time() - start + # Store Bitbucket's refs so next cycle can detect real changes + with open(refs_file, "w") as f: + f.write(bb_heads) + + self.state.update_sync_time(project_key, repo_slug) + logger.info( + "Synced %s/%s -> %s in %.1fs", + project_key, repo_slug, target_label, elapsed, + ) + return True diff --git a/config.yaml.example b/config.yaml.example new file mode 100644 index 0000000..0d5f151 --- /dev/null +++ b/config.yaml.example @@ -0,0 +1,115 @@ +bitbucket: + # Bitbucket Server base URL (no trailing slash) + base_url: "https://bitbucket.mycompany.com" + # Personal access token for REST API calls (or set BB_TOKEN env var) + token: "YOUR_BITBUCKET_TOKEN" + # SSH base URL for git clone operations + ssh_url: "ssh://git@bitbucket.mycompany.com:7999" + # Additional Bitbucket SSH hostnames (for submodule URL matching). + # Some .gitmodules may use a different hostname variant (e.g. with FQDN). + # The hostname from ssh_url is matched automatically — list extras here. + # ssh_hostnames: + # - "bitbucket.mycompany.com" + # - "bitbucket.internal.mycompany.com" + # Set to false if Bitbucket uses a self-signed or internal CA certificate + verify_ssl: true + # Optional: limit migration to specific projects (omit to migrate all) + # projects: + # - PROJ1 + # - PROJ2 + +github: + # GitHub Enterprise API base URL + base_url: "https://github.mycompany.com/api/v3" + # Personal access token with repo + admin:org scopes (or set GH_TOKEN env var) + token: "YOUR_GITHUB_TOKEN" + # Default target organization on GitHub (used when no project-specific mapping exists) + org: "my-org" + # SSH hostname for GitHub (used for submodule URL remapping) + ssh_host: "github.mycompany.com" + # SSH URL prefix for GitHub (used for push and submodule remapping) + # Use this if your GHE SSH user isn't "git" (e.g. ssh://myuser@host) + # ssh_url: "ssh://myuser@github.mycompany.com" + +sync: + # Sync interval in seconds + interval_seconds: 60 + # Local directory for bare repo clones + work_dir: "/data/mirror" + # Delay between repos during migration (seconds) to avoid SSH/API rate limits + migrate_delay_seconds: 2 + # Max time (seconds) for LFS migration during sync (default: 60) + sync_lfs_timeout_seconds: 60 + # Branches on GitHub to protect from sync deletion (e.g., CI migration branches) + # When set, sync uses --all --force instead of --mirror to preserve these branches + # protected_branches: + # - ci/github-actions-migration + # Projects to exclude from continuous sync (still migrated, just not synced) + # exclude_projects: + # - UPSTREAM + +# Optional: map Bitbucket projects/repos to specific GitHub orgs/repo names. +# Without this section, all repos go to github.org with their original slug as name. +repo_mapping: + # Default naming template for GitHub repos. + # Available variables: {project}, {project_lower}, {slug} + # Default: "{slug}" (just the Bitbucket repo slug) + name_template: "{project_lower}-{slug}" + + # Per-project overrides + projects: + INFRA: + # Send INFRA repos to a different GitHub org + github_org: "infra-team" + # Optional: override the name template for this project only + # name_template: "{slug}" + # + # Optional: migrate ONLY these repos from INFRA (allowlist). + # If set, all other repos in this project are skipped. + # include_repos: + # - my-service + # - my-api + repos: + # Optional: explicit per-repo name overrides + legacy-monolith: + github_name: "infra-monolith" + PLATFORM: + github_org: "platform-eng" + # Optional: migrate all repos EXCEPT these (denylist). + # Ignored if include_repos is set. + # exclude_repos: + # - deprecated-tool + # - archived-spike + # Projects not listed here use github.org and the global name_template + +# Optional: trim history for specific large repos. +# Only commits within the retention period are migrated/synced. +# HEAD commit hashes are preserved (uses git shallow clones). +# Format: PROJECT/slug: "Ny" (years), "Nm" (months), or "Nd" (days) +# trim_history: +# UPSTREAM/linux: "2y" +# UPSTREAM/git: "1y" + +# Optional: repos too large for --mirror push (>2GB pack). +# These are pushed branch-by-branch instead. +# push_by_branch: +# - UPSTREAM/linux +# - BGAN_UT_RM/bganut-linux + +# Optional: map old Bitbucket project keys to current keys. +# Used for .gitmodules remapping when URLs reference renamed projects. +# project_aliases: +# OLD_PROJECT_KEY: CURRENT_PROJECT_KEY + +# Optional: auto-convert large files to Git LFS before pushing to GitHub. +# GitHub rejects files >100MB. This rewrites history to store them as LFS objects. +lfs: + enabled: false + # Files above this size are converted to LFS pointers + threshold: "100mb" + +# Optional: map Bitbucket usernames to GitHub usernames +# Used for PR author attribution and reviewer assignments +# user_mapping: +# bb_user1: gh_user1 +# bb_user2: gh_user2 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..28d22e7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +services: + # One-time bulk migration + migrate: + build: . + command: ["migrate", "--config", "/config/config.yaml"] + volumes: + - ./config:/config:ro + - mirror-data:/data/mirror + - ${SSH_KEY_PATH:-~/.ssh}:/root/.ssh:ro + profiles: + - migrate + + # Continuous sync (runs as a long-lived service) + sync: + build: . + command: ["sync", "--config", "/config/config.yaml"] + volumes: + - ./config:/config:ro + - mirror-data:/data/mirror + - ${SSH_KEY_PATH:-~/.ssh}:/root/.ssh:ro + restart: unless-stopped + + # PR migration (one-time) + migrate-prs: + build: . + command: ["migrate-prs", "--config", "/config/config.yaml"] + volumes: + - ./config:/config:ro + - mirror-data:/data/mirror + - ${SSH_KEY_PATH:-~/.ssh}:/root/.ssh:ro + profiles: + - migrate-prs + +volumes: + mirror-data: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..860eb81 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +click>=8.0 +requests>=2.28 +PyGithub>=1.59 +pyyaml>=6.0 +gitpython>=3.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7081e10 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +from setuptools import setup, find_packages + +setup( + name="bb2gh", + version="1.0.0", + description="Bitbucket Server to GitHub Enterprise migration tool", + packages=find_packages(), + install_requires=[ + "click>=8.0", + "requests>=2.28", + "PyGithub>=1.59", + "pyyaml>=6.0", + "gitpython>=3.1", + ], + entry_points={ + "console_scripts": [ + "bb2gh=bb2gh.cli:cli", + ], + }, + python_requires=">=3.9", +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..8378668 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,249 @@ +"""Tests for the config module, especially repo_mapping resolution.""" + +import os +import tempfile + +import pytest +import yaml + +from bb2gh.config import Config + + +def _write_config(tmp_path, data): + """Write a config dict to a YAML file and return the path.""" + path = tmp_path / "config.yaml" + with open(path, "w") as f: + yaml.dump(data, f) + return str(path) + + +@pytest.fixture +def base_config(): + """Minimal valid config dict.""" + return { + "bitbucket": { + "base_url": "https://bitbucket.example.com", + "ssh_url": "ssh://git@bitbucket.example.com:7999", + "token": "fake", + }, + "github": { + "base_url": "https://github.example.com/api/v3", + "org": "default-org", + "token": "fake", + }, + } + + +class TestResolveTarget: + def test_defaults_to_slug_and_default_org(self, tmp_path, base_config): + """Without repo_mapping, returns default org and slug as-is.""" + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("PROJ", "my-repo") + assert org == "default-org" + assert name == "my-repo" + + def test_global_name_template(self, tmp_path, base_config): + """Global name_template applies to all repos.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("INFRA", "my-service") + assert org == "default-org" + assert name == "infra-my-service" + + def test_per_project_org(self, tmp_path, base_config): + """Per-project github_org overrides the default org.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": {"github_org": "infra-team"}, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("INFRA", "my-service") + assert org == "infra-team" + assert name == "my-service" # default template is "{slug}" + + def test_per_project_name_template(self, tmp_path, base_config): + """Per-project name_template overrides the global template.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + "projects": { + "INFRA": { + "github_org": "infra-team", + "name_template": "infra-{slug}", + }, + }, + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # INFRA uses its own template + org, name = config.resolve_target("INFRA", "my-service") + assert org == "infra-team" + assert name == "infra-my-service" + + # Other projects use the global template + org, name = config.resolve_target("PLATFORM", "api") + assert org == "default-org" + assert name == "platform-api" + + def test_per_repo_override(self, tmp_path, base_config): + """Explicit per-repo github_name overrides all templates.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + "projects": { + "INFRA": { + "github_org": "infra-team", + "repos": { + "legacy-monolith": {"github_name": "the-monolith"}, + }, + }, + }, + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # Explicit override + org, name = config.resolve_target("INFRA", "legacy-monolith") + assert org == "infra-team" + assert name == "the-monolith" + + # Non-overridden repo in same project uses global template + # (no per-project template set, so falls back to global) + org, name = config.resolve_target("INFRA", "other-repo") + assert org == "infra-team" + assert name == "infra-other-repo" + + def test_unmapped_project_uses_defaults(self, tmp_path, base_config): + """Projects not listed in repo_mapping use the default org and global template.""" + base_config["repo_mapping"] = { + "name_template": "{project_lower}-{slug}", + "projects": { + "INFRA": {"github_org": "infra-team"}, + }, + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("OTHER", "some-repo") + assert org == "default-org" + assert name == "other-some-repo" + + def test_template_with_project_variable(self, tmp_path, base_config): + """Template can use {project} (original case).""" + base_config["repo_mapping"] = { + "name_template": "{project}-{slug}", + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + org, name = config.resolve_target("MyProject", "api") + assert name == "MyProject-api" + + +class TestShouldMigrateRepo: + def test_no_filters_migrates_everything(self, tmp_path, base_config): + """Without include/exclude, all repos migrate.""" + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("PROJ", "any-repo") is True + assert config.should_migrate_repo("OTHER", "other-repo") is True + + def test_include_repos_acts_as_allowlist(self, tmp_path, base_config): + """include_repos limits migration to listed repos only.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": { + "include_repos": ["my-service", "my-api"], + }, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("INFRA", "my-service") is True + assert config.should_migrate_repo("INFRA", "my-api") is True + assert config.should_migrate_repo("INFRA", "other-repo") is False + + def test_exclude_repos_acts_as_denylist(self, tmp_path, base_config): + """exclude_repos skips listed repos, migrates the rest.""" + base_config["repo_mapping"] = { + "projects": { + "PLATFORM": { + "exclude_repos": ["deprecated-tool", "archived-spike"], + }, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("PLATFORM", "api-gateway") is True + assert config.should_migrate_repo("PLATFORM", "deprecated-tool") is False + assert config.should_migrate_repo("PLATFORM", "archived-spike") is False + + def test_include_takes_precedence_over_exclude(self, tmp_path, base_config): + """When both are set, include_repos wins (exclude is ignored).""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": { + "include_repos": ["my-service"], + "exclude_repos": ["my-service"], # should be ignored + }, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # include wins + assert config.should_migrate_repo("INFRA", "my-service") is True + assert config.should_migrate_repo("INFRA", "other") is False + + def test_empty_include_skips_everything(self, tmp_path, base_config): + """An empty include_repos list means migrate nothing from that project.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": {"include_repos": []}, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + assert config.should_migrate_repo("INFRA", "any-repo") is False + + def test_filters_scoped_to_project(self, tmp_path, base_config): + """Filters on one project don't affect other projects.""" + base_config["repo_mapping"] = { + "projects": { + "INFRA": {"include_repos": ["my-service"]}, + } + } + path = _write_config(tmp_path, base_config) + config = Config(path) + + # INFRA is filtered + assert config.should_migrate_repo("INFRA", "my-service") is True + assert config.should_migrate_repo("INFRA", "other") is False + # OTHER project has no filter, migrates everything + assert config.should_migrate_repo("OTHER", "anything") is True + + +class TestConfigValidation: + def test_missing_bitbucket_section(self, tmp_path): + data = {"github": {"base_url": "x", "org": "y"}} + path = _write_config(tmp_path, data) + with pytest.raises(ValueError, match="bitbucket"): + Config(path) + + def test_missing_github_section(self, tmp_path): + data = {"bitbucket": {"base_url": "x", "ssh_url": "y"}} + path = _write_config(tmp_path, data) + with pytest.raises(ValueError, match="github"): + Config(path) diff --git a/tests/test_jenkins_prep.py b/tests/test_jenkins_prep.py new file mode 100644 index 0000000..9a521c5 --- /dev/null +++ b/tests/test_jenkins_prep.py @@ -0,0 +1,123 @@ +"""Tests for Jenkins workspace preparation.""" + +from unittest.mock import MagicMock + +import pytest + +from bb2gh.jenkins_prep import find_jenkins_files, parse_jenkinsfile_refs + + +class TestFindJenkinsFiles: + def test_finds_jenkinsfile(self): + paths = ["README.md", "Jenkinsfile", "src/main.py"] + jf, deps = find_jenkins_files(paths) + assert jf == ["Jenkinsfile"] + assert deps == [] + + def test_finds_jenkinsfile_case_insensitive(self): + paths = ["jenkinsfile", "JENKINSFILE"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + + def test_finds_jenkinsfile_with_suffix(self): + paths = ["Jenkinsfile.deploy", "Jenkinsfile.staging", "README.md"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + assert "Jenkinsfile.deploy" in jf + assert "Jenkinsfile.staging" in jf + + def test_finds_dot_jenkinsfile(self): + paths = ["pipelines/build.jenkinsfile", "deploy.jenkinsfile"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + + def test_finds_nested_jenkinsfile(self): + paths = ["ci/Jenkinsfile", "ci/pipelines/Jenkinsfile.nightly"] + jf, deps = find_jenkins_files(paths) + assert len(jf) == 2 + + def test_finds_vars_directory(self): + paths = ["Jenkinsfile", "vars/myHelper.groovy", "vars/deploy.groovy"] + jf, deps = find_jenkins_files(paths) + assert jf == ["Jenkinsfile"] + assert "vars/myHelper.groovy" in deps + assert "vars/deploy.groovy" in deps + + def test_finds_src_groovy(self): + paths = ["Jenkinsfile", "src/org/company/Pipeline.groovy"] + jf, deps = find_jenkins_files(paths) + assert "src/org/company/Pipeline.groovy" in deps + + def test_finds_resources(self): + paths = ["Jenkinsfile", "resources/config.yaml"] + jf, deps = find_jenkins_files(paths) + assert "resources/config.yaml" in deps + + def test_finds_colocated_groovy(self): + paths = ["ci/Jenkinsfile", "ci/helpers.groovy", "other/utils.groovy"] + jf, deps = find_jenkins_files(paths) + assert "ci/helpers.groovy" in deps + assert "other/utils.groovy" not in deps + + def test_no_jenkinsfiles(self): + paths = ["README.md", "src/main.py", "Makefile"] + jf, deps = find_jenkins_files(paths) + assert jf == [] + assert deps == [] + + def test_empty_paths(self): + jf, deps = find_jenkins_files([]) + assert jf == [] + assert deps == [] + + +class TestParseJenkinsfileRefs: + def test_extracts_load_single_quotes(self): + content = "load 'scripts/deploy.groovy'" + refs = parse_jenkinsfile_refs(content) + assert "scripts/deploy.groovy" in refs + + def test_extracts_load_double_quotes(self): + content = 'load "scripts/deploy.groovy"' + refs = parse_jenkinsfile_refs(content) + assert "scripts/deploy.groovy" in refs + + def test_extracts_readfile(self): + content = "def config = readFile('config/settings.yaml')" + refs = parse_jenkinsfile_refs(content) + assert "config/settings.yaml" in refs + + def test_extracts_evaluate_readfile(self): + content = """evaluate(readFile('scripts/helper.groovy'))""" + refs = parse_jenkinsfile_refs(content) + assert "scripts/helper.groovy" in refs + + def test_multiple_references(self): + content = """ + load 'scripts/build.groovy' + def cfg = readFile('config.yaml') + load "scripts/deploy.groovy" + """ + refs = parse_jenkinsfile_refs(content) + assert len(refs) == 3 + assert "scripts/build.groovy" in refs + assert "config.yaml" in refs + assert "scripts/deploy.groovy" in refs + + def test_no_references(self): + content = """ + pipeline { + agent any + stages { + stage('Build') { + steps { sh 'make build' } + } + } + } + """ + refs = parse_jenkinsfile_refs(content) + assert refs == set() + + def test_empty_content(self): + refs = parse_jenkinsfile_refs("") + assert refs == set() diff --git a/tests/test_migrator.py b/tests/test_migrator.py new file mode 100644 index 0000000..0888b53 --- /dev/null +++ b/tests/test_migrator.py @@ -0,0 +1,202 @@ +"""Tests for the bulk migrator module.""" + +import os +import json +import tempfile +from unittest.mock import MagicMock, patch + +import pytest + +from bb2gh.config import Config +from bb2gh.migrator import migrate_repos, _clean_hidden_refs + + +@pytest.fixture +def mock_config(tmp_path): + """Create a mock config object.""" + config = MagicMock(spec=Config) + config.bb_base_url = "https://bitbucket.example.com" + config.bb_token = "fake-bb-token" + config.bb_ssh_url = "ssh://git@bitbucket.example.com:7999" + config.bb_projects = ["PROJ1"] + config.bb_verify_ssl = True + config.gh_base_url = "https://github.example.com/api/v3" + config.gh_token = "fake-gh-token" + config.gh_org = "my-org" + config.work_dir = str(tmp_path) + config.user_mapping = {} + # Default resolve_target returns the default org with slug as name + config.resolve_target = MagicMock(side_effect=lambda proj, slug: ("my-org", slug)) + # Default: no filtering — all repos migrate + config.should_migrate_repo = MagicMock(return_value=True) + config.lfs_enabled = False + config.lfs_threshold = "100mb" + config.gh_ssh_host = "" + config.gh_ssh_url = "" + config.migrate_delay = 0 + config.get_trim_since = MagicMock(return_value=None) + config.push_by_branch = set() + return config + + +class TestCleanHiddenRefs: + @patch("bb2gh.migrator._run_git") + def test_removes_pull_refs(self, mock_git): + mock_git.return_value = ( + "abc123 refs/heads/main\n" + "def456 refs/pull/1/head\n" + "ghi789 refs/pull/2/head\n" + ) + + _clean_hidden_refs("/fake/repo.git") + + # Should call show-ref once, then update-ref -d for each pull ref + assert mock_git.call_count == 3 + mock_git.assert_any_call(["update-ref", "-d", "refs/pull/1/head"], cwd="/fake/repo.git") + mock_git.assert_any_call(["update-ref", "-d", "refs/pull/2/head"], cwd="/fake/repo.git") + + @patch("bb2gh.migrator._run_git") + def test_no_hidden_refs(self, mock_git): + mock_git.return_value = "abc123 refs/heads/main\ndef456 refs/tags/v1.0\n" + + _clean_hidden_refs("/fake/repo.git") + + # Only show-ref called, no deletions + assert mock_git.call_count == 1 + + +class TestMigrateRepos: + @patch("bb2gh.migrator.remap_submodules_in_bare_repo", return_value=0) + @patch("bb2gh.migrator.State") + @patch("bb2gh.migrator.GithubClient") + @patch("bb2gh.migrator.BitbucketClient") + @patch("bb2gh.migrator._run_git") + def test_migrates_new_repo(self, mock_git, MockBB, MockGH, MockState, mock_remap, mock_config): + # Setup mocks + bb_instance = MockBB.return_value + bb_instance.list_repos.return_value = [ + { + "slug": "my-repo", + "name": "My Repo", + "description": "A test repo", + "links": {"clone": [{"name": "ssh", "href": "ssh://git@bb:7999/proj1/my-repo.git"}]}, + } + ] + + gh_instance = MockGH.return_value + gh_instance.get_clone_url.return_value = "https://github.example.com/my-org/my-repo.git" + + state_instance = MockState.return_value + state_instance.is_migrated.return_value = False + + mock_git.return_value = "" + + migrated, skipped, failed = migrate_repos(mock_config) + + assert migrated == 1 + assert skipped == 0 + assert failed == 0 + mock_config.resolve_target.assert_called_once_with("PROJ1", "my-repo") + state_instance.mark_migrated.assert_called_once() + call_kwargs = state_instance.mark_migrated.call_args + assert call_kwargs[0][:2] == ("PROJ1", "my-repo") + assert call_kwargs[1]["gh_org"] == "my-org" + assert call_kwargs[1]["gh_repo_name"] == "my-repo" + + @patch("bb2gh.migrator.remap_submodules_in_bare_repo", return_value=0) + @patch("bb2gh.migrator.State") + @patch("bb2gh.migrator.GithubClient") + @patch("bb2gh.migrator.BitbucketClient") + @patch("bb2gh.migrator._run_git") + def test_migrates_to_mapped_org(self, mock_git, MockBB, MockGH, MockState, mock_remap, mock_config): + """Test that repos are migrated to the correct org when mapping is configured.""" + # Override resolve_target to return a different org + mock_config.resolve_target = MagicMock( + side_effect=lambda proj, slug: ("infra-team", f"infra-{slug}") + ) + + bb_instance = MockBB.return_value + bb_instance.list_repos.return_value = [ + { + "slug": "my-repo", + "name": "My Repo", + "description": "A test repo", + "links": {"clone": [{"name": "ssh", "href": "ssh://git@bb:7999/proj1/my-repo.git"}]}, + } + ] + + gh_instance = MockGH.return_value + gh_instance.get_clone_url.return_value = "https://github.example.com/infra-team/infra-my-repo.git" + + state_instance = MockState.return_value + state_instance.is_migrated.return_value = False + + mock_git.return_value = "" + + migrated, skipped, failed = migrate_repos(mock_config) + + assert migrated == 1 + # GitHub repo should be created with the mapped name and org + gh_instance.create_repo.assert_called_once_with( + "infra-my-repo", description="A test repo", private=True, org_name="infra-team" + ) + gh_instance.get_clone_url.assert_called_once_with("infra-my-repo", org_name="infra-team", ssh_url=None) + state_instance.mark_migrated.assert_called_once() + call_kwargs = state_instance.mark_migrated.call_args + assert call_kwargs[1]["gh_org"] == "infra-team" + assert call_kwargs[1]["gh_repo_name"] == "infra-my-repo" + + @patch("bb2gh.migrator.State") + @patch("bb2gh.migrator.GithubClient") + @patch("bb2gh.migrator.BitbucketClient") + def test_skips_already_migrated(self, MockBB, MockGH, MockState, mock_config): + bb_instance = MockBB.return_value + bb_instance.list_repos.return_value = [{"slug": "my-repo", "name": "My Repo"}] + + state_instance = MockState.return_value + state_instance.is_migrated.return_value = True + + migrated, skipped, failed = migrate_repos(mock_config) + + assert migrated == 0 + assert skipped == 1 + assert failed == 0 + + @patch("bb2gh.migrator.remap_submodules_in_bare_repo", return_value=0) + @patch("bb2gh.migrator.State") + @patch("bb2gh.migrator.GithubClient") + @patch("bb2gh.migrator.BitbucketClient") + @patch("bb2gh.migrator._run_git") + def test_respects_repo_filter(self, mock_git, MockBB, MockGH, MockState, mock_remap, mock_config): + """Test that repos filtered out by include/exclude_repos are skipped.""" + # Only allow "keep-me" through the filter + mock_config.should_migrate_repo = MagicMock( + side_effect=lambda proj, slug: slug == "keep-me" + ) + + bb_instance = MockBB.return_value + bb_instance.list_repos.return_value = [ + {"slug": "keep-me", "name": "Keep", + "links": {"clone": [{"name": "ssh", "href": "ssh://bb/p/keep-me.git"}]}}, + {"slug": "skip-me", "name": "Skip"}, + {"slug": "skip-also", "name": "Skip Also"}, + ] + + gh_instance = MockGH.return_value + gh_instance.get_clone_url.return_value = "https://github/my-org/keep-me.git" + + state_instance = MockState.return_value + state_instance.is_migrated.return_value = False + + mock_git.return_value = "" + + migrated, skipped, failed = migrate_repos(mock_config) + + assert migrated == 1 + assert skipped == 2 # two filtered out + assert failed == 0 + gh_instance.create_repo.assert_called_once() + state_instance.mark_migrated.assert_called_once() + call_kwargs = state_instance.mark_migrated.call_args + assert call_kwargs[0][:2] == ("PROJ1", "keep-me") + assert call_kwargs[1]["gh_org"] == "my-org" diff --git a/tests/test_pr_migrator.py b/tests/test_pr_migrator.py new file mode 100644 index 0000000..393b982 --- /dev/null +++ b/tests/test_pr_migrator.py @@ -0,0 +1,544 @@ +"""Tests for the PR migrator module.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from bb2gh.config import Config +from bb2gh.pr_migrator import ( + migrate_pull_requests, + Throttler, + _format_pr_body, + _format_comment, + _map_reviewers, + _iter_comment_activities, +) + + +@pytest.fixture(autouse=True) +def fast_throttler(monkeypatch): + """Ensure Throttler defaults to zero-delay for all tests.""" + original_init = Throttler.__init__ + + def zero_init(self, api_delay=0.5, pr_delay=3.0, max_retries=5): + original_init(self, api_delay=0, pr_delay=0, max_retries=max_retries) + + monkeypatch.setattr(Throttler, "__init__", zero_init) + + +@pytest.fixture +def mock_config(): + config = MagicMock(spec=Config) + config.bb_base_url = "https://bitbucket.example.com" + config.bb_token = "fake-token" + config.bb_verify_ssl = True + config.gh_base_url = "https://github.example.com/api/v3" + config.gh_token = "fake-gh-token" + config.gh_org = "my-org" + config.work_dir = "/tmp/test" + config.user_mapping = {"john.doe": "johndoe"} + config.resolve_target = MagicMock(return_value=("my-org", "my-repo")) + return config + + +@pytest.fixture +def sample_pr(): + return { + "id": 42, + "title": "Fix login bug", + "description": "This fixes the login timeout issue.", + "author": { + "user": {"name": "john.doe", "displayName": "John Doe"} + }, + "fromRef": { + "displayId": "fix/login-bug", + "repository": {"slug": "my-repo", "project": {"key": "PROJ"}}, + }, + "toRef": { + "displayId": "main", + "repository": {"slug": "my-repo", "project": {"key": "PROJ"}}, + }, + "reviewers": [ + {"user": {"name": "john.doe", "displayName": "John Doe"}}, + {"user": {"name": "jane.smith", "displayName": "Jane Smith"}}, + ], + "createdDate": 1711234567000, + } + + +class TestFormatPrBody: + def test_includes_metadata(self, sample_pr, mock_config): + body = _format_pr_body(sample_pr, mock_config) + + assert "Migrated from Bitbucket" in body + assert "John Doe" in body + assert "PROJ" in body + assert "my-repo" in body + assert "42" in body + assert "This fixes the login timeout issue." in body + + def test_handles_empty_description(self, sample_pr, mock_config): + sample_pr["description"] = None + body = _format_pr_body(sample_pr, mock_config) + + assert "Migrated from Bitbucket" in body + + +class TestFormatComment: + def test_formats_comment(self, mock_config): + activity = { + "action": "COMMENTED", + "comment": { + "author": {"name": "jane", "displayName": "Jane Smith"}, + "text": "Looks good to me!", + "createdDate": 1711234567000, + }, + } + + result = _format_comment(activity, mock_config) + + assert "Jane Smith" in result + assert "Looks good to me!" in result + + +class TestMapReviewers: + def test_maps_known_users(self, sample_pr, mock_config): + reviewers = _map_reviewers(sample_pr, mock_config) + + assert "johndoe" in reviewers # mapped + assert "jane.smith" in reviewers # passthrough (no mapping) + + def test_empty_reviewers(self, mock_config): + pr = {"reviewers": []} + assert _map_reviewers(pr, mock_config) == [] + + +class TestMigratePullRequests: + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_dry_run(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.return_value = [sample_pr] + + migrated, skipped, failed = migrate_pull_requests(mock_config, dry_run=True) + + assert migrated == 1 + assert skipped == 0 + assert failed == 0 + + # GitHub should NOT have been called in dry run + MockGH.return_value.create_pull_request.assert_not_called() + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_skips_already_migrated(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = True + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.return_value = [sample_pr] + + migrated, skipped, failed = migrate_pull_requests(mock_config, dry_run=False) + + assert migrated == 0 + assert skipped == 1 + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_migrates_pr_with_comments(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.return_value = [sample_pr] + bb_instance.get_pr_activities.return_value = [ + { + "action": "COMMENTED", + "comment": { + "author": {"name": "reviewer", "displayName": "Reviewer"}, + "text": "LGTM", + "createdDate": 1711234567000, + }, + }, + {"action": "APPROVED"}, # Non-comment activity, should be skipped + ] + + mock_pr = MagicMock() + mock_pr.number = 99 + MockGH.return_value.create_pull_request.return_value = mock_pr + + migrated, skipped, failed = migrate_pull_requests(mock_config, dry_run=False) + + assert migrated == 1 + assert failed == 0 + + # Verify PR was created with correct org + MockGH.return_value.create_pull_request.assert_called_once() + call_kwargs = MockGH.return_value.create_pull_request.call_args + assert call_kwargs[1]["org_name"] == "my-org" + assert call_kwargs[1]["repo_name"] == "my-repo" + + # Verify comment was added (only 1 — the APPROVED activity is skipped) + MockGH.return_value.add_pr_comment.assert_called_once() + + # Verify state recorded + state_instance.record_pr_mapping.assert_called_once_with("PROJ", "my-repo", 42, 99) + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_uses_mapped_org_for_pr(self, MockBB, MockGH, MockState, mock_config, sample_pr): + """Test that PRs are created in the correct mapped org.""" + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + # Simulate a repo migrated to a different org + state_instance.get_github_target.return_value = ("infra-team", "infra-my-repo") + state_instance.is_pr_migrated.return_value = False + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.return_value = [sample_pr] + bb_instance.get_pr_activities.return_value = [] + + mock_pr = MagicMock() + mock_pr.number = 5 + MockGH.return_value.create_pull_request.return_value = mock_pr + + migrated, skipped, failed = migrate_pull_requests(mock_config, dry_run=False) + + assert migrated == 1 + + # Verify PR was created in the mapped org with the mapped repo name + call_kwargs = MockGH.return_value.create_pull_request.call_args + assert call_kwargs[1]["org_name"] == "infra-team" + assert call_kwargs[1]["repo_name"] == "infra-my-repo" + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_fallback_to_config_resolve(self, MockBB, MockGH, MockState, mock_config, sample_pr): + """Test fallback to config.resolve_target when state has no GitHub target.""" + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + # Simulate old state without gh_org/gh_repo_name + state_instance.get_github_target.return_value = (None, None) + state_instance.is_pr_migrated.return_value = False + + mock_config.resolve_target.return_value = ("fallback-org", "fallback-repo") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.return_value = [sample_pr] + bb_instance.get_pr_activities.return_value = [] + + mock_pr = MagicMock() + mock_pr.number = 10 + MockGH.return_value.create_pull_request.return_value = mock_pr + + migrated, _, _ = migrate_pull_requests(mock_config, dry_run=False) + + assert migrated == 1 + mock_config.resolve_target.assert_called_once_with("PROJ", "my-repo") + call_kwargs = MockGH.return_value.create_pull_request.call_args + assert call_kwargs[1]["org_name"] == "fallback-org" + assert call_kwargs[1]["repo_name"] == "fallback-repo" + + +class TestIterCommentActivities: + def test_filters_and_sorts_chronologically(self): + activities = [ + {"action": "COMMENTED", "comment": {"text": "second", "createdDate": 200}}, + {"action": "APPROVED"}, + {"action": "COMMENTED", "comment": {"text": "first", "createdDate": 100}}, + {"action": "MERGED"}, + {"action": "COMMENTED", "comment": {"text": "third", "createdDate": 300}}, + ] + + result = list(_iter_comment_activities(activities)) + + assert len(result) == 3 + assert result[0]["comment"]["text"] == "first" + assert result[1]["comment"]["text"] == "second" + assert result[2]["comment"]["text"] == "third" + + def test_returns_empty_when_no_comments(self): + activities = [{"action": "APPROVED"}, {"action": "MERGED"}] + assert list(_iter_comment_activities(activities)) == [] + + +class TestMigrateClosedAsIssue: + """Closed PRs are migrated as closed GitHub Issues (no branch recreation).""" + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_closed_pr_becomes_closed_issue(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + merged_pr = dict(sample_pr, id=99, state="MERGED", title="Old merged PR") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "MERGED": [merged_pr], "DECLINED": [], + }.get(state, []) + bb_instance.get_pr_activities.return_value = [ + { + "action": "COMMENTED", + "comment": { + "author": {"name": "reviewer", "displayName": "Reviewer"}, + "text": "Great work", + "createdDate": 1711234567000, + }, + }, + ] + + mock_issue = MagicMock() + mock_issue.number = 42 + mock_repo = MagicMock() + mock_repo.create_issue.return_value = mock_issue + MockGH.return_value.get_repo.return_value = mock_repo + + migrated, _, failed = migrate_pull_requests( + mock_config, dry_run=False, closed_only=True, + ) + + assert migrated == 1 + assert failed == 0 + + # No PR created — it's an issue + MockGH.return_value.create_pull_request.assert_not_called() + + # Issue was created with the right title prefix + mock_repo.create_issue.assert_called_once() + issue_kwargs = mock_repo.create_issue.call_args.kwargs + assert "MERGED PR #99" in issue_kwargs["title"] + assert "Old merged PR" in issue_kwargs["title"] + assert "migrated-pr" in issue_kwargs["labels"] + assert "merged" in issue_kwargs["labels"] + + # Comment was added on the issue + mock_issue.create_comment.assert_called_once() + + # Issue was closed + mock_issue.edit.assert_called_once_with(state="closed") + + # State recorded the mapping + state_instance.record_pr_mapping.assert_called_once_with( + "PROJ", "my-repo", 99, 42, + ) + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_declined_pr_labeled_declined(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + declined_pr = dict(sample_pr, id=7, state="DECLINED", title="Won't fix") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "MERGED": [], "DECLINED": [declined_pr], + }.get(state, []) + bb_instance.get_pr_activities.return_value = [] + + mock_issue = MagicMock() + mock_issue.number = 8 + mock_repo = MagicMock() + mock_repo.create_issue.return_value = mock_issue + MockGH.return_value.get_repo.return_value = mock_repo + + migrate_pull_requests(mock_config, closed_only=True) + + issue_kwargs = mock_repo.create_issue.call_args.kwargs + assert "declined" in issue_kwargs["labels"] + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_falls_back_when_labels_missing(self, MockBB, MockGH, MockState, mock_config, sample_pr): + """If the labels don't exist on the repo yet, create the issue anyway.""" + from github import GithubException + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + merged_pr = dict(sample_pr, id=11, state="MERGED", title="Merged x") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "MERGED": [merged_pr], "DECLINED": [], + }.get(state, []) + bb_instance.get_pr_activities.return_value = [] + + mock_issue = MagicMock() + mock_issue.number = 12 + mock_repo = MagicMock() + # First call (with labels) fails 422; second call (no labels) succeeds + mock_repo.create_issue.side_effect = [ + GithubException(422, {"message": "label not found"}, {}), + mock_issue, + ] + MockGH.return_value.get_repo.return_value = mock_repo + + migrated, _, failed = migrate_pull_requests(mock_config, closed_only=True) + + assert migrated == 1 + assert failed == 0 + assert mock_repo.create_issue.call_count == 2 + # Second call had no labels kwarg + second_call = mock_repo.create_issue.call_args_list[1] + assert "labels" not in second_call.kwargs + + +class TestFormatPrBodyClosed: + def test_includes_closed_state(self, sample_pr, mock_config): + body = _format_pr_body(sample_pr, mock_config, closed_state="MERGED") + assert "MERGED" in body + assert "Original status" in body + + def test_no_status_for_open(self, sample_pr, mock_config): + body = _format_pr_body(sample_pr, mock_config) + assert "Original status" not in body + + +class TestMigrateClosedDryRun: + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_dry_run_includes_closed_prs(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + merged_pr = dict(sample_pr, id=99, state="MERGED", + title="Already merged") + merged_pr["fromRef"] = dict(sample_pr["fromRef"], latestCommit="aaa") + + bb_instance = MockBB.return_value + bb_instance.list_pull_requests.side_effect = lambda proj, repo, state: { + "OPEN": [sample_pr], + "MERGED": [merged_pr], + "DECLINED": [], + }[state] + + migrated, skipped, failed = migrate_pull_requests( + mock_config, dry_run=True, include_closed=True, + ) + + assert migrated == 2 + assert skipped == 0 + MockGH.return_value.create_pull_request.assert_not_called() + + @patch("bb2gh.pr_migrator.State") + @patch("bb2gh.pr_migrator.GithubClient") + @patch("bb2gh.pr_migrator.BitbucketClient") + def test_closed_only_skips_open(self, MockBB, MockGH, MockState, mock_config, sample_pr): + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + state_instance.is_pr_migrated.return_value = False + + merged_pr = dict(sample_pr, id=99, state="MERGED", title="Old merged PR") + merged_pr["fromRef"] = dict(sample_pr["fromRef"], latestCommit="aaa") + + bb_instance = MockBB.return_value + calls = [] + def list_prs(proj, repo, state): + calls.append(state) + return {"MERGED": [merged_pr], "DECLINED": []}.get(state, []) + bb_instance.list_pull_requests.side_effect = list_prs + + migrated, _, _ = migrate_pull_requests( + mock_config, dry_run=True, closed_only=True, + ) + + # Only one PR (the merged one), OPEN was never queried + assert migrated == 1 + assert "OPEN" not in calls + assert "MERGED" in calls + assert "DECLINED" in calls + + +class TestThrottler: + def test_call_retries_on_rate_limit(self, monkeypatch): + from bb2gh.pr_migrator import Throttler + import github + + throttler = Throttler(api_delay=0, pr_delay=0, max_retries=3) + + # Fake rate-limit exception on first two calls, success on third + attempts = {"n": 0} + def flaky(): + attempts["n"] += 1 + if attempts["n"] < 3: + exc = github.GithubException(429, {"message": "too fast"}, {"Retry-After": "0"}) + raise exc + return "ok" + + result = throttler.call(flaky) + assert result == "ok" + assert attempts["n"] == 3 + + def test_call_raises_after_max_retries(self): + from bb2gh.pr_migrator import Throttler + import github + + throttler = Throttler(api_delay=0, pr_delay=0, max_retries=2) + + def always_fail(): + raise github.GithubException(429, {"message": "no"}, {"Retry-After": "0"}) + + with pytest.raises(github.GithubException): + throttler.call(always_fail) + + def test_call_reraises_non_rate_limit(self): + from bb2gh.pr_migrator import Throttler + import github + + throttler = Throttler(api_delay=0, pr_delay=0, max_retries=3) + + def not_found(): + raise github.GithubException(404, {"message": "gone"}, {}) + + with pytest.raises(github.GithubException) as exc_info: + throttler.call(not_found) + assert exc_info.value.status == 404 + + def test_wait_api_spacing(self, monkeypatch): + from bb2gh.pr_migrator import Throttler + + sleeps = [] + monkeypatch.setattr("bb2gh.pr_migrator.time.sleep", lambda s: sleeps.append(s)) + + # First call returns 100.1 (elapsed check), second returns 100.5 (record) + times = iter([100.1, 100.5]) + monkeypatch.setattr( + "bb2gh.pr_migrator.time.monotonic", lambda: next(times), + ) + + throttler = Throttler(api_delay=0.5, pr_delay=0, max_retries=0) + throttler.api_delay = 0.5 # override autouse zeroing + throttler._last_call = 100.0 # simulate a previous call at t=100.0 + throttler.wait_api() + + # elapsed = 100.1 - 100.0 = 0.1, so remaining = 0.5 - 0.1 = 0.4 + assert sleeps and abs(sleeps[0] - 0.4) < 0.01 diff --git a/tests/test_submodules.py b/tests/test_submodules.py new file mode 100644 index 0000000..bb4ab59 --- /dev/null +++ b/tests/test_submodules.py @@ -0,0 +1,225 @@ +"""Tests for submodule URL remapping.""" + +from unittest.mock import MagicMock + +import pytest + +from bb2gh.config import Config +from bb2gh.submodules import remap_submodule_urls + + +@pytest.fixture +def mock_config(): + config = MagicMock(spec=Config) + config.bb_ssh_url = "ssh://git@cph1-eud-rep001:7999" + config.bb_base_url = "https://cph1-eud-rep001" + config.bb_ssh_hostnames = ["cph1-eud-rep001.satcom.global"] + config.gh_base_url = "https://gatehousesatcom.ghe.com/api/v3" + config.bb_projects = ["SYS_YAHSAT_NGSP", "SYS_COM", "DCKR", "NGRM"] + config.bb_verify_ssl = True + config.gh_ssh_host = "gatehousesatcom.ghe.com" + config.gh_ssh_url = "ssh://gatehousesatcom@gatehousesatcom.ghe.com" + config.project_aliases = {} + config.should_migrate_repo = MagicMock(return_value=True) + config.resolve_target = MagicMock( + side_effect=lambda proj, slug: ("networks-ngsp", slug) + ) + return config + + +class TestRemapSubmoduleUrls: + def test_remaps_ssh_urls_to_ssh(self, mock_config): + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_lib.git\n" + "[submodule \"cai_def\"]\n" + "\tpath = cai_def\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_def.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/cai_def.git" in result + assert "cph1-eud-rep001" not in result + + def test_remaps_fqdn_hostname_variant(self, mock_config): + """URLs using the FQDN variant should also be matched.""" + content = ( + "[submodule \"lib\"]\n" + "\tpath = lib\n" + "\turl = ssh://git@cph1-eud-rep001.satcom.global:7999/sys_yahsat_ngsp/lib.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/lib.git" in result + assert "cph1-eud-rep001" not in result + + def test_remaps_http_urls_to_https(self, mock_config): + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = https://cph1-eud-rep001/scm/sys_yahsat_ngsp/cai_lib.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "https://gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + + def test_preserves_non_url_lines(self, mock_config): + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/cai_lib.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert '[submodule "cai_lib"]' in result + assert "\tpath = cai_lib" in result + + def test_skips_entirely_when_url_unresolvable(self, mock_config): + """If any BB URL can't be resolved, return content unchanged.""" + mock_config.bb_projects = ["SYS_YAHSAT_NGSP"] + content = ( + "[submodule \"known\"]\n" + "\tpath = known\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/known.git\n" + "[submodule \"unknown\"]\n" + "\tpath = unknown\n" + "\turl = ssh://git@cph1-eud-rep001:7999/other_project/unknown.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + # Entire file should be unchanged + assert result == content + + def test_skips_entirely_when_excluded_repo(self, mock_config): + """If any BB URL points to an excluded repo, skip entirely.""" + mock_config.should_migrate_repo = MagicMock( + side_effect=lambda proj, slug: slug != "excluded-repo" + ) + content = ( + "[submodule \"included\"]\n" + "\tpath = included\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/included.git\n" + "[submodule \"excluded\"]\n" + "\tpath = excluded\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/excluded-repo.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert result == content + + def test_ignores_already_github_urls(self, mock_config): + """URLs already pointing to GitHub should be ignored.""" + content = ( + "[submodule \"bb_repo\"]\n" + "\tpath = bb_repo\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/bb_repo.git\n" + "[submodule \"gh_repo\"]\n" + "\tpath = gh_repo\n" + "\turl = ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/gh_repo.git" in result + + def test_ignores_external_urls(self, mock_config): + """URLs pointing to external hosts (not BB) should be left alone.""" + content = ( + "[submodule \"bb_repo\"]\n" + "\tpath = bb_repo\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/bb_repo.git\n" + "[submodule \"external\"]\n" + "\tpath = external\n" + "\turl = https://github.com/some/external.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/bb_repo.git" in result + assert "https://github.com/some/external.git" in result + + def test_handles_uppercase_project_in_url(self, mock_config): + content = ( + "[submodule \"cai_lib\"]\n" + "\tpath = cai_lib\n" + "\turl = ssh://git@cph1-eud-rep001:7999/SYS_YAHSAT_NGSP/cai_lib.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/cai_lib.git" in result + + def test_empty_content(self, mock_config): + assert remap_submodule_urls("", mock_config) == "" + + def test_no_matching_urls(self, mock_config): + content = ( + "[submodule \"lib\"]\n" + "\tpath = lib\n" + "\turl = https://github.com/some/other.git\n" + ) + result = remap_submodule_urls(content, mock_config) + assert result == content + + def test_mixed_ssh_and_http(self, mock_config): + content = ( + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + "[submodule \"b\"]\n" + "\tpath = b\n" + "\turl = https://cph1-eud-rep001/scm/sys_yahsat_ngsp/repo_b.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result + assert "cph1-eud-rep001" not in result + + def test_mixed_hostnames_all_resolved(self, mock_config): + """Both short and FQDN hostname variants should be resolved together.""" + content = ( + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + "[submodule \"b\"]\n" + "\tpath = b\n" + "\turl = ssh://git@cph1-eud-rep001.satcom.global:7999/sys_com/repo_b.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_b.git" in result + assert "cph1-eud-rep001" not in result + + def test_uses_correct_org_per_project(self, mock_config): + def resolve(proj, slug): + orgs = {"SYS_YAHSAT_NGSP": "networks-ngsp", "DCKR": "networks-docker"} + return orgs.get(proj, "default-org"), slug + + mock_config.resolve_target = MagicMock(side_effect=resolve) + + content = ( + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + "[submodule \"b\"]\n" + "\tpath = b\n" + "\turl = ssh://git@cph1-eud-rep001:7999/dckr/repo_b.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result + assert "ssh://gatehousesatcom@gatehousesatcom.ghe.com/networks-docker/repo_b.git" in result + + def test_falls_back_to_https_when_no_ssh(self, mock_config): + mock_config.gh_ssh_host = "" + mock_config.gh_ssh_url = "" + content = ( + "[submodule \"a\"]\n" + "\tpath = a\n" + "\turl = ssh://git@cph1-eud-rep001:7999/sys_yahsat_ngsp/repo_a.git\n" + ) + result = remap_submodule_urls(content, mock_config) + + assert "https://gatehousesatcom.ghe.com/networks-ngsp/repo_a.git" in result diff --git a/tests/test_syncer.py b/tests/test_syncer.py new file mode 100644 index 0000000..db455aa --- /dev/null +++ b/tests/test_syncer.py @@ -0,0 +1,156 @@ +"""Tests for the continuous syncer module.""" + +import os +from unittest.mock import MagicMock, patch, call + +import pytest + +from bb2gh.config import Config +from bb2gh.syncer import Syncer, _clean_hidden_refs + + +@pytest.fixture +def mock_config(tmp_path): + config = MagicMock(spec=Config) + config.work_dir = str(tmp_path) + config.sync_interval = 1 + config.lfs_enabled = False + config.lfs_threshold = "100mb" + config.migrate_delay = 0 + config.sync_exclude_projects = set() + config.get_trim_since = MagicMock(return_value=None) + config.push_by_branch = set() + config.sync_protected_branches = [] + config.bb_base_url = "https://bitbucket.example.com" + config.bb_token = "fake" + config.bb_verify_ssl = True + return config + + +class TestSyncer: + @patch("bb2gh.syncer.BitbucketClient") + @patch("bb2gh.syncer.State") + @patch("bb2gh.syncer._run_git") + def test_sync_repo_with_changes(self, mock_git, MockState, MockBB, mock_config, tmp_path): + """Test syncing a repo when changes are detected (no prior snapshot).""" + bare_path = tmp_path / "PROJ__my-repo.git" + bare_path.mkdir() + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + + def side_effect(args, cwd=None, quiet=False): + if args == ["show-ref", "--heads", "--tags"]: + return "abc123 refs/heads/master" + return "" + + mock_git.side_effect = side_effect + + # No prior snapshot file → should detect changes + syncer = Syncer(mock_config) + syncer._sync_all() + + mock_git.assert_any_call(["fetch", "origin", "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*"], cwd=str(bare_path)) + mock_git.assert_any_call(["push", "github", "--mirror"], cwd=str(bare_path)) + state_instance.update_sync_time.assert_called_once_with("PROJ", "my-repo") + + @patch("bb2gh.syncer.BitbucketClient") + @patch("bb2gh.syncer.State") + @patch("bb2gh.syncer._run_git") + def test_sync_skips_when_no_changes(self, mock_git, MockState, MockBB, mock_config, tmp_path): + """Test that sync skips push when BB refs match stored snapshot.""" + bare_path = tmp_path / "PROJ__my-repo.git" + bare_path.mkdir() + + # Write a prior snapshot matching what show-ref will return + refs_file = bare_path / "bb2gh_last_sync_refs" + refs_file.write_text("abc123 refs/heads/master") + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("PROJ", "my-repo")] + state_instance.get_github_target.return_value = ("my-org", "my-repo") + + def side_effect(args, cwd=None, quiet=False): + if args == ["show-ref", "--heads", "--tags"]: + return "abc123 refs/heads/master" + return "" + + mock_git.side_effect = side_effect + + syncer = Syncer(mock_config) + syncer._sync_all() + + # Push should NOT be called + for c in mock_git.call_args_list: + assert c[0][0] != ["push", "github", "--mirror"] + state_instance.update_sync_time.assert_not_called() + + @patch("bb2gh.syncer.BitbucketClient") + @patch("bb2gh.syncer.State") + def test_no_migrated_repos(self, MockState, MockBB, mock_config): + """Test sync when no repos are migrated yet.""" + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [] + + syncer = Syncer(mock_config) + syncer._sync_all() # Should not raise + + @patch("bb2gh.syncer.BitbucketClient") + @patch("bb2gh.syncer.State") + @patch("bb2gh.syncer._run_git") + def test_sync_handles_failure(self, mock_git, MockState, MockBB, mock_config, tmp_path): + """Test that sync continues if one repo fails.""" + bare1 = tmp_path / "PROJ__repo1.git" + bare1.mkdir() + bare2 = tmp_path / "PROJ__repo2.git" + bare2.mkdir() + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [ + ("PROJ", "repo1"), + ("PROJ", "repo2"), + ] + state_instance.get_github_target.return_value = ("my-org", "repo1") + + def side_effect(args, cwd=None, quiet=False): + if "repo1" in str(cwd) and args[0] == "fetch": + raise Exception("Network error") + if args == ["show-ref", "--heads", "--tags"]: + return "abc123 refs/heads/master" + return "" + + mock_git.side_effect = side_effect + + syncer = Syncer(mock_config) + syncer._sync_all() + + # repo1 failed on fetch, repo2 skipped (no changes) + # Neither should have update_sync_time called + + @patch("bb2gh.syncer.BitbucketClient") + @patch("bb2gh.syncer.State") + @patch("bb2gh.syncer._run_git") + def test_sync_logs_github_target(self, mock_git, MockState, MockBB, mock_config, tmp_path): + """Test that sync uses the stored GitHub target for logging.""" + bare_path = tmp_path / "INFRA__my-service.git" + bare_path.mkdir() + # No prior snapshot → will detect changes + + state_instance = MockState.return_value + state_instance.get_migrated_repos.return_value = [("INFRA", "my-service")] + state_instance.get_github_target.return_value = ("infra-team", "infra-my-service") + + def side_effect(args, cwd=None, quiet=False): + if args == ["show-ref", "--heads", "--tags"]: + return "aaa refs/heads/main" + return "" + + mock_git.side_effect = side_effect + + syncer = Syncer(mock_config) + syncer._sync_all() + + state_instance.get_github_target.assert_called_once_with("INFRA", "my-service")