fix(ci): reject a PR whose merge into main would leave two Alembic heads

.github/workflows/migration-order.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
     types: [opened, synchronize, reopened]
   merge_group:
+  push:
+    branches: [main]
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -15,12 +18,40 @@ concurrency:
 jobs:
   migration-order:
     name: Migration order
+    if: github.event_name != 'push'
     runs-on: blacksmith-4vcpu-ubuntu-2404
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
 
+      # Asserts the *merge result* resolves to a single head, not just this
+      # branch. A branch cut before another migration landed on main is
+      # perfectly linear on its own and still forks the chain once merged.
       - name: Check Alembic migration order
         run: python scripts/check_migration_order.py origin/main
+
+  # The job above is only ever as fresh as the PR's last push. GitHub does not
+  # re-run a PR's checks when its *base* moves, and this repo does not require
+  # branches to be up to date before merging, so a green check can quietly go
+  # stale and stay mergeable -- which is exactly what shipped the 2026-07-28
+  # two-head outage. Re-check every open PR whenever main moves.
+  recheck-open-prs:
+    name: Re-check open PRs against new main
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    permissions:
+      contents: read
+      pull-requests: read
+      statuses: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - name: Re-check open PRs
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ./scripts/recheck_open_pr_migrations.sh

CLAUDE.md

@@ -50,6 +50,18 @@ async-substrate-interface). Wire shapes are `ditto/api_models` (Pydantic).
 - **Migrations own the schema.** `ditto/db/models.py` describes it in Python but
   Alembic under `alembic/versions/` is the source of truth — keep them in sync and
   add a migration for any schema change.
+- **One Alembic head, always — rebase before you add a migration.** Alembic
+  linears the chain by `down_revision`, *not* by merge date, so two branches
+  that each extend the same parent stay divergent however git merges them, and
+  `alembic upgrade head` then refuses to run at all (`Multiple head revisions
+  are present`) — taking the deploy and every DB test with it. Rebase onto
+  current `origin/main` and point `down_revision` at its head before opening a
+  PR, renumbering the `YYYY_MM_DD_` filename if main has moved past your date.
+  This is enforced, not just advised: `Migration order` resolves the **merge
+  result** rather than your branch alone, and every push to `main` re-checks
+  each open PR — so a PR that was green when you pushed it goes red the moment
+  someone else's migration lands, instead of staying mergeable. Reproduce
+  either locally with `python scripts/check_migration_order.py origin/main`.
 - **Adding a column to a hot table? Use `safe_add_column`.** `op.add_column`
   holds an `AccessExclusiveLock` until the migration commits, so a plain
   add-then-backfill stalls every writer for the length of the backfill — that is
@@ -113,6 +125,15 @@ enforces all four; Python checks run on 3.11 and 3.12.
 - Put unit tests next to the package they cover under `ditto/tests/<package>`.
 - `make test-db-reset` forces a template rebuild; `make test-db-clean` reaps
   every harness-owned database.
+- **Hundreds of DB failures after touching migrations? Reset the template
+  before believing them.** The harness migrates `ditto_test_template` once and
+  then clones it, so a template built while the chain was broken keeps failing
+  every DB test long after the chain is fixed — and it fails in a way that
+  reads as *your* change's fault. During the 2026-07-28 two-head fix the first
+  run on a clean worktree came back with 513 failures from exactly this; the
+  same worktree was green after `make test-db-reset`. The tell is breadth: a
+  real regression fails tests near what you touched, a stale template fails
+  everything that opens a database.
 
 ## Gotchas
 

ditto/tests/scripts/test_check_migration_order.py

@@ -9,7 +9,9 @@
 from scripts.check_migration_order import (
     Migration,
     MigrationError,
+    merge_result,
     parse_migration,
+    require_single_merged_head,
     resolve_working_tree_head,
     validate_linear_history,
 )
@@ -132,12 +134,12 @@ def test_head_mode_prints_the_single_head(tmp_path: Path) -> None:
 def test_head_mode_names_every_head_and_the_merge_that_fixes_it(
     tmp_path: Path,
 ) -> None:
-    """The 2026-07-25 shape: two PRs extended the same parent and both merged.
+    """Two migrations extended the same parent and both landed.
 
-    Each passed its own migration-order check against `main` at the time, and
-    the divergence only existed once both had landed -- so the deploy is the
-    first place it can be caught. The message has to carry the revisions and
-    the remedy, because alembic's own error carries neither.
+    By the time the deploy sees this the damage is done -- the pre-merge
+    guard below is what is supposed to stop it -- but `update.sh` still has
+    to explain itself. The message has to carry the revisions, the files and
+    the remedy, because alembic's own error carries none of them.
     """
     _write_history(tmp_path, diverged=True)
 
@@ -147,7 +149,247 @@ def test_head_mode_names_every_head_and_the_merge_that_fixes_it(
     assert "2 head revisions are present" in result.stderr
     assert "e5b8c31d47af" in result.stderr
     assert "e7b4c02a5d18" in result.stderr
+    assert "2026_07_02_first.py" in result.stderr
+    assert "2026_07_02_second.py" in result.stderr
     assert (
         'uv run alembic merge -m "merge heads" e5b8c31d47af e7b4c02a5d18'
         in result.stderr
     )
+
+
+# --- the merge result -------------------------------------------------------
+#
+# The 2026-07-28 outage in one sentence: two PRs that were each individually
+# linear against the `main` they were cut from produced a second head once
+# both had merged, because Alembic linears by down_revision and not by merge
+# date. Everything below is about catching that *before* the merge.
+
+
+def test_merge_result_is_the_union_of_both_file_sets() -> None:
+    base = [migration("root", None), migration("landed", "root")]
+    head = [migration("root", None), migration("mine", "root")]
+
+    merged = merge_result(base, head)
+
+    assert [m.revision for m in merged] == ["landed", "mine", "root"]
+
+
+def test_merge_result_keeps_the_base_copy_of_a_shared_path() -> None:
+    """Migrations are immutable, so the base's copy is the one that survives."""
+    base = [Migration("shared.py", "base-revision", None)]
+    head = [Migration("shared.py", "head-revision", None)]
+
+    assert merge_result(base, head) == [Migration("shared.py", "base-revision", None)]
+
+
+def test_single_merged_head_returns_the_head_when_the_branch_extends_base() -> None:
+    base = [migration("root", None)]
+    head = [migration("root", None), migration("mine", "root")]
+
+    merged = merge_result(base, head)
+
+    assert require_single_merged_head(merged, "origin/main", {"root"}) == "mine"
+
+
+def test_merged_heads_name_both_revisions_their_files_and_both_remedies() -> None:
+    """The exact 2026-07-28 fork, with the revisions and files it really had."""
+    base = [
+        Migration(
+            "alembic/versions/2026_07_27_record_evicted.py", "b2e9d4a17c60", None
+        ),
+        Migration(
+            "alembic/versions/2026_07_27_add_ticket_failure_detail.py",
+            "a7c14f8bd260",
+            "b2e9d4a17c60",
+        ),
+        Migration(
+            "alembic/versions/2026_07_27_reinstate_an_evicted_submission.py",
+            "c7a4f1e2b903",
+            "a7c14f8bd260",
+        ),
+    ]
+    head = [
+        base[0],
+        Migration(
+            "alembic/versions/2026_07_27_add_never_disclose_release_policy.py",
+            "f4b7d2c91ae5",
+            "b2e9d4a17c60",
+        ),
+    ]
+
+    with pytest.raises(MigrationError) as excinfo:
+        require_single_merged_head(
+            merge_result(base, head), "origin/main", {"c7a4f1e2b903"}
+        )
+
+    message = str(excinfo.value)
+    assert "would leave 2 Alembic heads" in message
+    # Both offending revisions, each named with the file it lives in.
+    assert "c7a4f1e2b903  alembic/versions/2026_07_27_reinstate_an_evicted" in message
+    assert "f4b7d2c91ae5  alembic/versions/2026_07_27_add_never_disclose" in message
+    # And which side each came from, so the author knows what to rebase onto.
+    assert "already a head on origin/main" in message
+    assert "added by this branch" in message
+    # Both remedies.
+    assert "rebase onto current origin/main" in message
+    assert 'uv run alembic merge -m "merge heads" c7a4f1e2b903 f4b7d2c91ae5' in message
+
+
+def _git(repo: Path, *args: str) -> str:
+    return subprocess.run(
+        ["git", *args],
+        cwd=repo,
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+
+
+def _commit(repo: Path, name: str, revision: str, down: str | None) -> None:
+    path = repo / "alembic" / "versions" / name
+    path.write_text(MIGRATION.format(revision=revision, down=down))
+    _git(repo, "add", str(path))
+    _git(repo, "commit", "-m", f"add {revision}")
+
+
+def _run_check(
+    repo: Path, base_ref: str, head_ref: str
+) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        [
+            sys.executable,
+            str(ROOT / "scripts" / "check_migration_order.py"),
+            base_ref,
+            head_ref,
+        ],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+@pytest.fixture
+def stale_branch_repo(tmp_path: Path) -> Path:
+    """The 2026-07-28 fork, rebuilt as two real branches.
+
+    `main` is at b2e9d4a17c60 when `never-disclose` is cut from it. Two more
+    migrations then land on `main` (#553, then #524). Neither branch is ever
+    rebased, and both are individually linear.
+    """
+    repo = tmp_path / "repo"
+    (repo / "alembic" / "versions").mkdir(parents=True)
+    _git(repo, "init", "-q", "-b", "main")
+    _git(repo, "config", "user.email", "test@example.com")
+    _git(repo, "config", "user.name", "test")
+
+    _commit(repo, "2026_07_26_add_artifact_fetch_audit.py", "e8b3c05d7a41", None)
+    _commit(repo, "2026_07_27_record_evicted_leases.py", "b2e9d4a17c60", "e8b3c05d7a41")
+
+    # PR #505 is cut here, while b2e9d4a17c60 is still the head of main.
+    _git(repo, "switch", "-q", "-c", "never-disclose")
+    _commit(
+        repo,
+        "2026_07_27_add_never_disclose_release_policy.py",
+        "f4b7d2c91ae5",
+        "b2e9d4a17c60",
+    )
+
+    # Meanwhile main moves on, twice, and nobody rebases #505 onto it.
+    _git(repo, "switch", "-q", "main")
+    _commit(
+        repo, "2026_07_27_add_ticket_failure_detail.py", "a7c14f8bd260", "b2e9d4a17c60"
+    )
+    _commit(
+        repo,
+        "2026_07_27_reinstate_an_evicted_submission.py",
+        "c7a4f1e2b903",
+        "a7c14f8bd260",
+    )
+    return repo
+
+
+def test_the_stale_branch_was_genuinely_green_against_the_base_it_was_cut_from(
+    stale_branch_repo: Path,
+) -> None:
+    """Why the old check passed: at the time it ran, nothing was wrong.
+
+    This is the run that happened on #505 at 2026-07-27T18:59. It is correct,
+    and it stayed on the PR as a green check for the ~21 hours until #505
+    merged -- long after it had stopped being true.
+    """
+    # main as it stood when the branch was cut: b2e9d4a17c60 was the head.
+    base = _git(stale_branch_repo, "rev-parse", "main~2").strip()
+
+    result = _run_check(stale_branch_repo, base, "never-disclose")
+
+    assert result.returncode == 0, result.stderr
+    assert "merged head f4b7d2c91ae5" in result.stdout
+
+
+def test_merging_the_stale_branch_into_current_main_is_rejected(
+    stale_branch_repo: Path,
+) -> None:
+    """The guard. Same branch, same content, re-checked against main as it is now.
+
+    Nothing about the branch changed -- only what it would be merging into.
+    """
+    result = _run_check(stale_branch_repo, "main", "never-disclose")
+
+    assert result.returncode == 1
+    assert "would leave 2 Alembic heads" in result.stderr
+    assert "c7a4f1e2b903" in result.stderr
+    assert "f4b7d2c91ae5" in result.stderr
+    assert "2026_07_27_reinstate_an_evicted_submission.py" in result.stderr
+    assert "2026_07_27_add_never_disclose_release_policy.py" in result.stderr
+    assert "rebase onto current main" in result.stderr
+    assert 'alembic merge -m "merge heads" c7a4f1e2b903 f4b7d2c91ae5' in result.stderr
+
+
+def test_a_migration_missing_only_because_main_moved_is_not_a_deletion(
+    stale_branch_repo: Path,
+) -> None:
+    """Removal is judged against the merge base, not against the base tip.
+
+    The stale branch does not contain #524's migration. It did not delete it
+    -- it was cut before it existed -- and reporting that as a deletion would
+    bury the real finding.
+    """
+    result = _run_check(stale_branch_repo, "main", "never-disclose")
+
+    assert "existing migrations were removed" not in result.stderr
+
+
+def test_rebasing_the_branch_onto_current_main_clears_the_failure(
+    stale_branch_repo: Path,
+) -> None:
+    """The remedy the message recommends actually resolves it."""
+    _git(stale_branch_repo, "switch", "-q", "never-disclose")
+    _git(stale_branch_repo, "rebase", "-q", "main")
+    path = (
+        stale_branch_repo
+        / "alembic"
+        / "versions"
+        / "2026_07_27_add_never_disclose_release_policy.py"
+    )
+    path.write_text(MIGRATION.format(revision="f4b7d2c91ae5", down="c7a4f1e2b903"))
+    _git(stale_branch_repo, "commit", "-qam", "repoint onto current main head")
+
+    result = _run_check(stale_branch_repo, "main", "never-disclose")
+
+    assert result.returncode == 0, result.stderr
+    assert "merged head f4b7d2c91ae5" in result.stdout
+
+
+def test_a_branch_that_inherits_a_divergent_base_is_not_blamed_for_it() -> None:
+    """`main` broken under a PR is `main`'s fault, and the message says so."""
+    base = [
+        migration("root", None),
+        migration("one", "root"),
+        migration("two", "root"),
+    ]
+
+    with pytest.raises(MigrationError, match="did not cause the divergence"):
+        require_single_merged_head(
+            merge_result(base, base), "origin/main", {"one", "two"}
+        )