diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 1a30e5f72..6c2d61ade 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -169,6 +169,10 @@ jobs: runs-on: [self-hosted] + # Below GitHub's 360-minute default so an overrun fails as a timeout with + # the log intact, rather than being reaped at the platform cap. + timeout-minutes: 350 + env: CARGO_TERM_COLOR: always RUSTUP_TOOLCHAIN: stable @@ -240,7 +244,34 @@ jobs: LIBRA_STORAGE_BUCKET: ${{ secrets.LIBRA_STORAGE_BUCKET }} LIBRA_STORAGE_ACCESS_KEY: ${{ secrets.LIBRA_STORAGE_ACCESS_KEY }} LIBRA_STORAGE_SECRET_KEY: ${{ secrets.LIBRA_STORAGE_SECRET_KEY }} - run: cargo test --all + run: | + set -euo pipefail + # `cargo test --all` minus the `command_test` binary, which + # `compat-offline-command` shards instead. That one target compiles + # 150 modules (~2950 tests) into a SINGLE process, and roughly a + # third of them serialize on the process-global cwd lock that every + # `ChangeDirGuard` takes — so it runs at about one core regardless of + # the machine. It went past the 360-minute cap the first time the lib + # suite stopped failing early and cargo actually reached it. + # + # Coverage is unchanged: lib, bins and doctests run here, so does + # every other integration target, and `command_test` runs there. + mapfile -t TARGETS < <( + cargo metadata --no-deps --format-version 1 \ + | jq -r '.packages[].targets[] | select(.kind[] == "test") | .name' \ + | grep -vx command_test | sort -u + ) + if [ "${#TARGETS[@]}" -lt 100 ]; then + echo "::error::enumerated only ${#TARGETS[@]} integration targets; refusing to run a truncated suite" + exit 1 + fi + echo "running lib + bins + doctests + ${#TARGETS[@]} integration targets" + ARGS=() + for target in "${TARGETS[@]}"; do + ARGS+=(--test "$target") + done + cargo test --lib --bins "${ARGS[@]}" + cargo test --doc # Phase 6 — Local TUI Automation Control scenario suite (docs/improvement/agent.md Part C). # Without `--features test-provider` + `LIBRA_ENABLE_TEST_PROVIDER=1`, the scenarios @@ -303,6 +334,85 @@ jobs: if-no-files-found: ignore retention-days: 7 + # The `command_test` half of what `cargo test --all` used to do in one job. + # See the note on compat-offline-core's test step for why it is split out: + # the binary is lock-bound rather than CPU-bound, so the only thing that + # shortens it is running it in more than one PROCESS. Shards are separate + # jobs, so each gets its own cwd lock and they scale with the runner pool. + command-tests: + name: compat-offline-command + + runs-on: [self-hosted] + + timeout-minutes: 350 + + strategy: + # One shard failing must not cancel the others: the point of the split is + # to see the whole binary's result in one run. + fail-fast: false + matrix: + shard: [0, 1] + + env: + CARGO_TERM_COLOR: always + RUSTUP_TOOLCHAIN: stable + LIBRA_SKIP_WEB_BUILD: "1" + CARGO_PROFILE_TEST_DEBUG: "0" + CARGO_BUILD_JOBS: "1" + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + submodules: recursive + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: "22" + + - name: Enable pnpm + run: | + corepack enable + corepack prepare pnpm@11.10.0 --activate + + - name: Run command_test shard ${{ matrix.shard }} + env: + LIBRA_TEST_GITHUB_TOKEN: ${{ secrets.LIBRA_TEST_GITHUB_TOKEN }} + LIBRA_TEST_GITHUB_NAMESPACE: ${{ secrets.LIBRA_TEST_GITHUB_NAMESPACE }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + LIBRA_D1_ACCOUNT_ID: ${{ secrets.LIBRA_D1_ACCOUNT_ID }} + LIBRA_D1_API_TOKEN: ${{ secrets.LIBRA_D1_API_TOKEN }} + LIBRA_D1_DATABASE_ID: ${{ secrets.LIBRA_D1_DATABASE_ID }} + LIBRA_STORAGE_ENDPOINT: ${{ secrets.LIBRA_STORAGE_ENDPOINT }} + LIBRA_STORAGE_BUCKET: ${{ secrets.LIBRA_STORAGE_BUCKET }} + LIBRA_STORAGE_ACCESS_KEY: ${{ secrets.LIBRA_STORAGE_ACCESS_KEY }} + LIBRA_STORAGE_SECRET_KEY: ${{ secrets.LIBRA_STORAGE_SECRET_KEY }} + run: | + set -euo pipefail + # Partition by ENUMERATED TEST NAME, not by module prefix. The modulo + # split is exhaustive and disjoint by construction, so a renamed or + # newly added test cannot silently fall out of every shard the way a + # hand-maintained filter list would eventually let one do. + cargo test --test command_test --no-run + mapfile -t ALL < <( + cargo test --test command_test -- --list --format terse \ + | sed -n 's/: test$//p' | sort + ) + total=${#ALL[@]} + if [ "$total" -lt 2000 ]; then + echo "::error::enumerated only $total command tests; refusing to run a truncated shard" + exit 1 + fi + MINE=() + for index in "${!ALL[@]}"; do + if [ "$(( index % 2 ))" -eq "${{ matrix.shard }}" ]; then + MINE+=("${ALL[$index]}") + fi + done + echo "shard ${{ matrix.shard }} of 2: ${#MINE[@]} of $total tests" + cargo test --test command_test -- --exact "${MINE[@]}" + network-remotes: name: compat-network-remotes runs-on: [self-hosted] diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index dad221604..da1e099e0 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -185,6 +185,7 @@ compatibility guard is pinned by `compat_global_config_schema_future`. | op | intentionally-different | Libra command-level operation history inspection/restore extension, not a Git command. Since W1 (§C.9) sequencer control actions (`cherry-pick`/`revert`/`rebase`/`am`/`bisect` start/continue/skip/abort/quit/mark/reset) appear in `op log` through boundary recording, and `op restore` REFUSES them with `LBR-CONFLICT-002`: the snapshot covers HEAD and refs, while a control action also moved an index, a working tree and sequencer state, so replaying it would move HEAD while leaving an in-progress sequence pointing at a todo that no longer matches. Restorability is a stored property of the operation, not a guess from its command name. `op restore` also refuses an operation that is still `running` — including a claim a crashed command left behind. Undo a control action with that command's own `--abort`. | | reflog | supported | `show`/`delete`/`exists`/`expire` subcommands. `expire` prunes by time + reachability + `--stale-fix` (`--all`/`--expire`/`--expire-unreachable`/`--rewrite`/`--updateref`/`-n`/`-v`), reads `gc.reflogExpire`/`gc.reflogExpireUnreachable` (90/30-day defaults, never written). Intentional differences: no-ref expire is an explicit error (exit 128) vs Git's silent no-op; `--stale-fix` checks only that the new value loads as a commit (no transitive object walk); `--updateref` skips symbolic `HEAD` / remote-tracking refs | | worktree | intentionally-different | `remove` keeps disk dir by default (no implicit data loss); since W3-s1b (§C.7) the keep-dir form DETACHES instead of dropping: the entry moves to `detached_from_registry`, the worktree's scoped DB rows (HEAD/reflog/sequencer/dirty/layer/sparse) are PRESERVED — deleting them would leave a directory that still operates but lost its HEAD — and a gitdir marker fail-closes every command inside the directory (actionable re-add/delete hint) until `worktree add ` re-attaches it (identity-checked against the registry's persisted id, mismatches refused) or `--delete-dir` finishes the removal. `--delete-dir` gives Git-style removal, refuses on a dirty worktree, deletes + fsyncs the parent BEFORE cleaning scoped rows, and on a cleanup failure keeps a `tombstone` entry that `worktree repair` retries. Both remove modes refuse while the worktree has an in-progress rebase/cherry-pick/bisect. `prune` only handles entries whose path stat says NotFound (a permission error never classifies a worktree as missing), skips tombstones (repair's job) and scopes with active sequencer state. add/move/remove/prune record a durable intent journal row before mutating (SQLite cannot join a filesystem rename into one transaction — a crash is rolled forward/back by `worktree repair`, which also retries tombstones and reconciles detached markers; recovery never deletes directories). Every MUTATING repair action (no-arg, ``, non-dry-run `--migrate-layout`) requires `--confirm` — without it the command is refused with `LBR-CONFLICT-002` before any registry lock, database write or filesystem touch — and records exactly one operation-log audit row per executed action (`libra op log`); `--migrate-layout --dry-run` stays read-only and confirmation-free, and `repair --resolve-identity` keeps its dedicated `--yes` The down path of migration 2026072402 refuses while any detached/tombstone/journal state or linked-scope sequencer/rebase/bisect state exists; live workspace leases block via migration 2026072501's own down, which every deeper rollback passes through first. lore.md 2.1: linked worktrees now have ISOLATED HEAD + index + HEAD-reflog — each has a real `.libra/` (a `commondir` pointer to the shared db/objects/hooks + a stable `worktree_id`) and its own `index`; a commit/switch/reset in one worktree never moves another's HEAD or touches its index. The canonical add surface is `worktree add []` / `--detach` / `-b []` (W3-s2 §C.7). WITHOUT a target the new worktree is created DETACHED at the source commit — intentionally different from Git, which creates and checks out a basename-named branch (influenced by `worktree.guessRemote`). A branch target checks the branch out ATTACHED, refused before any side effect when ANY scope (including the invoking worktree) already has it out; a nonexistent branch fails closed — Git's remote-branch DWIM (auto `-b --track` on a unique remote match), `worktree.guessRemote`, and `--track`/`--no-track` are deferred. A commit target (and `--detach`, which also forces branch targets detached) seeds a detached HEAD populated from THAT commit. `-b` creates and checks out a new branch from the start point (default: source HEAD) with FULL rollback on any later failure — no branch-only or orphan-registry residue; `-B`/`--force` (collision bypass / copying uncommitted state), `--lock [--reason]` (use the separate `worktree lock`), `--orphan`, and `--no-checkout` are deferred. Shared: the object store, `refs/heads`/`refs/tags`/`refs/remotes`, stash, config. `worktree list --porcelain` emits each worktree's OWN `HEAD ` plus a `branch `/`detached` line (resolved from that worktree's scoped HEAD row); an entry with no resolvable HEAD (legacy layout / missing scope) omits those lines rather than being mislabeled. Deferred (see the deferred-features list): per-worktree ref namespaces (`refs/bisect`/`refs/worktree`) and pseudo-refs (ORIG_HEAD/MERGE_HEAD/…). The ENTIRE sequencer family — `cherry-pick`, `am`, `revert`, `merge`, `bisect`, and `rebase` (v0.19.42) — IS allowed in a linked worktree: their state is fully worktree-scoped (`cherry-pick`/`am` via the `sequence_state` row keyed by `worktree_id`; `revert` via `revert-state.json`, `merge` via `merge-state.json`/`merge-autostash.json`, all in the local gitdir; `bisect` via the `bisect_state` row keyed by `worktree_id`, with its checkouts/`reset` moving only that worktree's scoped HEAD and files; `rebase` via the `rebase_state` row keyed by `worktree_id` plus the worktree-local `rebase-aux.json` (exec queue / update-refs plan / rewrites / held autostash), with GC tracing every scope's state rows and sidecars as reachability roots; the editor buffers `CHERRY_PICK_MSG`/`REVERT_EDITMSG` are local too), and the start-time sequencer mutex resolves each per-worktree, so two worktrees can run them on their own branches concurrently without interfering. The dirty-set cache is worktree-scoped since W1 (§C.4.1.1, migration 2026072302): `dirty` and `status --scan/--cached/--check-dirty` run in any worktree against that worktree's own rows and freshness meta. The layer registry is worktree-scoped since W1 too (§C.4.1.1, migration 2026072303): every `layer` subcommand runs in any worktree against that worktree's own registrations/ownership (same name and destination may exist independently per worktree; removing a worktree purges its layer rows only when the directory is deleted too — a retained directory keeps its ownership rows so the still-materialized overlay files stay un-stageable). The sparse view is worktree-scoped since W1 as well (§C.4.1.1, migration 2026072304): every `sparse-view` subcommand and the `ls-files`/`diff`/`hydrate` gates act on the current worktree's own patterns and toggle (worktree remove/prune GCs them under the same directory-gone rule as layer rows). The stash stack completes the set since W2 (§C.4.3): the stack itself stays deliberately repository-shared while push/apply/pop act on the acting worktree's own index/workdir under a stack lock with by-id CAS deletion — so `stash` (all subcommands) and `pull --rebase --autostash` run in linked worktrees too, and NO command remains refused in a linked worktree on repository-global-state grounds (`pull`'s fetch phase writes only repository-scoped state; merge/rebase modes run on scoped state as before). `maintenance run` gc/repack run in multi-worktree repositories since W2 too (§C.4.3 typed `GcObjectSource` inventory): reachability roots span every worktree's private index (all stages), every scope's sequencer/rebase/bisect rows, every gitdir's held-autostash + merge/revert/rebase-aux sidecars, the shared refs/reflogs/stash reflog, note blobs, undo view snapshots, agent-run findings manifests, and AI capture checkpoints — with a schema-scan guard test that fails when a new OID-bearing column ships un-inventoried. The inventory is typed: `ReachabilityRoot` keeps objects alive, `AntiRoot` (obliteration tombstones) must never be resurrected, `Boundary` (`.libra/shallow`) stops the traversal without demanding the graft's absent parents, and `IndexOnly` (`object_index` and the ordinal/dirty caches) keeps nothing alive and is invalidated before the deletion it describes. `FETCH_HEAD` is deliberately a NON-root (matching Git): fetch records advertised tips that are already up to date and have no local destination, so rooting it would pin objects nothing references; freshly fetched objects are protected by the one-hour prune grace window instead. `fetch` is allowed in a linked worktree — its `FETCH_HEAD` is now worktree-local, and its other writes (`refs/remotes/*` and the object store) are repository-scoped by design; fetching into a branch checked out in another worktree is still refused. The worktree registry is versioned since W3 (§C.7, migration 2026072401) and is at `schema_version: 3` since W1 (migration 2026073005): v3 adds a durable registration GENERATION (`epoch_counter` plus each entry's `epoch`, reported by `worktree list`) that fences `libra service` dirty-mark requests — instance ids are path-derived, so a worktree removed and re-added in place is otherwise indistinguishable from its predecessor. A v2-era binary would drop the generations on rewrite, so the v3 capability marker refuses it at connect time, and the migration will not roll back while a generation is live. Two entries claiming one identity (possible from an older binary via `add A` → `move A B` → `add A`) refuse every worktree MUTATION; `worktree doctor` names them and `worktree repair --resolve-identity --yes` detaches one to resolve it. `worktrees.json` carries `{schema_version: 3, entries: [...]}` with each linked entry's stable `worktree_id` persisted; a legacy v1 file upgrades in place under the registry lock on the first MUTATING worktree command (ids backfilled from each gitdir; lockless readers never rewrite it), every worktree command applies pending migrations before touching the registry so pre-v2 binaries are refused at connect time by the capability marker, and the renamed top-level key makes a v1 parser fail closed as a second belt (a file mixing both shapes, a malformed v2 document, or a v2 entry violating the identity invariants — main with an id, linked without one — is refused rather than reinterpreted). `repair []` diverges from Git's `worktree repair [...]` by design: Git rebuilds the bidirectional gitdir↔worktree links (and must sometimes run from the moved worktree itself); Libra's no-arg form dedupes the single-file registry and re-ensures the main entry, while `repair ` restores a linked worktree's missing/corrupt `.libra/worktree_id` + `commondir` from the registry's PERSISTED stable id — identity is never guessed, a commondir validly pointing at a different storage is refused without touching either file (never re-homed), unregistered paths, main, and a still-v1 registry (no persisted identities — the no-arg repair upgrades it first) are refused, and it always runs from wherever the shared storage resolves (typically main). BARE repositories (`init --bare` / mirror layouts, detected config-first via the recorded `core.bare` so even a bare directory literally named `.libra` is caught) refuse the ENTIRE worktree family with a stable error (`LBR-REPO-003`) before any registry IO (repository migrations may already apply at the CLI preflight, as for any repository command using the standard schema preflight — not a worktree side effect) — Git allows `git worktree add` from a bare repo, Libra defers bare worktree semantics by design (§C.4.1). Legacy symlink-layout worktrees (W3-s3 §C.6): read-only commands keep working through the shared HEAD/index (no regression), but the worktree-STATE mutation surface (add/commit/switch/checkout/restore/reset/merge/merge-file/rebase/cherry-pick/revert/am/bisect/stash/rm/mv/clean/pull/dirty/hydrate/rerere/read-tree/update-index, the mutating `sparse-view` and `layer` subcommands, the `symbolic-ref` write form, and non-dry-run `op restore`) refuses with `LBR-REPO-003` and a migrate hint — committing there would silently move MAIN's HEAD. `worktree list` (JSON + a new porcelain `layout` line) reports each entry's layout: `main`/`linked-v2`/`legacy-symlink`/`missing`/`corrupt` (plus `task-fuse` for FUSE task worktrees). `worktree doctor` (Libra extension, no Git equivalent) reports per-worktree scope diagnostics — layout, lifecycle state, whether the worktree's own identity is still one the registry knows, and what to do about each finding — and is STRICTLY READ-ONLY: registry, database, lease state and filesystem are byte-identical before and after. Repair actions are separate explicit subcommands, so no hint promises that a bare `doctor` will fix anything. JSON uses the `worktree.doctor` envelope with `schema_version`/`diagnostics[]`/`next_cursor`, an opaque cursor that pages `diagnostics[]` (`workspace_id` ascending, default limit 50, cap 500) per the W4 machine interface. `repair --migrate-layout [--dry-run] []` migrates legacy worktrees from the MAIN worktree under the registry lock via a journaled state machine (migration 2026072403 admits the 'migrate' intent; its down refuses while one is in flight): the target `.libra` must be a symlink resolving exactly to this repository's storage (no-follow verified); a prepared journal-stamped gitdir is installed by atomic renames with the legacy link kept as an identity-named backup until verification passes; the new worktree seeds a DETACHED HEAD at the shared snapshot and rebuilds its private index from that commit — tracked/untracked FILES are never touched (they show as dirty/untracked afterwards) and SHARED staged state is never copied (its worktree ownership is unprovable — commit or stash it in main first); an unmerged shared index, an active main-scope sequencer, or an unreadable HEAD refuses before any rename; crash recovery in `worktree repair` advances or rolls back each journal window by IDENTITY (symlink target, journal-stamped marker), never by bare existence, and keeps materials + the journal on any mismatch. Target-oriented `worktree remove ` (both modes) and `worktree repair ` also refuse a legacy-symlink target with the migrate hint (their writes would route through the shared symlink into MAIN storage), and `worktree move` refuses an entry with a pending migration journal. `worktree doctor [] [--limit N] [--cursor C]` (Libra-only, no Git equivalent) is the READ-ONLY diagnosis of Agent workspace scopes (`workspace_record`, §C.8): no invocation writes a row, registry entry, lease, or file. Without an id it pages `data.diagnostics[]` + an opaque `data.next_cursor` (`workspace_id` ASC, default limit 50, cap 500); with an id it returns the singular `data.diagnostic` and no pagination keys — combining the id with `--limit`/`--cursor` is `LBR-CLI-002`. Each diagnostic carries `workspace_id`/`repo_id`/`lease_state` (`none`|`held`|`expired`) and a `scope_diagnostics[]` of `{code,severity,detail}` findings (foreign repository identity, orphaned workspace, expired lease, missing path, missing/detached/tombstoned registry entry, corrupt or legacy-symlink layout). Records written under a PREVIOUS repository identity are visible here and nowhere else. A cursor this command did not issue fails closed with `LBR-WORKTREE-001` instead of restarting at page one, and a scope that cannot be read — unparseable registry, unreadable record, missing repository identity — fails closed with `LBR-WORKTREE-002` rather than reporting a partial diagnosis. Capture rows written before migration 2026080401 are `legacy_unknown` and excluded from capture/import/export writes until an operator explicitly attributes one through `worktree doctor --adopt-capture-session --confirm`. The command requires a live target workspace fence, converts every legacy capture row for that provider session across session/export/import tables, refuses if any scoped row already exists for the provider session, and appends an immutable audit record; it never changes the frozen read-only `worktree.doctor` JSON schema. | +| scorpiofs-worker | intentionally-different | Hidden Libra-owned ScorpioFS worker, not a Git command. `worktree scorpiofs attach` spawns it from Libra's own executable because a FUSE session must outlive the short-lived CLI that started it; it links the ScorpioFS crate directly (Linux + the `scorpiofs-direct` feature — elsewhere it refuses, and `--endpoint` selects the compatibility HTTP mode instead) and serves only a loopback control endpoint. The embedded service is configured NOT to persist or recover state: durable desired state stays with Libra's `worktree scorpiofs` commands. Never invoked directly by users | | cloud | intentionally-different | Libra cloud backup/restore extension, not a Git command | | publish | intentionally-different | Libra Cloudflare publish extension, not a Git command | | agent | intentionally-different | Libra external-agent capture extension, not a Git command. Historical Claude/Codex/OpenCode sessions can be backfilled through the consented, current-repository-scoped `agent import`; typed redaction, coverage/import fences, local erase tombstones, atomic no-clobber loose-object publication, and doctor-visible crash markers make replays idempotent and fail closed. `agent graph ` is a read-only capture projection (session → turn → revision → subagent), distinct from the orchestrator-thread `libra graph`; its frozen JSON v1 uses a strict metadata whitelist, preserves shared checkpoint evidence, reports legacy captures as `unindexed`, and shows local tombstones as `erased` without resurrection. Claude `/subagents/*.jsonl` content is captured as independent source-revisioned `scope=subagent` checkpoints; hook boundaries remain distinct evidence, and association stays unresolved unless a provider-stable id matches exactly one boundary. The per-source import read cap consumes `agent.max_transcript_read_bytes` but never exceeds the 16 MiB adapter hard cap. `agent list --json` defaults to frozen schema v1; `--schema-version 2` opts into method availability. Deferred parity (non-goals for the current wave, tracked canonically in [`docs/development/tracing/agent.md`](docs/development/tracing/agent.md) 「还未实现的功能」 with each item's handling + restart condition): the unstable `agent add`/`remove` `--local-dev`/`--force` flags stay unpublished (canonical `enable`/`disable` + their `add`/`remove` aliases only); provider-specific transcript compaction/reassemble traits (the writer already stores large transcripts as manifest-relative chunks, but no provider-specific compactor exists yet); the optional capability traits (`ProtectedFilesProvider`/`TranscriptCompactor`/`HookResponseWriter`/`RestoredSessionPathResolver`, …) beyond the landed `DeclaredAgentCaps` matrix; external-RPC method families beyond the v2 `info`/capability gate (undeclared capabilities stay fail-closed); and the non-first-batch roster — `gemini`/`cursor`/`copilot`/`factory-ai` stay `supported=false` (unsupported, not hook-installable, not launchable for review/investigate), vs the first batch `claude-code`/`codex`/`opencode`; `agent workspace list|show` (Part C W4) is the read-only keyset-paginated machine interface over the `workspace_record` registry (states provisioning/active/releasing/released/orphaned, lease fence/expiry, canonical path; schema v1; lease mutation never exposed) | diff --git a/Cargo.lock b/Cargo.lock index 0c2f03b89..ee6ae6b4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -461,6 +461,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -494,6 +505,30 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "asyncfuse" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73944bd789372ebf1f10a25ee59296bbb437b31dbe234dcd959e98a1f3238fac" +dependencies = [ + "aligned_box", + "async-notify", + "async-trait", + "bincode 1.3.3", + "bytes", + "dashmap", + "futures-channel", + "futures-util", + "libc", + "nix 0.29.0", + "serde", + "slab", + "tokio", + "tracing", + "trait-make", + "which", +] + [[package]] name = "atoi" version = "2.0.0" @@ -681,6 +716,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -1510,6 +1565,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -1565,7 +1633,7 @@ dependencies = [ "document-features", "futures-core", "mio", - "parking_lot", + "parking_lot 0.12.5", "rustix 1.1.4", "signal-hook", "signal-hook-mio", @@ -1806,7 +1874,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.12", ] [[package]] @@ -1846,6 +1914,37 @@ dependencies = [ "zeroize", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "deltae" version = "0.3.2" @@ -2229,6 +2328,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "endian-type" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" + [[package]] name = "enum-map" version = "2.7.3" @@ -2249,6 +2354,29 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2509,6 +2637,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2571,7 +2709,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -2626,6 +2764,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "gag" version = "1.0.0" @@ -3423,6 +3570,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -3478,6 +3634,42 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "jni" version = "0.21.1" @@ -3698,7 +3890,7 @@ dependencies = [ "memmap2", "moka", "nix 0.29.0", - "radix_trie", + "radix_trie 0.2.1", "reqwest 0.12.28", "rfuse3", "serde", @@ -3711,6 +3903,37 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "libfuse-fs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf5f1ba3c5498a7b893d4398dfd6de0f21d87d16f1385258950d62d5a7806af8" +dependencies = [ + "async-trait", + "asyncfuse", + "bitflags 2.11.0", + "bytes", + "clap", + "futures", + "futures-util", + "itertools 0.14.0", + "libc", + "lru", + "memmap2", + "moka", + "nix 0.29.0", + "radix_trie 0.2.1", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "libloading" version = "0.8.9" @@ -3753,6 +3976,7 @@ dependencies = [ "dirs", "fastrand", "flate2", + "fs2", "futures", "futures-core", "futures-util", @@ -3767,7 +3991,7 @@ dependencies = [ "keyring", "lazy_static", "libc", - "libfuse-fs", + "libfuse-fs 0.1.13", "libvault", "lru-mem", "mime_guess", @@ -3793,6 +4017,7 @@ dependencies = [ "rpassword", "rust-embed", "scopeguard", + "scorpiofs", "sea-orm", "seccompiler", "serde", @@ -3811,7 +4036,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite 0.29.0", "tokio-util", - "toml", + "toml 0.8.23", "tower", "tower-http", "tracing", @@ -3893,7 +4118,7 @@ dependencies = [ "pem", "pgp", "priority-queue", - "radix_trie", + "radix_trie 0.2.1", "rand 0.9.2", "rand_chacha 0.3.1", "regex", @@ -3915,7 +4140,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", - "toml", + "toml 0.8.23", "tonic", "tracing", "ureq 2.12.1", @@ -4136,7 +4361,7 @@ dependencies = [ "equivalent", "event-listener", "futures-util", - "parking_lot", + "parking_lot 0.12.5", "portable-atomic", "smallvec", "tagptr", @@ -4421,7 +4646,7 @@ dependencies = [ "hyper", "itertools 0.14.0", "md-5 0.10.6", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", "quick-xml", "rand 0.10.2", @@ -4705,6 +4930,17 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -4712,7 +4948,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -5486,7 +5736,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" dependencies = [ - "endian-type", + "endian-type 0.1.2", + "nibble_vec", +] + +[[package]] +name = "radix_trie" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" +dependencies = [ + "endian-type 0.2.0", "nibble_vec", ] @@ -5680,6 +5940,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -5896,7 +6165,7 @@ dependencies = [ "aligned_box", "async-notify", "async-trait", - "bincode", + "bincode 1.3.3", "bytes", "dashmap", "futures-channel", @@ -6384,6 +6653,46 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scorpiofs" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5805127aaaff2b35f99b008a5b9570257ea69ee2f181c2154ea9b30d73e2d416" +dependencies = [ + "async-recursion", + "async-trait", + "asyncfuse", + "axum", + "bincode 2.0.1", + "bytes", + "clap", + "clap_complete", + "crossbeam", + "dashmap", + "env_logger", + "futures", + "git-internal", + "hex", + "libc", + "libfuse-fs 0.2.0", + "log", + "once_cell", + "radix_trie 0.3.0", + "reqwest 0.13.2", + "ring", + "serde", + "serde_json", + "sled", + "thiserror 2.0.18", + "tokio", + "toml 0.9.12+spec-1.1.0", + "tracing", + "tracing-subscriber", + "url", + "uuid", + "whoami 1.6.1", +] + [[package]] name = "sdd" version = "3.0.10" @@ -6701,6 +7010,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -6797,7 +7115,7 @@ dependencies = [ "futures-executor", "futures-util", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "scc", "serial_test_derive", ] @@ -6989,6 +7307,22 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -7516,7 +7850,7 @@ dependencies = [ "atomic", "crossbeam-channel", "getrandom 0.2.17", - "parking_lot", + "parking_lot 0.12.5", "rand 0.8.5", "seahash", "thiserror 1.0.69", @@ -7960,7 +8294,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", "socket2", @@ -8061,11 +8395,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", + "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_edit 0.22.27", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap 2.13.0", + "serde_core", + "serde_spanned 1.1.1", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -8075,6 +8424,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_datetime" version = "1.0.0+spec-1.1.0" @@ -8092,7 +8450,7 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.13.0", "serde", - "serde_spanned", + "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_write", "winnow", @@ -8125,6 +8483,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tonic" version = "0.14.5" @@ -8555,6 +8919,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "2.12.1" @@ -8679,6 +9049,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "vm-memory" version = "0.16.2" @@ -9028,7 +9404,7 @@ checksum = "7aafc5e81e847f05d6770e074faf7b1cd4a5dec9a0e88eac5d55e20fdfebee9a" dependencies = [ "event-listener", "futures-core", - "parking_lot", + "parking_lot 0.12.5", "triomphe", ] @@ -9052,6 +9428,7 @@ checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", "wasite", + "web-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b2fc629c1..eb5e22973 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,8 @@ categories = ["command-line-utilities", "development-tools"] readme = "README.md" [features] -default = [] +default = ["scorpiofs-direct"] +scorpiofs-direct = ["dep:scorpiofs"] worktree-fuse = [] # Unix FUSE-backed worktree commands (optional) test-network = [] # L2: tests requiring outbound network but no secrets test-live-ai = [] # L3: tests calling real LLM APIs @@ -70,6 +71,7 @@ sea-orm = { version = "2.0.0", features = [ ]} serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" +fs2 = "0.4.3" sha1 = "0.11.0" sha2 = "0.10" thiserror = "2.0.18" @@ -158,6 +160,7 @@ rfuse3 = { version = "0.0.8", features = ["tokio-runtime", "unprivileged"] } [target.'cfg(target_os = "linux")'.dependencies] # Linux-only seccomp BPF compiler seccompiler = { version = "0.5.0", features = ["json"] } +scorpiofs = { version = "=0.4.0", optional = true } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61.2", features = ["Win32_Storage_FileSystem"] } diff --git a/docs/commands/worktree.md b/docs/commands/worktree.md index 96c0c79b9..c8b14f399 100644 --- a/docs/commands/worktree.md +++ b/docs/commands/worktree.md @@ -8,6 +8,8 @@ Manage multiple working trees attached to this repository. ``` libra worktree add +libra worktree scorpiofs attach --remote-path --job-id +libra worktree scorpiofs detach libra worktree list libra worktree doctor libra worktree lock [--reason ] @@ -26,6 +28,12 @@ libra worktree doctor [] [--limit ] [--cursor ] `libra worktree` manages multiple working trees that share a single repository database and object store. This allows you to have several checkouts of the same repository simultaneously, which is useful for working on multiple branches at once, running builds while editing code, or testing changes in isolation. +Mounted worktrees use the backend-neutral architecture documented in +[`worktree-storage-backends.md`](../development/integration/worktree-storage-backends.md). +ScorpioFS is a remote-revision projection backend; BrewFS is modeled as a +persistent distributed-volume backend. Git objects, refs, index state, and +authoritative backend lifecycle state remain owned by Libra. + Each linked worktree is a directory containing its own real `.libra` gitdir — a local directory (not a symlink) that holds the worktree's private `HEAD`, index, and `HEAD` reflog, plus a `commondir` pointer to the shared storage and a stable `worktree_id`. The main worktree is the original repository directory. All worktrees share the same SQLite database, object store, branch/tag/remote refs, and configuration, but each keeps its own checked-out branch and staging state. (A worktree created by an older Libra version may still use the legacy shared-`.libra` symlink layout; `libra worktree doctor` reports it, and `libra worktree repair --migrate-layout --dry-run` previews the migration read-only.) The registry file `worktrees.json` is versioned (`schema_version: 3`; v2 since v0.19.57): each linked entry persists its stable `worktree_id`, a legacy v1 file is upgraded in place by the first mutating worktree command (ids backfilled from each worktree's gitdir; lockless readers like `worktree list` read a v1 file without rewriting it), and older binaries are refused at the database layer before they can misread or rewrite the file. v3 adds a durable registration GENERATION — `epoch_counter` on the registry and `epoch` on each entry, reported by `worktree list`. Instance ids are path-derived, so a worktree removed and re-added in the same place has the same id and path as its predecessor; the generation is what tells the two registrations apart, and the `libra service` dirty-mark endpoint requires it as a fence. A v2-era binary would parse a v3 file and drop the generations on rewrite, so the v3 capability marker refuses those binaries at connect time; for the same reason this migration does not roll back while any generation is live. **Resolving an ambiguous registry.** A registry written by an older binary can end up with two entries claiming one path-derived identity (`add A` → `move A B` → `add A`). Every worktree MUTATION is refused while that holds — including the `remove` that would fix it, because it goes through the same loader. `libra worktree doctor` names the colliding entries, and `libra worktree repair --resolve-identity --yes` is the one action that runs against the ambiguous registry: it DETACHES the named entry (files and scoped state kept, every command inside that directory fails closed) so the remaining claimant owns the identity again. Finish with `worktree remove --delete-dir `, or `worktree add ` to re-attach it. @@ -67,6 +75,36 @@ libra worktree add --detach ../probe v1.2.0 libra worktree add -b topic ../topic main ``` +### Subcommand: `scorpiofs attach` + +Creates or recovers an idempotent Antares mount, waits for readiness, and +attaches persistent Libra linked-worktree metadata to the returned mountpoint. +By default on Linux, Libra starts a resident worker that links the ScorpioFS +crate directly. Libra owns the worker and desired mount state in +`.libra/scorpiofs/state.json`; ScorpioFS owns only live filesystem +materialization. Libra also continues to own the index, objects, commits, +refs, fetch, and push. + +```bash +libra worktree scorpiofs attach \ + --config-path scorpio.toml \ + --remote-path /project/aardvark-dns \ + --job-id dev-aardvark +``` + +Pass `--endpoint http://127.0.0.1:2725/antares` to use an externally managed +ScorpioFS daemon as a compatibility transport instead. + +### Subcommand: `scorpiofs detach` + +Refuses to detach a dirty worktree, removes its persistent linked-worktree +metadata, and asks Antares to delete the mount by job ID. Repeated remote +cleanup is idempotent. + +```bash +libra worktree scorpiofs detach /var/lib/antares/mounts/ +``` + ### Subcommand: `list` List all registered worktrees and their state. `--porcelain` emits a stable, diff --git a/docs/development/commands/README.md b/docs/development/commands/README.md index 039bd86db..0ca850110 100644 --- a/docs/development/commands/README.md +++ b/docs/development/commands/README.md @@ -62,6 +62,7 @@ | [`credential`](credential.md) | `partial` | Vault-backed Git credential helper `fill`/`store`/`erase`; AES-256-GCM-encrypted, keyed by a digest of protocol/host/path; side-channel-free `fill` (hit/miss both exit 0); expiry (default 30d); secrets never logged/echoed. `credential-cache`, multi-username, and the consumer-side helper chain deferred | | [`describe`](describe.md) | `partial` | basic describe, `--tags`, `--always`, `--abbrev`, `--exact-match`, `--long`, `--dirty[=]`, `--first-parent`, `--match`, `--exclude`, `--candidates` (0 ⇒ exact-match), `--all` (any ref, prefixed), and `--contains` (git name-rev: nearest descendant tag, `~^` form) supported | | [`service`](service.md) | `intentionally-different` | 无头本地服务（lore.md 1.11）：notification v1 总线 + 令牌门 dirty 标记摄入；双重环回强制，绝不开对外端口 | +| [`scorpiofs-worker`](scorpiofs-worker.md) | `intentionally-different` | 隐藏的 Libra 自有 ScorpioFS 常驻 worker，非 Git 命令：由 `worktree scorpiofs attach` 从 Libra 自身可执行文件拉起，直连 ScorpioFS crate（Linux + `scorpiofs-direct` feature），只暴露环回控制端点；内嵌服务不持久化、不自恢复状态，权威状态留在 Libra 侧 | | [`auth`](auth.md) | `intentionally-different` | 主机作用域 HTTP 令牌 v1（lore.md 1.6）：加密落盘、过期检测、免解密撤销、降级重定向拒绝 | | [`commit-tree`](commit-tree.md) | `partial` | 低层修订组合（lore.md 1.15）：tree+parents+message→commit 对象，零副作用；配合 --index-file scratch 索引 | | [`revision`](revision.md) | `intentionally-different` | 修订序号索引（lore.md 1.16）：逐 ref first-parent 链 1..N，tip+replace 指纹每读校验，快进追加/重写重建 | diff --git a/docs/development/commands/scorpiofs-worker.md b/docs/development/commands/scorpiofs-worker.md new file mode 100644 index 000000000..58f95a810 --- /dev/null +++ b/docs/development/commands/scorpiofs-worker.md @@ -0,0 +1,44 @@ +# `libra scorpiofs-worker` 开发设计 + +## 命令实现目标 + +`libra scorpiofs-worker` 是隐藏的常驻 worker（`hide = true`），不面向用户直接调用。它存在的唯一理由是生命周期不匹配：Libra 的公开 CLI 是短命进程，而一次 FUSE 挂载必须在 `libra worktree scorpiofs attach` 返回之后继续存活。因此 attach 从 Libra 自己的可执行文件（`std::env::current_exe()`）拉起这个子命令，由它持有 FUSE 会话。 + +worker 直连 ScorpioFS crate，只监听一个环回控制端点（`--bind 127.0.0.1:`，端口由 attach 预留后传入）。内嵌的 ScorpioFS 服务被显式配置为**不持久化、不自恢复**状态：权威的期望状态始终留在 Libra 侧（`.libra/scorpiofs/` 下的 `LibraScorpioFsState`），worker 崩溃后由下一次 attach 重建，而不是由 ScorpioFS 自己恢复。 + +## 对比 Git 与兼容性 + +- 兼容级别：`intentionally-different`。Git 没有对应命令。 +- 平台/特性门：仅在 `target_os = "linux"` 且启用 `scorpiofs-direct` feature 时可用。其他组合下命令本身仍然存在（保持 CLI 表面稳定），但立刻以 `LBR-UNSUPPORTED` 拒绝，并提示改用 `worktree scorpiofs attach --endpoint ` 的兼容 HTTP 模式。 +- 参数全部是必填的内部契约，不做用户级校验糖：`--config-path`、`--bind`、`--upper-root`、`--cl-root`、`--mount-root`、`--runtime-state-file`。 +- 命令作用域（`src/cli.rs::command_scope`）登记为 `Repository`——与 `libra service` 同形态：按写者归类以免作用域被低估；同时它被列入 `command_holds_shared_maintenance_lock` 的长驻豁免名单，否则挂载存活期间会一直持有共享 maintenance 锁，饿死每一个删除阶段。 + +## 设计方案 + +- 入口与分发：`src/cli.rs::Commands::ScorpiofsWorker`（`hide = true`）→ `command::scorpiofs_worker::execute_safe`。 +- 源码分层：`src/command/scorpiofs_worker.rs` 只做参数转译与错误包装；挂载编排、期望状态与回滚都在 `src/internal/scorpiofs_backend.rs`，后者实现 `src/internal/worktree_backend.rs` 定义的后端中立接口。 +- 配置优先级：路径类参数经 `scorpiofs::cli::antares_overrides` 转成 config override map 后交给 `util::config::init_config_with`，以保持 ScorpioFS 文档承诺的 `CLI > env > file > default` 次序，而不是事后改写 `AntaresPaths`。 +- 日志：attach 侧把 worker 的 stdout/stderr 追加重定向到 `.libra/scorpiofs/worker.log`，worker 自身不另开日志文件。 + +```mermaid +flowchart TD + A["worktree scorpiofs attach"] --> B["预留环回端口 + 建运行目录"] + B --> C["spawn current_exe() scorpiofs-worker
stdout/stderr → .libra/scorpiofs/worker.log"] + C --> D["scorpiofs_worker::execute_safe"] + D --> E["cli::antares_overrides → config::init_config_with"] + E --> F["AntaresServiceImpl::new_external_state(None)"] + F --> G["AntaresDaemon::serve(127.0.0.1:port)"] + A --> H["HttpScorpioFsClient 轮询 /health 直到 ready"] + H --> G +``` + +## 当前状态 + +- 已实现：托管 worker 的拉起、健康探测、失活检测（worker 消失时把 `ManagedCrate` 传输的挂载标记为 `RecoverableError` 并要求重新 attach）、detach 时在最后一个挂载消失后停止 worker。 +- 依赖：`scorpiofs = "=0.4.0"`（Linux-only、optional）。 + +## 还未实现的功能 + +- worker 自身没有重启/看门狗：进程消失后由下一次 attach 重建，期间已挂载路径不可用。 +- 没有多仓库共享 worker：运行目录按 storage 路径哈希隔离，每个仓库一个 worker。 +- 非 Linux 平台没有直连模式，只能走 `--endpoint` 兼容 HTTP 模式。 diff --git a/docs/development/integration/scorpiofs-worktree-backend.md b/docs/development/integration/scorpiofs-worktree-backend.md new file mode 100644 index 000000000..07d29d79d --- /dev/null +++ b/docs/development/integration/scorpiofs-worktree-backend.md @@ -0,0 +1,557 @@ +# ScorpioFS remote worktree backend + +Status: MVP implemented and validated against a real Mega-backed FUSE mount + +This document describes the ScorpioFS-specific adapter. The backend-neutral +contracts, BrewFS extension point, storage ownership, and process model are +defined in +[`worktree-storage-backends.md`](worktree-storage-backends.md). + +## Libra-owned state and direct crate execution + +Libra is the authoritative owner of ScorpioFS desired state. It persists the +worker identity and every requested mount under +`.libra/scorpiofs/state.json`, including lifecycle transitions through +`mounting`, `ready`, `unmounting`, and `recoverable_error`. + +On Linux, `worktree scorpiofs attach` starts or reuses a hidden +`libra scorpiofs-worker` process by default. That worker links the `scorpiofs` +crate directly and keeps FUSE sessions alive after the invoking CLI process +exits. ScorpioFS runs with external state ownership, so its in-memory mount +registry is an execution cache only: it must not persist, recover, or decide +the desired mount set. + +The HTTP transport remains available only as an explicit compatibility mode: + +```text +libra worktree scorpiofs attach \ + --endpoint http://127.0.0.1:2725/antares \ + --remote-path /project/aardvark-dns \ + --job-id aardvark-dns +``` + +Without `--endpoint`, Libra starts the crate-backed worker using +`--config-path scorpio.toml`. When the final Libra-owned mount is detached, +Libra asks its worker to shut down gracefully. + +## Summary + +Libra integrates ScorpioFS as a remote-projection worktree backend. Libra +remains the only owner of version-control semantics and desired lifecycle +state. ScorpioFS owns remote file materialization, FUSE execution, changelist +layers, writable upper layers, and live mount sessions. + +The integration must not: + +- reimplement Git object, index, ref, merge, or transport logic in ScorpioFS; +- link the complete Libra application into ScorpioFS; +- mount Libra's FUSE worktree on top of a ScorpioFS FUSE mount; +- store the persistent Libra repository database inside an ephemeral Antares + upper layer; +- expose arbitrary Libra command execution through ScorpioFS's unauthenticated + HTTP API. + +## Ownership + +### Libra owns + +- repository identity; +- common object storage; +- the index; +- HEAD, branches, refs, and reflogs; +- commit and tree construction; +- status, diff, restore, checkout, merge, rebase, and stash semantics; +- remotes, credentials, fetch, pull, and push; +- hooks and signing; +- Mega single-commit push preflight. + +### ScorpioFS owns + +- the Mega-backed read-only base filesystem; +- lazy tree and blob materialization; +- optional changelist layers; +- per-mount writable upper layers; +- FUSE inode and file-handle lifecycle; +- mount creation, readiness, recovery, and deletion; +- efficient reporting of paths changed in the writable view; +- switching a mount to a different immutable base snapshot. + +### The integration layer owns + +- mapping a Libra linked worktree to a ScorpioFS mount; +- the local protocol and capability negotiation; +- lifecycle and operation locks; +- recreating the `.libra` worktree pointer after remount; +- coordinating base-snapshot changes for pull, switch, and reset; +- converting service errors into stable Libra errors. + +## Why the integration belongs in Libra + +Libra already exposes a library entry point and implements the full VCS command +surface. It also has linked-worktree scoping for local HEAD, index, FETCH_HEAD, +sequencer, rebase, and advisory state. + +ScorpioFS already exposes an Antares control plane and an isolated userspace +overlay. Making ScorpioFS a Libra worktree backend therefore adds one adapter +instead of duplicating VCS behavior. + +Libra's optional `worktree-fuse` feature is not used for this backend. That +feature creates a local overlay from a local lower directory. A ScorpioFS +backend is already a mounted remote overlay; nesting the two introduces +duplicate mount ownership, cleanup ambiguity, and unnecessary filesystem +overhead. + +## Storage layout + +Persistent Libra state lives outside the ScorpioFS mount: + +```text +/.libra/ +├── libra.db +├── objects/ +├── refs/ +└── worktrees/ + └── scorpiofs/ + └── / + ├── commondir + ├── worktree_id + ├── index + ├── HEAD + ├── FETCH_HEAD + └── backend.json +``` + +The mounted worktree contains only a reconstructable `.libra` gitdir pointer: + +```text +/.libra +``` + +The pointer resolves to the persistent worktree gitdir. It may be recreated +after every mount without changing repository history or worktree identity. + +`backend.json` contains no credentials: + +```json +{ + "schema_version": 1, + "backend": "scorpiofs", + "endpoint": "unix:///run/scorpiofs/control.sock", + "mount_id": "1a78c97f-68b7-4873-bfe2-2d67f3768b23", + "job_id": "build-123", + "remote_path": "/project/aardvark-dns", + "base_oid": "0123456789abcdef", + "cl": "1XFJ4PGK" +} +``` + +The canonical identity is a stable Libra `worktree_id`, not the transient +ScorpioFS `mount_id`. + +## User-facing command model + +The backend is managed through Libra: + +```text +libra worktree scorpiofs attach \ + --endpoint http://127.0.0.1:2725/antares \ + --remote-path /project/aardvark-dns \ + --job-id dev-aardvark + +libra worktree scorpiofs detach + +libra worktree list +libra worktree repair +libra worktree remove +``` + +After the worktree is ready, ordinary Libra commands run inside it: + +```text +libra status +libra add . +libra commit -m "..." +libra fetch origin +libra push --dry-run origin main:main +``` + +Backend-specific options belong to `worktree scorpiofs attach`; normal VCS +commands must not grow ScorpioFS-specific flags. + +An attached ScorpioFS worktree uses a private detached HEAD. Pushes from it +must therefore name both the remote and an explicit source/destination +refspec, such as `main:main`. A default push that needs Libra to infer the +current branch remains rejected. The `--dry-run` form validates Mega discovery +and the update plan without mutating the remote. + +The existing local-copy and optional local-FUSE worktree backends remain +compatible. A serialized worktree record gains a backward-compatible backend +descriptor whose default is `local`. + +## Control protocol + +### Transport + +Production integration uses a local Unix domain socket. A loopback HTTP endpoint +may be supported for development, but must require an explicit opt-in and must +not accept credentials or arbitrary commands. + +The first implementation may use the existing Antares loopback HTTP API behind +the backend client. The public Rust interface must hide the transport so it can +move to the Unix socket without changing command code. + +### Version negotiation + +Every client begins with: + +```json +{ + "protocol_version": 1, + "client": "libra", + "client_version": "0.19.40" +} +``` + +The service responds with: + +```json +{ + "protocol_version": 1, + "service": "scorpiofs", + "capabilities": [ + "mount.v1", + "ready.v1", + "changes.v1", + "base-snapshot.v1" + ] +} +``` + +Libra must fail closed when a required capability is unavailable. Optional +capabilities may select a documented slower fallback. + +### Mount request + +```json +{ + "job_id": "dev-aardvark", + "path": "/project/aardvark-dns", + "cl": null, + "base_oid": "0123456789abcdef" +} +``` + +Response: + +```json +{ + "mount_id": "1a78c97f-68b7-4873-bfe2-2d67f3768b23", + "mountpoint": "/var/lib/scorpiofs/antares/mnt/1a78c97f", + "base_oid": "0123456789abcdef", + "ready": false +} +``` + +Create-by-`job_id` remains idempotent. + +### Changed-path request + +Libra must not recursively scan the full remote monorepo for `status` or +`add .`. ScorpioFS reports candidate paths from the CL and writable upper +layers: + +```json +{ + "mount_id": "1a78c97f-68b7-4873-bfe2-2d67f3768b23", + "generation": 42, + "changes": [ + { "kind": "modified", "path": "src/lib.rs" }, + { "kind": "added", "path": "notes.txt" }, + { "kind": "deleted", "path": "src/old.rs" }, + { + "kind": "renamed", + "path": "src/new.rs", + "source_path": "src/previous.rs" + } + ] +} +``` + +This is a candidate set, not authoritative Git status. Libra still applies +ignore rules, pathspecs, index comparison, content hashing, rename policy, and +Git-compatible output. + +If `changes.v1` is unavailable, Libra may scan only the physical writable +layer. It must warn before falling back to a full mounted-tree walk. + +## Worktree creation transaction + +`libra worktree scorpiofs attach` performs: + +1. Validate the current Libra repository and requested remote path. +2. Negotiate backend capabilities. +3. Reserve a stable Libra worktree ID. +4. Create the persistent per-worktree gitdir and `backend.json`. +5. Request or recover the idempotent ScorpioFS mount. +6. Wait for mount readiness with a bounded timeout. +7. Attach the worktree gitdir pointer inside the mount. +8. Seed the worktree index from the selected Libra commit without populating + files. +9. Register the worktree in Libra's common worktree state. +10. Mark the backend record ready. + +Failures roll back in reverse order. A mount that cannot be deleted is recorded +as orphaned and reported with a repair command; it must not be silently +forgotten. + +## Status and detach consistency + +ScorpioFS is the authority for the writable-view candidate set. Libra does not +walk the full mounted monorepo during `status`, `add`, or the dirty-worktree +check that precedes `detach`. It asks `changes.v1` for candidate paths and then +applies normal Libra index, ignore, hashing, and rename rules to only those +paths. + +This rule is also required for correct cleanup. A FUSE mount can contain +implementation-local upper-layer artifacts that are not part of the +ScorpioFS-reported writable view. A raw recursive disk scan could therefore +make an already committed worktree impossible to detach. Detach uses the same +candidate-path collection as `status`; it still refuses to detach when the +service reports staged or unstaged Libra-visible changes. The persistent +`.libra` pointer is metadata, not a user file or staged change. + +## Normal command behavior + +### Status + +1. Resolve the current linked-worktree scope. +2. Load and validate `backend.json`. +3. Ask ScorpioFS for changed-path candidates. +4. Let Libra compare HEAD, index, and mounted file content. +5. Enumerate untracked files from the writable layer, not the remote base. + +### Add + +Libra applies pathspec and ignore semantics, reads selected mounted files, +writes blobs, and updates the worktree-local index. Deleted candidates stage as +deletions. ScorpioFS does not create Git objects. + +### Commit + +Libra builds trees and commits from the index, updates refs and reflogs, runs +hooks, and signs when configured. ScorpioFS is not involved. + +### Fetch + +Libra updates objects and refs. Fetch does not change the mounted base or +writable filesystem. + +### Push + +Libra performs transport, authentication, pack construction, and ref updates. +Mega-specific single-commit policy is checked in Libra before transport. +ScorpioFS does not receive credentials or push data. + +The initial validation uses an explicit refspec and `--dry-run`: + +```text +libra push --dry-run origin main:main +``` + +This proves remote discovery, Smart HTTP planning, detached-worktree refspec +handling, and Mega update preparation without changing the remote. A real +push remains an explicit user action and is subject to Libra's Mega +single-commit preflight. + +## Branch and base-snapshot changes + +There are two implementation stages. + +### Stage A: upper-layer delta + +Checkout, switch, restore, and reset write the difference between the immutable +base tree and target tree into the writable layer. Deletions use the overlay's +supported whiteout representation. + +This provides correctness first but may grow the upper layer after repeated +branch switches. + +### Stage B: transactional base switch + +For clean worktrees, Libra requests a new immutable base snapshot: + +1. Acquire the exclusive VCS and mount lifecycle leases. +2. Verify or stash local changes. +3. Prepare a replacement mount for the target commit. +4. Attach the existing persistent Libra worktree gitdir. +5. Verify the new view and index. +6. Atomically publish the replacement mount. +7. Delete the old mount. + +If publication fails, the old mount remains active. If old-mount cleanup fails, +the operation succeeds with an explicit orphan warning and repair record. + +`pull`, branch `switch`, `checkout`, `reset --hard`, and `rebase` must not update +Libra refs while leaving the user on an unrelated old base view. + +## Locks and lifecycle + +Each backend worktree has: + +- a shared read lease for status, diff, log, and read-only inspection; +- an exclusive VCS lease for add, commit, checkout, merge, rebase, and reset; +- an exclusive lifecycle lease for mount, base switch, repair, and unmount. + +Unmount refuses to race with a VCS operation. Shutdown stops admitting new +operations, waits for bounded graceful completion, persists recovery state, and +then unmounts. + +The backend state machine is: + +```text +Detached + -> Mounting + -> Ready + -> SwitchingBase + -> Ready + -> Unmounting + -> Detached + +Any state may enter RecoverableError. +``` + +## Error contract + +Backend failures map to stable Libra error categories: + +- backend unavailable; +- protocol incompatible; +- mount rejected; +- mount readiness timeout; +- stale mount identity; +- changed-path generation lost; +- base snapshot unavailable; +- worktree busy; +- cleanup incomplete; +- backend state corrupt. + +Messages include the operation, worktree path, job ID, and recovery action. They +must not include tokens, credential-bearing URLs, signing material, or file +contents. + +## Security + +- Prefer a Unix socket owned by the current user or service group. +- Validate that every returned mountpoint is inside the configured ScorpioFS + mount root. +- Validate that every worktree gitdir is inside Libra common storage. +- Never pass credentials on a process command line. +- Do not expose a generic "run Libra command" ScorpioFS endpoint. +- Treat remote paths, CL names, mount IDs, and changed paths as untrusted. +- Reject absolute changed paths and paths containing parent traversal. +- Preserve the unauthenticated Antares API warning until a protected transport + is available. + +## Compatibility + +- Existing Libra repositories and local worktrees default to backend `local`. +- Existing serialized worktree records load without migration. +- `worktree-fuse` remains optional and independent. +- Native Git fallback remains available for explicitly unsupported Libra + behavior. +- Hooks remain under Libra metadata; no synthetic `.git/hooks` directory is + created. +- Advanced commands that are not safe in linked worktrees remain guarded until + their state is worktree-scoped. + +## Implementation phases + +### Phase 1: backend substrate + +- Add versioned backend types and persistent backend records. +- Add a transport-independent ScorpioFS client. +- Add backend-aware worktree registration, list, repair, and removal. +- Use the existing Antares HTTP API for mount, readiness, and delete. + +### Phase 2: core VCS workflow + +- Run status, add, commit, fetch, and push in the attached linked worktree. +- Add Mega single-commit push preflight. +- Add lifecycle locking and recovery tests. + +### Phase 3: changed paths + +- Add `changes.v1` to ScorpioFS. +- Consume candidates in Libra status and add. +- Add generation, overflow, rename, deletion, and ignore tests. + +### Phase 4: mutable worktree operations + +- Validate restore, path checkout, switch, reset, merge, and stash on the + writable overlay. +- Add whiteout and metadata-operation coverage. + +### Phase 5: immutable base snapshots + +- Add commit-addressed bases and transactional base switching to ScorpioFS. +- Integrate pull, branch switching, reset, and rebase. +- Add crash recovery and orphan cleanup. + +### Phase 6: production validation + +- **Validated** mount/open, status, add, commit, fetch, explicit push planning, + and detach against `project/aardvark-dns` on Mega. The test ran in an + isolated Linux user and mount namespace and verified that + `.libra/scorpiofs/state.json` has an empty `mounts` map after detach. +- **Implemented test coverage** includes endpoint validation, state locking, + lifecycle transitions, changed-path validation, attach idempotency, and the + detach regression where an unreported local artifact must not block a clean + ScorpioFS worktree. +- Validate Buck2 builds on the same mount. +- Test restart recovery, concurrent worktrees, cancellation, and cleanup. +- Document native Git fallback and operational diagnostics. + +## Verified command trace + +The end-to-end test executes this sequence against the deployed Mega +subrepository: + +```text +libra clone https://git.rk8s.xuanwu.openatom.cn/project/aardvark-dns control +libra worktree scorpiofs attach --config-path scorpio.toml \ + --remote-path /project/aardvark-dns --job-id +cd +libra status --porcelain +libra fetch origin +libra add libra-scorpiofs-e2e.txt +libra commit -m "test: validate Libra ScorpioFS backend" +libra push --dry-run origin main:main +libra worktree scorpiofs detach +``` + +The test creates only a temporary local commit in the isolated worktree. It +does not publish a remote commit or alter the deployed Mega branch. + +## Deliberate current limits + +- The compatibility HTTP control transport is still supported; a protected + Unix-socket transport is the production target. +- Libra uses ScorpioFS as a POSIX data plane and does not duplicate Git logic + in the filesystem service. +- Automatic transactional base switching, restart recovery, and concurrent + mount stress are not yet validated end-to-end. +- Buck2-on-ScorpioFS validation remains separate work; passing VCS lifecycle + tests is not a Buck2 compatibility guarantee. + +## Acceptance criteria + +The first production-capable milestone is complete when: + +- a ScorpioFS mount attaches as a persistent Libra linked worktree; +- normal Libra `status`, `add`, `commit`, `fetch`, and `push` work inside it; +- unmount/remount preserves Libra metadata and worktree identity; +- status and `add .` do not walk the full remote monorepo; +- no credentials pass through ScorpioFS; +- mount and VCS operations cannot race destructively; +- failures leave a diagnosable and repairable state; +- focused unit tests and a real Mega/FUSE end-to-end test pass. diff --git a/docs/development/integration/worktree-storage-backends.md b/docs/development/integration/worktree-storage-backends.md new file mode 100644 index 000000000..7221ce2e8 --- /dev/null +++ b/docs/development/integration/worktree-storage-backends.md @@ -0,0 +1,244 @@ +# Worktree storage backend architecture + +Status: backend-neutral substrate implemented; ScorpioFS adapter implemented; +BrewFS SDK runtime boundary implemented + +Implementation labels in this document: + +- **Implemented**: present in this Libra branch and covered by automated tests. +- **Validated**: exercised against a deployed ScorpioFS and Mega service. +- **Planned**: an architectural direction, not a currently available command. + +## Purpose + +Libra supports worktrees whose POSIX files may come from different storage +systems. The Git model must remain identical across local directories, +repository-aware lazy projections such as ScorpioFS, and persistent distributed +volumes such as BrewFS. + +The architecture separates three planes: + +```text +Git plane + Libra refs, index, objects, commits, fetch, and push + +Control plane + Libra worktree coordinator, desired state, locks, worker supervision, + backend capability negotiation, recovery, and cleanup + +Data plane + Local directory, ScorpioFS FUSE mount, BrewFS FUSE mount, or a future + POSIX-visible backend +``` + +Libra owns the first two planes. A backend driver owns only its data-plane +session. + +## Core contract + +`internal::worktree_backend` defines: + +- `BackendKind`; +- `BackendCapabilities`; +- `BackendMountSource`; +- `BackendMountRequest`; +- `BackendMountSession`; +- `BackendHealth`; +- `BackendLifecycle`; +- `WorktreeBackendDriver`; +- `BackendRegistry`. + +The driver contract contains lifecycle operations rather than a duplicate +filesystem API: + +```text +mount +health +changed_paths (optional) +flush (optional) +unmount +recover +``` + +Build tools, editors, and ordinary Libra commands access the mounted POSIX +path. They do not call a backend SDK for individual reads and writes. + +## Capability model + +| Capability | Local | ScorpioFS | BrewFS | +|---|---:|---:|---:| +| POSIX worktree | yes | yes | yes | +| Revision projection | no | yes | no | +| Native changed paths | no | yes | no | +| Persistent volume | no | no | yes | +| Multi-client storage | no | no | yes | +| Flush before commit | no | no | yes | + +Command code must branch on capabilities, not concrete backend names. + +## Backend source types + +The generic mount request distinguishes: + +```text +local_directory +remote_projection +persistent_volume +``` + +ScorpioFS accepts `remote_projection`, including a monorepo path, base object +ID, and optional change layer. + +BrewFS accepts `persistent_volume`, including a volume and optional subpath. +BrewFS does not inherently project a Mega commit. Libra must populate or import +the selected Git tree before treating a new BrewFS volume as a worktree. + +## Process model + +The target process model is: + +```text +libra CLI + -> Libra worktree supervisor + -> ScorpioFS worker linked to the ScorpioFS crate + -> BrewFS worker linked to the BrewFS crate +``` + +FUSE sessions outlive an individual CLI invocation. Backend workers also +isolate filesystem crashes and dependency runtimes from the Git command +process. Unix domain sockets should replace loopback HTTP as the default local +control transport; the current ScorpioFS loopback protocol remains a +compatibility transport during migration. + +Libra owns worker selection, desired state, and recovery. ScorpioFS owns only +live mount state. The configured endpoint is a control-plane address, never a +Git remote and never a credential container. + +## Persistent layout + +Libra metadata remains on a host-local filesystem: + +```text +/.libra/ + objects/ + refs/ + worktrees/ + backends/ + desired-state.json + state.lock +``` + +Backend caches and runtime state remain separate: + +```text +~/.cache/libra/backends/scorpiofs// +~/.cache/libra/backends/brewfs// +/run/user//libra/ +``` + +`.libra` must not be stored in a ScorpioFS upper layer or a BrewFS volume. A +mounted worktree contains only a reconstructable `.libra` pointer to its +host-local per-worktree gitdir. + +### State reconciliation + +The persistent Libra record is authoritative. A live backend mount is an +execution resource that may disappear after a process or machine restart. +Recovery must read and validate host-local Libra state, query the backend by +its durable cleanup key, recreate the pointer only after containment checks, +and record failures as recoverable rather than silently dropping desired state. + +## ScorpioFS adapter + +`ScorpioFsDriver` implements `WorktreeBackendDriver` by translating generic +remote-projection requests into Antares mount requests. It exposes native +changed-path candidates and idempotent cleanup by job ID. + +The existing ScorpioFS command and state files remain compatible while command +orchestration is incrementally moved onto the generic driver. + +## BrewFS SDK boundary + +`BrewFsDriver` accepts a `BrewFsRuntime`. The runtime is responsible for: + +- constructing BrewFS metadata and object backends from named profiles; +- retaining the BrewFS SDK client and FUSE handle; +- mounting a persistent volume; +- reporting health; +- draining writes before Git commit publication; +- unmounting the session. + +Configuration stores profile names, not credentials: + +```toml +[backends.brewfs.team] +volume = "team-workspace" +mount_root = "/home/alice/libra-workspaces" +metadata_profile = "production-redis" +data_profile = "production-s3" +``` + +BrewFS 0.1.2 exports filesystem clients but keeps the complete mount assembly +used by its binary private. Libra therefore does not claim direct embedded +mount support until BrewFS exports a stable mount builder/session API. The +runtime trait is the integration seam for that API. + +The minimum upstream SDK shape Libra needs is: + +```rust +pub struct MountBuilder { /* metadata, object, cache, and FUSE options */ } + +impl MountBuilder { + pub async fn mount(self, mountpoint: &Path) -> Result; +} + +pub struct MountedFs { /* SDK client and FUSE handle */ } + +impl MountedFs { + pub fn client(&self) -> &brewfs::Client; + pub async fn health(&self) -> Result; + pub async fn flush(&self) -> Result<()>; + pub async fn unmount(self) -> Result<()>; +} +``` + +The handle must retain all background workers and expose bounded graceful +shutdown. Configuration construction must accept credential references or +preconstructed backends so Libra never serializes secrets into `.libra`. + +## Commit durability + +For a backend with `flush_before_commit`, Libra must: + +1. finish index updates; +2. request backend flush; +3. wait for durable completion or fail the commit; +4. construct and publish the Git commit; +5. update refs and reflogs. + +This ordering prevents a commit from naming worktree content that remains only +in an unflushed client buffer. + +## Migration + +1. Keep existing `worktree scorpiofs attach/detach` behavior. +2. Route ScorpioFS changed-path discovery through `ScorpioFsDriver`. +3. Move attach, health, recovery, and detach orchestration to the generic + driver. +4. Introduce a generic `worktree create --backend` command. +5. Add the BrewFS crate after its stable mount session API is available. +6. Implement a BrewFS SDK runtime and persistent-volume checkout/import flow. +7. Migrate legacy `.libra/scorpiofs/state.json` into versioned backend-neutral + desired state. + +## Current validation boundary + +The ScorpioFS implementation is validated for an attached remote projection: +`attach`, `status`, `fetch`, `add`, `commit`, explicit-refspec +`push --dry-run`, and `detach`. Validation uses the deployed Mega +`project/aardvark-dns` repository inside an isolated Linux user and mount +namespace. + +This does not claim every mutable Git operation or automatic base switch is +production-ready. Transactional base switching, restart recovery, concurrent +mount stress, and a direct embedded BrewFS SDK mount remain planned work. diff --git a/src/cli.rs b/src/cli.rs index a37e7a498..c4bbb349b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -737,6 +737,8 @@ enum Commands { hide = true )] Hooks(command::hooks::HooksArgs), + #[command(about = "Run a Libra-owned ScorpioFS worker", hide = true)] + ScorpiofsWorker(command::scorpiofs_worker::ScorpioFsWorkerArgs), } #[derive(Subcommand, Debug)] @@ -1801,6 +1803,12 @@ fn command_scope(command: &Commands) -> CommandScope { | Commands::Revision(_) | Commands::Hooks(_) | Commands::Service(_) + // The hidden resident ScorpioFS worker is a daemon like `service`: + // it owns no Libra state (durable desired state stays with the + // `worktree scorpiofs` commands), but it is classified as a writer + // rather than read-only so a future publication cannot slip through + // an under-claimed scope. Its LIFETIME hold is dropped below. + | Commands::ScorpiofsWorker(_) | Commands::Lfs(_) | Commands::Deps(_) | Commands::Auth(_) @@ -1918,6 +1926,10 @@ fn command_holds_shared_maintenance_lock(command: &Commands) -> bool { | Commands::Automation(_) | Commands::Sandbox(_) | Commands::Service(_) + // The ScorpioFS worker outlives the `worktree scorpiofs attach` + // that spawned it; a lifetime hold would starve every deletion + // phase for as long as the mount is up. + | Commands::ScorpiofsWorker(_) | Commands::Agent(_) | Commands::Review(_) | Commands::Investigate(_) @@ -2888,6 +2900,9 @@ async fn parse_async_scoped(argv: Vec) -> CliResult<()> { command::agent::investigate::execute_safe(cmd_args, &output).await? } Commands::Hooks(cmd_args) => command::hooks::execute_safe(cmd_args, &output).await?, + Commands::ScorpiofsWorker(cmd_args) => { + command::scorpiofs_worker::execute_safe(cmd_args).await? + } Commands::Bisect(bisect_cmd) => { command::bisect::execute_safe(bisect_cmd, &output).await? } diff --git a/src/command/add.rs b/src/command/add.rs index b5173e4fa..a24b1d9d3 100644 --- a/src/command/add.rs +++ b/src/command/add.rs @@ -538,13 +538,27 @@ pub async fn run_add(args: &AddArgs) -> CliResult { ignore_case, }; - let (mut visible_changes, mut ignored_changes) = if args.force { - status::changes_to_be_staged_split_force_with_ignore_case(ignore_case) - .map_err(|source| AddError::Status { source })? - } else { - status::changes_to_be_staged_split_safe_with_ignore_case(ignore_case) - .map_err(|source| AddError::Status { source })? - }; + let backend_changes = crate::internal::scorpiofs_backend::current_worktree_changes() + .await + .map_err(|error| { + CliError::fatal(format!( + "failed to query ScorpioFS worktree changes: {error}" + )) + })?; + let backend_candidates = backend_changes + .as_ref() + .map(|changes| changes.candidate_paths()); + let (mut visible_changes, mut ignored_changes) = + if let Some(candidates) = backend_candidates.as_deref() { + status::changes_to_be_staged_split_for_paths_with_ignore_case(ignore_case, candidates) + .map_err(|source| AddError::Status { source })? + } else if args.force { + status::changes_to_be_staged_split_force_with_ignore_case(ignore_case) + .map_err(|source| AddError::Status { source })? + } else { + status::changes_to_be_staged_split_safe_with_ignore_case(ignore_case) + .map_err(|source| AddError::Status { source })? + }; if args.force { visible_changes.extend(ignored_changes.clone()); ignored_changes = Changes::default(); diff --git a/src/command/maintenance.rs b/src/command/maintenance.rs index 704f9fad4..7ed0ce264 100644 --- a/src/command/maintenance.rs +++ b/src/command/maintenance.rs @@ -2821,6 +2821,17 @@ pub const GC_OBJECT_FILE_SOURCE_INVENTORY: &[GcObjectSource] = &[ corruption: GcCorruptionPolicy::NotApplicable, note: "the shared rr-cache stores pre/postimage FILE BYTES under conflict-id directories, outside the object store entirely; `rerere gc` ages them out on its own schedule", }, + GcObjectSource { + origin: GcSourceOrigin::File, + location: "/scorpiofs", + column: "", + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::JsonManifest, + schema: "LibraScorpioFsState (schema_version, worker record, mounts[] with request.base_oid), plus the state lock and the managed worker's log", + read_bound: "never read by GC", + corruption: GcCorruptionPolicy::NotApplicable, + note: "the ScorpioFS backend's DESIRED-state file: `base_oid` names a REMOTE monorepo revision the mount projects, not a local object-store id, so rooting it would demand objects this repository never wrote (same shape as MERGE_RR). Libra owns the durable desired state here precisely because the embedded worker persists none; the mount's own Git writes land in the object store through the ordinary commit path, which roots them", + }, GcObjectSource { origin: GcSourceOrigin::File, location: "/merge-file-backup", diff --git a/src/command/mod.rs b/src/command/mod.rs index 0e6dd2b3d..bc83bc4de 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -108,6 +108,7 @@ pub mod rev_parse; pub mod revert; pub mod revision; pub mod sandbox; +pub mod scorpiofs_worker; pub mod service; pub mod shortlog; pub mod show; diff --git a/src/command/push.rs b/src/command/push.rs index aace829fc..fbc9424a3 100644 --- a/src/command/push.rs +++ b/src/command/push.rs @@ -785,12 +785,16 @@ fn validate_push_args(args: &PushArgs) -> Result<(), PushError> { Ok(()) } -async fn validate_local_refspecs(args: &PushArgs, current_branch: &str) -> Result<(), PushError> { +async fn validate_local_refspecs( + args: &PushArgs, + current_branch: Option<&str>, +) -> Result<(), PushError> { if args.mirror { return Ok(()); } if args.refspecs.is_empty() && !args.tags { + let current_branch = current_branch.ok_or(PushError::DetachedHead)?; resolve_local_ref(current_branch).await?; } @@ -819,13 +823,14 @@ pub async fn run_push(args: PushArgs, output: &OutputConfig) -> Result name, - Head::Detached(_) => return Err(PushError::DetachedHead), + Head::Branch(name) => Some(name), + Head::Detached(_) => None, }; let repository = match args.repository.clone() { Some(repo) => repo, None => { - let remote = ConfigKv::get_remote(¤t_branch).await.ok().flatten(); + let current_branch = current_branch.as_deref().ok_or(PushError::DetachedHead)?; + let remote = ConfigKv::get_remote(current_branch).await.ok().flatten(); match remote { Some(remote) => remote, None => return Err(PushError::NoRemoteConfigured), @@ -850,7 +855,7 @@ pub async fn run_push(args: PushArgs, output: &OutputConfig) -> Result Result Result crate::utils::test::ChangeDirGuard { + let gitdir = root.join(crate::utils::util::ROOT_DIR); + std::fs::create_dir_all(gitdir.join("objects")).expect("create object store"); + std::fs::write(gitdir.join(crate::utils::util::DATABASE), b"") + .expect("create repository db"); + crate::utils::test::ChangeDirGuard::new(root) + } + fn regular(evidence: BlobEvidence, size: u64) -> BlobRef { BlobRef { kind: BlobKind::Regular, @@ -2288,6 +2304,12 @@ mod tests { const SHRUNK: u64 = 4; const GROWTH: u64 = 4096; let dir = tempfile::tempdir().expect("tempdir"); + // The read path classifies LFS through `attribute_state_for_path`, + // which resolves `working_dir()` — infallibly, from the PROCESS cwd, + // on the pooled io thread. Outside a repository that panics the + // worker and the caller only ever sees the resulting `IoTimeout`, + // so the fixture has to be a repository. + let _repo = repo_fixture(dir.path()); let path = dir.path().join("grows.txt"); std::fs::write(&path, vec![b'a'; (SHRUNK + GROWTH) as usize]) .expect("write the full-size file"); @@ -2456,6 +2478,11 @@ mod tests { use std::time::Duration; let dir = tempfile::tempdir().expect("tempdir"); + // See `worktree_total_charges_bytes_read_not_stale_stat`: the LFS + // classification resolves `working_dir()` on the io thread, so a + // repo-less fixture fails with `IoTimeout` no matter what the seams + // are doing — which is precisely the outcome this test denies. + let _repo = repo_fixture(dir.path()); let path = dir.path().join("fast.txt"); std::fs::write(&path, b"content").expect("write fixture"); @@ -2466,8 +2493,21 @@ mod tests { std::env::set_var("LIBRA_TEST_SLOW_WORKTREE_READ_MS", "5000"); std::env::set_var("LIBRA_TEST_SLOW_LFS_ATTRIBUTES_MS", "5000"); } - let mut budget = WorktreeReadBudget::new(1024, 1024, 8, Duration::from_millis(1500)); - let outcome = budget.read_worktree_blob(&path); + // A jammed I/O pool and a fired seam both surface as `IoTimeout`, and + // the pool is process-global: the seam tests above abandon reads that + // keep sleeping in a worker, and their slots outlive them. So drain + // the pool first and retry — with the seams inert one attempt + // succeeds as soon as a worker is free, while a 5s seam that really + // did fire would blow the 1.5s batch on EVERY attempt. + let mut outcome = ContentOutcome::Skipped(SkipReason::IoTimeout); + for _ in 0..8 { + crate::command::status_probe::wait_for_idle_io_pool(); + let mut budget = WorktreeReadBudget::new(1024, 1024, 8, Duration::from_millis(1500)); + outcome = budget.read_worktree_blob(&path); + if matches!(outcome, ContentOutcome::Content(_)) { + break; + } + } unsafe { std::env::remove_var("LIBRA_TEST_SLOW_WORKTREE_STAT_MS"); std::env::remove_var("LIBRA_TEST_SLOW_WORKTREE_READ_MS"); diff --git a/src/command/scorpiofs_worker.rs b/src/command/scorpiofs_worker.rs new file mode 100644 index 000000000..8651fd67b --- /dev/null +++ b/src/command/scorpiofs_worker.rs @@ -0,0 +1,73 @@ +//! Libra-owned resident ScorpioFS worker. +//! +//! The public Libra CLI is intentionally short-lived, while a FUSE session +//! must remain alive after `worktree scorpiofs attach` returns. Libra therefore +//! starts this hidden worker from its own executable. The worker links the +//! ScorpioFS crate directly and exposes only a loopback control endpoint. +//! Durable desired state remains in Libra; the embedded ScorpioFS service is +//! explicitly configured not to persist or recover state itself. + +use std::{net::SocketAddr, path::PathBuf}; + +use clap::Parser; + +use crate::utils::error::{CliError, CliResult, StableErrorCode}; + +#[derive(Parser, Debug)] +pub struct ScorpioFsWorkerArgs { + #[arg(long)] + pub config_path: PathBuf, + #[arg(long)] + pub bind: SocketAddr, + #[arg(long)] + pub upper_root: PathBuf, + #[arg(long)] + pub cl_root: PathBuf, + #[arg(long)] + pub mount_root: PathBuf, + #[arg(long)] + pub runtime_state_file: PathBuf, +} + +#[cfg(all(target_os = "linux", feature = "scorpiofs-direct"))] +pub async fn execute_safe(args: ScorpioFsWorkerArgs) -> CliResult<()> { + use std::sync::Arc; + + use scorpiofs::{ + cli, + daemon::antares::{AntaresDaemon, AntaresServiceImpl}, + util::config, + }; + + let config_path = args.config_path.to_str().ok_or_else(|| { + CliError::fatal("ScorpioFS config path is not valid UTF-8") + .with_stable_code(StableErrorCode::CliInvalidTarget) + })?; + let overrides = cli::antares_overrides( + Some(args.upper_root), + Some(args.cl_root), + Some(args.mount_root), + Some(args.runtime_state_file), + ); + config::init_config_with(config_path, overrides).map_err(|error| { + CliError::fatal(format!("failed to initialize embedded ScorpioFS: {error}")) + .with_stable_code(StableErrorCode::RepoStateInvalid) + })?; + + let service = Arc::new(AntaresServiceImpl::new_external_state(None).await); + AntaresDaemon::new(service) + .serve(args.bind) + .await + .map_err(|error| { + CliError::fatal(format!("Libra ScorpioFS worker failed: {error}")) + .with_stable_code(StableErrorCode::IoWriteFailed) + }) +} + +#[cfg(not(all(target_os = "linux", feature = "scorpiofs-direct")))] +pub async fn execute_safe(_args: ScorpioFsWorkerArgs) -> CliResult<()> { + Err( + CliError::fatal("the direct ScorpioFS worker requires Linux and scorpiofs-direct") + .with_stable_code(StableErrorCode::Unsupported), + ) +} diff --git a/src/command/status.rs b/src/command/status.rs index 9684f6c8d..47695753a 100644 --- a/src/command/status.rs +++ b/src/command/status.rs @@ -1330,11 +1330,30 @@ async fn collect_status_data( .await .map(|c| c.to_relative()) .map_err(CliError::from)?; - let worktree = status_untracked::collect_status_worktree_changes( - args.untracked_files.unwrap_or(UntrackedFiles::Normal), - args.ignored, - ignore_case, - ) + let backend_changes = crate::internal::scorpiofs_backend::current_worktree_changes() + .await + .map_err(|error| { + CliError::fatal(format!( + "failed to query ScorpioFS worktree changes: {error}" + )) + })?; + let backend_candidates = backend_changes + .as_ref() + .map(|changes| changes.candidate_paths()); + let worktree = if let Some(candidates) = backend_candidates.as_deref() { + status_untracked::collect_status_worktree_changes_for_paths( + args.untracked_files.unwrap_or(UntrackedFiles::Normal), + args.ignored, + ignore_case, + candidates, + ) + } else { + status_untracked::collect_status_worktree_changes( + args.untracked_files.unwrap_or(UntrackedFiles::Normal), + args.ignored, + ignore_case, + ) + } .map_err(CliError::from)?; let mut unstaged = status_untracked::changes_to_current_directory(worktree.unstaged); let unmerged = unmerged::collect(&worktree.index) @@ -6513,6 +6532,23 @@ pub(crate) fn changes_to_be_staged_split_safe_with_ignore_case( changes_to_be_staged_split_with_index(&workdir, &index, ignore_case) } +pub(crate) fn changes_to_be_staged_split_for_paths_with_ignore_case( + ignore_case: bool, + candidates: &[PathBuf], +) -> Result<(Changes, Changes), StatusError> { + let collected = status_untracked::collect_status_worktree_changes_for_paths( + UntrackedFiles::All, + true, + ignore_case, + candidates, + )?; + let ignored = Changes { + new: collected.ignored_files, + ..Default::default() + }; + Ok((collected.unstaged, ignored)) +} + /// List changes to be staged with --force semantics (recurse into ignored directories) pub fn changes_to_be_staged_split_force() -> Result<(Changes, Changes), StatusError> { let workdir = util::try_working_dir().map_err(|source| StatusError::Workdir { source })?; diff --git a/src/command/status_probe.rs b/src/command/status_probe.rs index a4f2114e4..ed15cba45 100644 --- a/src/command/status_probe.rs +++ b/src/command/status_probe.rs @@ -129,6 +129,24 @@ static IO_POOL: std::sync::OnceLock> = std::sync::O static IO_BUSY: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); static IO_WORKERS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); +/// Block until every pooled slot has been released. +/// +/// The deadline tests deliberately ABANDON reads: the caller gives up at its +/// deadline while the worker keeps sleeping inside the armed seam, and the +/// slot stays busy until that sleep ends — outliving the test that started +/// it. A later test whose contract is "this must NOT time out" therefore has +/// to start from an idle pool, or it measures the previous test's leftover +/// delay and fails for a reason that has nothing to do with what it asserts. +#[cfg(test)] +pub(crate) fn wait_for_idle_io_pool() { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while IO_BUSY.load(std::sync::atomic::Ordering::SeqCst) > 0 + && std::time::Instant::now() < deadline + { + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + /// Run one blocking filesystem operation with a wall-clock deadline. /// /// A hung syscall cannot be interrupted in safe Rust, so the operation runs diff --git a/src/command/status_untracked.rs b/src/command/status_untracked.rs index d90fa094b..a9906483d 100644 --- a/src/command/status_untracked.rs +++ b/src/command/status_untracked.rs @@ -67,6 +67,29 @@ pub(crate) fn collect_status_worktree_changes( untracked_mode: UntrackedFiles, include_ignored: bool, ignore_case: bool, +) -> Result { + collect_status_worktree_changes_inner(untracked_mode, include_ignored, ignore_case, None) +} + +pub(crate) fn collect_status_worktree_changes_for_paths( + untracked_mode: UntrackedFiles, + include_ignored: bool, + ignore_case: bool, + candidates: &[PathBuf], +) -> Result { + collect_status_worktree_changes_inner( + untracked_mode, + include_ignored, + ignore_case, + Some(candidates), + ) +} + +fn collect_status_worktree_changes_inner( + untracked_mode: UntrackedFiles, + include_ignored: bool, + ignore_case: bool, + candidates: Option<&[PathBuf]>, ) -> Result { let workdir = util::try_working_dir().map_err(|source| StatusError::Workdir { source })?; let index_path = path::try_index().map_err(|source| StatusError::Workdir { source })?; @@ -75,6 +98,17 @@ pub(crate) fn collect_status_worktree_changes( source, })?; let tracked = TrackedPaths::from_index(&index, ignore_case); + let tracked_candidates; + let tracked_files = if let Some(candidates) = candidates { + tracked_candidates = candidates + .iter() + .filter(|path| path.to_str().is_some_and(|path| index.tracked(path, 0))) + .cloned() + .collect::>(); + tracked_candidates.as_slice() + } else { + tracked.files() + }; let mut io_blocked: Vec = Vec::new(); // The index file's own mtime anchors the racily-clean guard: a stat // triple is only trusted for files strictly older than this snapshot. @@ -84,14 +118,18 @@ pub(crate) fn collect_status_worktree_changes( let mut unstaged = collect_tracked_worktree_changes( &workdir, &index, - tracked.files(), + tracked_files, &mut io_blocked, index_file_mtime, )?; let mut ignored_files = Vec::new(); if !matches!(untracked_mode, UntrackedFiles::No) { - let mut scan = scan_workdir(&workdir, &index, &tracked, untracked_mode, include_ignored)?; + let mut scan = if let Some(candidates) = candidates { + scan_candidate_paths(&workdir, &index, &tracked, candidates, include_ignored)? + } else { + scan_workdir(&workdir, &index, &tracked, untracked_mode, include_ignored)? + }; io_blocked.append(&mut scan.io_blocked); unstaged.new = if matches!(untracked_mode, UntrackedFiles::Normal) { collapse_untracked_directories(scan.untracked, &tracked) @@ -115,6 +153,45 @@ pub(crate) fn collect_status_worktree_changes( }) } +fn scan_candidate_paths( + workdir: &Path, + index: &Index, + tracked: &TrackedPaths, + candidates: &[PathBuf], + include_ignored: bool, +) -> Result { + let mut scan = WorkdirScan { + untracked: Vec::new(), + ignored: Vec::new(), + io_blocked: Vec::new(), + }; + for relative in candidates { + let path = workdir.join(relative); + let file_type = match path.symlink_metadata() { + Ok(metadata) => metadata.file_type(), + Err(source) if source.kind() == io::ErrorKind::NotFound => continue, + Err(source) => { + return Err(StatusError::WorktreeRead { + path: path.clone(), + source, + }); + } + }; + if file_type.is_file() || file_type.is_symlink() { + scan_file( + &mut scan, + workdir, + index, + tracked, + &path, + relative, + include_ignored, + )?; + } + } + Ok(scan) +} + pub(crate) fn changes_to_current_directory(mut changes: Changes) -> Changes { changes.new = changes .new diff --git a/src/command/worktree-fuse.rs b/src/command/worktree-fuse.rs index 86b8c5faf..b4fcffaa4 100644 --- a/src/command/worktree-fuse.rs +++ b/src/command/worktree-fuse.rs @@ -90,6 +90,11 @@ pub enum WorktreeSubcommand { #[clap(long, help = "Allow other users to access the mounted worktree")] allow_other: bool, }, + /// Manage a linked worktree backed by a ScorpioFS Antares mount. + Scorpiofs { + #[clap(subcommand)] + command: legacy::ScorpioFsSubcommand, + }, List { /// Emit a stable, machine-readable porcelain format (one attribute per /// line, blank line between worktrees). @@ -431,6 +436,15 @@ pub async fn execute_safe(args: WorktreeArgs, output: &OutputConfig) -> CliResul .map_err(|e| CliError::fatal(e.to_string())) } } + WorktreeSubcommand::Scorpiofs { command } => { + legacy::execute_safe( + legacy::WorktreeArgs { + command: legacy::WorktreeSubcommand::Scorpiofs { command }, + }, + output, + ) + .await + } WorktreeSubcommand::List { porcelain, schema_version, diff --git a/src/command/worktree.rs b/src/command/worktree.rs index 3d8c74986..310e35ae4 100644 --- a/src/command/worktree.rs +++ b/src/command/worktree.rs @@ -8,6 +8,7 @@ use std::{ collections::HashSet, env, fs, io, path::{Path, PathBuf}, + time::Duration, }; use clap::{Parser, Subcommand}; @@ -21,10 +22,18 @@ use crate::{ internal::{ branch::Branch, head::Head, + scorpiofs_backend::{ + CAPABILITY_CHANGES_V1, CAPABILITY_MOUNT_V1, CAPABILITY_READY_V1, HttpScorpioFsClient, + MountRequest, MountResponse, ScorpioFsBackendRecord, ScorpioFsControl, ScorpioFsDriver, + }, sequencer::WorktreeControl, workspace::{ self, RepoIdentity, WorkspaceKind, WorkspaceRecord, WorkspaceState, WorkspaceStore, }, + worktree_backend::{ + BackendKind as WorktreeBackendKind, BackendMountRequest as StorageMountRequest, + BackendMountSession as StorageMountSession, BackendMountSource, WorktreeBackendDriver, + }, }, utils::{ error::{CliError, CliResult, StableErrorCode}, @@ -42,6 +51,9 @@ EXAMPLES: libra worktree add --detach ../probe v1.2.0 Detached worktree at a commit-ish libra worktree add -b topic ../topic main Create branch `topic` from `main` and check it out + libra worktree scorpiofs attach --remote-path /project/crate --job-id dev-crate + Attach a ScorpioFS remote worktree + libra worktree scorpiofs detach Detach the remote worktree libra worktree list List every registered worktree libra worktree list --porcelain Machine-readable worktree list libra worktree lock ../feature-x --reason wip Lock a worktree to prevent prune/remove @@ -105,6 +117,11 @@ pub enum WorktreeSubcommand { #[clap(short = 'b', long = "create-branch", value_name = "NEW_BRANCH")] new_branch: Option, }, + /// Manage a linked worktree backed by a ScorpioFS Antares mount. + Scorpiofs { + #[clap(subcommand)] + command: ScorpioFsSubcommand, + }, /// List all known worktrees and their state. List { /// Emit a stable, machine-readable porcelain format (one attribute per @@ -266,6 +283,39 @@ pub enum WorktreeSubcommand { }, } +#[derive(Subcommand, Debug)] +pub enum ScorpioFsSubcommand { + /// Mount a remote monorepo path and attach it as a persistent linked worktree. + Attach { + /// Antares API base URL. + /// + /// When omitted on Linux, Libra starts and owns an embedded ScorpioFS + /// worker. Supplying this option selects the compatibility HTTP mode. + #[clap(long)] + endpoint: Option, + /// ScorpioFS configuration used by the Libra-owned worker. + #[clap(long, default_value = "scorpio.toml")] + config_path: PathBuf, + /// Absolute path inside the remote monorepo. + #[clap(long)] + remote_path: String, + /// Stable job identity used for idempotent mount creation and recovery. + #[clap(long)] + job_id: String, + /// Optional Mega changelist layer. + #[clap(long)] + cl: Option, + /// Maximum number of seconds to wait for mount readiness. + #[clap(long, default_value_t = 120)] + ready_timeout_secs: u64, + }, + /// Unmount and unregister a ScorpioFS-backed linked worktree. + Detach { + /// Mounted worktree path. + path: String, + }, +} + /// A single worktree entry persisted in `worktrees.json` (registry v2, /// plan-20260714 §C.7). /// @@ -799,6 +849,24 @@ struct WorktreeAddOutput { reattached: bool, } +#[derive(Debug, Serialize)] +struct ScorpioFsAttachOutput { + path: String, + worktree_id: String, + mount_id: String, + job_id: String, + already_exists: bool, +} + +#[derive(Debug, Serialize)] +struct ScorpioFsDetachOutput { + path: String, + worktree_id: String, + job_id: String, + unmounted: bool, + registry_removed: bool, +} + #[derive(Debug, Serialize)] struct WorktreeLockOutput { path: String, @@ -1161,6 +1229,34 @@ pub async fn execute_safe(args: WorktreeArgs, output: &OutputConfig) -> CliResul .map_err(WorktreeError::into_cli_error)?; render_add_worktree(&result, output) } + WorktreeSubcommand::Scorpiofs { command } => match command { + ScorpioFsSubcommand::Attach { + endpoint, + config_path, + remote_path, + job_id, + cl, + ready_timeout_secs, + } => { + let result = attach_scorpiofs_worktree( + endpoint, + config_path, + remote_path, + job_id, + cl, + ready_timeout_secs, + ) + .await + .map_err(WorktreeError::into_cli_error)?; + render_scorpiofs_attach(&result, output) + } + ScorpioFsSubcommand::Detach { path } => { + let result = detach_scorpiofs_worktree(path) + .await + .map_err(WorktreeError::into_cli_error)?; + render_scorpiofs_detach(&result, output) + } + }, WorktreeSubcommand::List { porcelain, schema_version, @@ -2937,6 +3033,912 @@ async fn reattach_worktree( }) } +async fn attach_scorpiofs_worktree( + endpoint: Option, + config_path: PathBuf, + remote_path: String, + job_id: String, + cl: Option, + ready_timeout_secs: u64, +) -> WorktreeResult { + let storage = util::storage_path(); + let _state_lock = crate::internal::scorpiofs_backend::ScorpioFsStateLock::acquire(&storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + let (endpoint, transport) = match endpoint { + Some(endpoint) => ( + endpoint, + crate::internal::scorpiofs_backend::BackendTransport::ExternalHttp, + ), + None => ( + ensure_managed_scorpiofs_worker( + &storage, + &config_path, + Duration::from_secs(ready_timeout_secs), + ) + .await?, + crate::internal::scorpiofs_backend::BackendTransport::ManagedCrate, + ), + }; + let seed_commit = Head::current_commit_result().await.map_err(|error| { + WorktreeError::IoRead(format!( + "failed to read HEAD before attaching ScorpioFS worktree: {error}" + )) + })?; + let client = HttpScorpioFsClient::new(&endpoint) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + let request = MountRequest { + job_id: job_id.clone(), + path: remote_path, + cl, + base_oid: None, + }; + let mut desired_state = crate::internal::scorpiofs_backend::LibraScorpioFsState::load(&storage) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + let previous_mountpoint = desired_state + .mounts + .get(&job_id) + .and_then(|desired| desired.mountpoint.clone()); + desired_state + .begin_mount(request.clone(), transport, endpoint.clone()) + .map_err(|error| WorktreeError::OperationBlocked(error.to_string()))?; + desired_state + .save(&storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + + let driver = ScorpioFsDriver::new(client.clone()); + let backend_request = StorageMountRequest { + instance_id: job_id.clone(), + worktree_id: job_id.clone(), + source: BackendMountSource::RemoteProjection { + remote_path: request.path.clone(), + base_oid: request.base_oid.clone(), + change_layer: request.cl.clone(), + }, + mountpoint_hint: None, + ready_timeout_secs, + }; + let backend_session = match driver.mount(&backend_request).await { + Ok(session) => session, + Err(error) => { + record_scorpiofs_attach_error(&storage, &job_id, error.to_string()); + return Err(WorktreeError::IoWrite(error.to_string())); + } + }; + let mount = MountResponse { + mount_id: backend_session.session_id, + mountpoint: backend_session.mountpoint.to_string_lossy().into_owned(), + base_oid: backend_session.base_oid, + ready: Some(true), + }; + + let target = match resolve_path(&mount.mountpoint, "ScorpioFS mountpoint") { + Ok(target) => target, + Err(error) => { + let _ = client.delete_by_job(&job_id).await; + record_scorpiofs_attach_error(&storage, &job_id, format!("{error:?}")); + return Err(error); + } + }; + if !target.is_dir() { + let _ = client.delete_by_job(&job_id).await; + let error = WorktreeError::InvalidTarget(format!( + "ScorpioFS mountpoint is not a directory: {}", + target.display() + )); + record_scorpiofs_attach_error(&storage, &job_id, format!("{error:?}")); + return Err(error); + } + if util::is_sub_path(&target, &storage) { + let _ = client.delete_by_job(&job_id).await; + let error = WorktreeError::InvalidTarget(format!( + "ScorpioFS mountpoint cannot be inside .libra storage: {}", + target.display() + )); + record_scorpiofs_attach_error(&storage, &job_id, format!("{error:?}")); + return Err(error); + } + + let mut state = load_state()?; + if let Some(previous_mountpoint) = previous_mountpoint.as_deref() + && Path::new(previous_mountpoint) != target + { + state + .entries + .retain(|entry| Path::new(&entry.path) != Path::new(previous_mountpoint)); + } + if state + .entries + .iter() + .any(|entry| Path::new(&entry.path) == target) + { + let gitdir = util::try_get_worktree_gitdir(Some(target.clone())).map_err(|error| { + WorktreeError::IoRead(format!( + "registered ScorpioFS worktree '{}' has invalid metadata: {error}", + target.display() + )) + })?; + let record = ScorpioFsBackendRecord::load(&gitdir) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + if record.job_id != job_id || record.remote_path != request.path { + return Err(WorktreeError::OperationBlocked(format!( + "worktree '{}' is already registered for ScorpioFS job '{}' and path '{}'", + target.display(), + record.job_id, + record.remote_path + ))); + } + desired_state + .mark_ready(&job_id, &mount, &read_worktree_id(&gitdir)?) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + desired_state + .save(&storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + return Ok(ScorpioFsAttachOutput { + path: target.to_string_lossy().into_owned(), + worktree_id: read_worktree_id(&gitdir)?, + mount_id: mount.mount_id, + job_id, + already_exists: true, + }); + } + + let backend_identity = match transport { + crate::internal::scorpiofs_backend::BackendTransport::ManagedCrate => "managed-crate", + crate::internal::scorpiofs_backend::BackendTransport::ExternalHttp => { + client.endpoint().as_str() + } + }; + let worktree_id = scorpiofs_worktree_id(backend_identity, &job_id, &request.path); + let gitdir = storage + .join("worktrees") + .join("scorpiofs") + .join(&worktree_id); + let pointer = target.join(util::ROOT_DIR); + let created_gitdir = !gitdir.exists(); + if created_gitdir { + create_worktree_gitdir(&storage, &gitdir, &worktree_id).map_err(|source| { + WorktreeError::IoWrite(format!( + "failed to create persistent ScorpioFS worktree gitdir '{}': {source}", + gitdir.display() + )) + })?; + } + + let record = match ScorpioFsBackendRecord::new_with_transport( + client.endpoint(), + &mount, + &request, + transport, + ) { + Ok(record) => record, + Err(error) => { + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + false, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoWrite(error.to_string())); + } + }; + + if !created_gitdir { + let existing = ScorpioFsBackendRecord::load(&gitdir).map_err(|error| { + WorktreeError::OperationBlocked(format!( + "persistent ScorpioFS worktree '{}' cannot be reused: {error}", + gitdir.display() + )) + })?; + if existing.job_id != job_id + || existing.remote_path != request.path + || existing.transport != record.transport + || (transport == crate::internal::scorpiofs_backend::BackendTransport::ExternalHttp + && existing.endpoint != record.endpoint) + { + let _ = client.delete_by_job(&job_id).await; + return Err(WorktreeError::OperationBlocked(format!( + "persistent ScorpioFS worktree id '{}' belongs to another backend attachment", + worktree_id + ))); + } + } + if let Err(error) = record.save(&gitdir) { + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + false, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoWrite(error.to_string())); + } + + let created_pointer = match attach_worktree_pointer(&pointer, &gitdir) { + Ok(created) => created, + Err(source) => { + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + false, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoWrite(format!( + "failed to attach persistent metadata at '{}': {source}", + pointer.display() + ))); + } + }; + + if let Some(commit) = seed_commit { + let guard = match DirGuard::change_to(&target) { + Ok(guard) => guard, + Err(error) => { + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + created_pointer, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoRead(format!( + "failed to enter ScorpioFS worktree '{}': {error}", + target.display() + ))); + } + }; + if let Err(error) = Head::update_result(Head::Detached(commit), None).await { + drop(guard); + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + created_pointer, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoWrite(format!( + "failed to seed ScorpioFS worktree HEAD: {error}" + ))); + } + if let Err(error) = restore::execute_checked(RestoreArgs { + overlay: false, + no_overlay: false, + ours: false, + theirs: false, + ignore_unmerged: false, + merge: false, + conflict: None, + pathspec: vec![".".to_string()], + source: Some("HEAD".to_string()), + worktree: false, + staged: true, + pathspec_from_file: None, + pathspec_file_nul: false, + no_progress: false, + }) + .await + { + drop(guard); + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + created_pointer, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoWrite(format!( + "failed to seed ScorpioFS worktree index: {error}" + ))); + } + } else { + let guard = match DirGuard::change_to(&target) { + Ok(guard) => guard, + Err(error) => { + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + created_pointer, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoRead(format!( + "failed to enter ScorpioFS worktree '{}': {error}", + target.display() + ))); + } + }; + let unborn_branch = format!("scorpiofs/{worktree_id}"); + if let Err(error) = Head::update_result(Head::Branch(unborn_branch), None).await { + drop(guard); + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + created_pointer, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoWrite(format!( + "failed to seed unborn ScorpioFS worktree HEAD: {error}" + ))); + } + } + + let registration_epoch = state.next_epoch(); + // A ScorpioFS mount is a linked worktree like any other, and the fact + // that one existed must OUTLIVE its entry (§C.4.3) — detach deletes the + // entry, and the ambiguous-sidecar rules ask "did one ever exist". + state.linked_history = LinkedHistory::Existed; + state.entries.push(WorktreeEntry { + path: target.to_string_lossy().into_owned(), + is_main: false, + locked: false, + lock_reason: None, + // v2 (§C.7): the mount's stable id, so `worktree repair ` can + // restore a corrupt gitdir identity from the registry. + worktree_id: Some(worktree_id.clone()), + state: WorktreeEntryState::Active, + // A fresh generation for every registration, so a client fenced on + // the previous one at this path/id is refused rather than served. + epoch: registration_epoch, + }); + if let Err(error) = write_state(&state) { + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + created_pointer, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(error); + } + + let mut desired_state = crate::internal::scorpiofs_backend::LibraScorpioFsState::load(&storage) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + desired_state + .mark_ready(&job_id, &mount, &worktree_id) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + if let Err(error) = desired_state.save(&storage) { + rollback_scorpiofs_attach( + &client, + &job_id, + &pointer, + created_pointer, + &gitdir, + created_gitdir, + &worktree_id, + ) + .await; + return Err(WorktreeError::IoWrite(error.to_string())); + } + + Ok(ScorpioFsAttachOutput { + path: target.to_string_lossy().into_owned(), + worktree_id, + mount_id: mount.mount_id, + job_id, + already_exists: false, + }) +} + +async fn detach_scorpiofs_worktree(path: String) -> WorktreeResult { + let storage = util::storage_path(); + let _state_lock = crate::internal::scorpiofs_backend::ScorpioFsStateLock::acquire(&storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + let target = resolve_path(&path, "ScorpioFS worktree path")?; + let mut state = load_state()?; + let index = state + .entries + .iter() + .position(|entry| Path::new(&entry.path) == target) + .ok_or_else(|| WorktreeError::NoSuchWorktree { path: path.clone() })?; + let entry = state.entries[index].clone(); + if entry.is_main { + return Err(WorktreeError::MainWorktree { + action: "detach", + path: target.to_string_lossy().into_owned(), + }); + } + if entry.locked { + return Err(WorktreeError::LockedWorktree { + action: "detach", + path: target.to_string_lossy().into_owned(), + }); + } + + let gitdir = util::try_get_worktree_gitdir(Some(target.clone())).map_err(|error| { + WorktreeError::IoRead(format!( + "failed to resolve ScorpioFS worktree metadata for '{}': {error}", + target.display() + )) + })?; + let managed_root = storage.join("worktrees").join("scorpiofs"); + if !util::is_sub_path(&gitdir, &managed_root) { + return Err(WorktreeError::InvalidTarget(format!( + "worktree '{}' is not managed by the ScorpioFS backend", + target.display() + ))); + } + let record = ScorpioFsBackendRecord::load(&gitdir) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + let worktree_id = read_worktree_id(&gitdir)?; + + { + let guard = DirGuard::change_to(&target).map_err(|error| { + WorktreeError::IoRead(format!( + "cannot enter ScorpioFS worktree '{}': {error}", + target.display() + )) + })?; + let staged = crate::command::status::changes_to_be_committed_safe() + .await + .map_err(|error| { + WorktreeError::IoRead(format!("failed to inspect staged changes: {error}")) + })?; + let backend_changes = crate::internal::scorpiofs_backend::current_worktree_changes() + .await + .map_err(|error| { + WorktreeError::IoRead(format!( + "failed to query ScorpioFS worktree changes before detach: {error}" + )) + })?; + let unstaged = if let Some(changes) = backend_changes { + let ignore_case = crate::utils::path_case::effective_ignore_case_for_dir_sync(&target) + .map_err(|error| { + WorktreeError::IoRead(format!( + "failed to resolve ScorpioFS worktree path case policy: {error}" + )) + })?; + let (visible, _) = + crate::command::status::changes_to_be_staged_split_for_paths_with_ignore_case( + ignore_case, + &changes.candidate_paths(), + ) + .map_err(|error| { + WorktreeError::IoRead(format!("failed to inspect ScorpioFS changes: {error}")) + })?; + visible + } else { + crate::command::status::changes_to_be_staged().map_err(|error| { + WorktreeError::IoRead(format!("failed to inspect unstaged changes: {error}")) + })? + }; + if !staged.is_empty() || !unstaged.is_empty() { + return Err(WorktreeError::DirtyWorktree { + path: target.to_string_lossy().into_owned(), + }); + } + drop(guard); + } + + let mut desired_state = crate::internal::scorpiofs_backend::LibraScorpioFsState::load(&storage) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + if let Some(desired) = desired_state.mounts.get_mut(&record.job_id) { + desired.lifecycle = crate::internal::scorpiofs_backend::BackendLifecycle::Unmounting; + desired.last_error = None; + } + desired_state + .save(&storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + + state.entries.remove(index); + write_state(&state)?; + + let client = HttpScorpioFsClient::new(&record.endpoint) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + let driver = ScorpioFsDriver::new(client); + let backend_session = StorageMountSession { + backend: WorktreeBackendKind::ScorpioFs, + session_id: record.mount_id.clone(), + mountpoint: target.clone(), + cleanup_key: record.job_id.clone(), + base_oid: record.base_oid.clone(), + }; + if let Err(error) = driver.unmount(&backend_session).await { + if let Some(desired) = desired_state.mounts.get_mut(&record.job_id) { + desired.lifecycle = crate::internal::scorpiofs_backend::BackendLifecycle::Ready; + desired.last_error = Some(error.to_string()); + } + let _ = desired_state.save(&storage); + state.entries.insert(index, entry); + if let Err(restore_error) = write_state(&state) { + return Err(WorktreeError::StateWrite { + path: state_path(), + source: io::Error::other(format!( + "ScorpioFS unmount failed ({error}); registry rollback also failed: \ + {restore_error:?}" + )), + }); + } + return Err(WorktreeError::IoWrite(error.to_string())); + } + + let pointer = target.join(util::ROOT_DIR); + match fs::remove_file(&pointer) { + Ok(()) => {} + Err(source) if source.kind() == io::ErrorKind::NotFound => {} + Err(source) => { + return Err(WorktreeError::IoWrite(format!( + "ScorpioFS mount was removed but worktree pointer '{}' could not be deleted: \ + {source}; run `libra worktree repair`", + pointer.display() + ))); + } + } + + // Detach removes the mountpoint AND the gitdir below, so the directory is + // gone for the strict sweep's purposes — layer ownership and the sparse + // view go with it. Instance ids are path-derived, so a later reattach at + // this same mountpoint would otherwise inherit these rows. + let db = crate::internal::db::get_db_conn_instance().await; + gc_worktree_scoped_rows_strict(&db, &worktree_id, true) + .await + .map_err(|error| { + WorktreeError::IoWrite(format!( + "ScorpioFS mount was removed but the scoped rows for worktree '{worktree_id}' \ + could not be cleared: {error}; run `libra worktree repair`" + )) + })?; + fs::remove_dir_all(&gitdir).map_err(|source| { + WorktreeError::IoWrite(format!( + "ScorpioFS mount was removed but persistent worktree metadata '{}' could not be \ + deleted: {source}; run `libra worktree repair`", + gitdir.display() + )) + })?; + + desired_state.mounts.remove(&record.job_id); + if desired_state.mounts.is_empty() { + stop_managed_scorpiofs_worker(&mut desired_state); + } + desired_state + .save(&storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + + Ok(ScorpioFsDetachOutput { + path: target.to_string_lossy().into_owned(), + worktree_id, + job_id: record.job_id, + unmounted: true, + registry_removed: true, + }) +} + +async fn rollback_scorpiofs_attach( + client: &HttpScorpioFsClient, + job_id: &str, + pointer: &Path, + created_pointer: bool, + gitdir: &Path, + created_gitdir: bool, + worktree_id: &str, +) { + if let Some(storage) = gitdir.ancestors().nth(3) { + record_scorpiofs_attach_error(storage, job_id, "attach transaction rolled back"); + } + if created_pointer { + let _ = fs::remove_file(pointer); + } + if created_gitdir { + let _ = fs::remove_dir_all(gitdir); + } + // Best-effort on the rollback path: the attach already failed, and the + // caller reports THAT error. A sweep failure here only leaves rows a + // later attach's strict pre-seed sweep has to clear, so warn and go on. + let db = crate::internal::db::get_db_conn_instance().await; + if let Err(error) = gc_worktree_scoped_rows_strict(&db, worktree_id, true).await { + tracing::warn!( + worktree_id, + error, + "failed to GC per-worktree rows while rolling back a ScorpioFS attach" + ); + } + if let Err(error) = client.delete_by_job(job_id).await { + tracing::warn!(job_id, error = %error, "failed to roll back ScorpioFS mount"); + } +} + +fn record_scorpiofs_attach_error(storage: &Path, job_id: &str, error: impl Into) { + match crate::internal::scorpiofs_backend::LibraScorpioFsState::load(storage) { + Ok(mut state) => { + state.mark_error(job_id, error); + if let Err(save_error) = state.save(storage) { + tracing::warn!(job_id, error = %save_error, "failed to persist ScorpioFS error state"); + } + } + Err(load_error) => { + tracing::warn!(job_id, error = %load_error, "failed to load Libra ScorpioFS state"); + } + } +} + +async fn ensure_managed_scorpiofs_worker( + storage: &Path, + config_path: &Path, + timeout: Duration, +) -> WorktreeResult { + #[cfg(not(all(target_os = "linux", feature = "scorpiofs-direct")))] + { + let _ = (storage, config_path, timeout); + return Err(WorktreeError::OperationBlocked( + "direct ScorpioFS requires Linux and the scorpiofs-direct feature; use --endpoint for compatibility mode" + .to_string(), + )); + } + + #[cfg(all(target_os = "linux", feature = "scorpiofs-direct"))] + { + use crate::internal::scorpiofs_backend::{ + LibraScorpioFsState, ManagedWorkerRecord, ScorpioFsControl, + }; + + let mut state = LibraScorpioFsState::load(storage) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + if let Some(worker) = state.worker.as_ref() { + if let Ok(client) = HttpScorpioFsClient::new(&worker.endpoint) + && client.service_info().await.is_ok() + { + return Ok(worker.endpoint.clone()); + } + for desired in state.mounts.values_mut() { + if desired.transport + == crate::internal::scorpiofs_backend::BackendTransport::ManagedCrate + { + desired.lifecycle = + crate::internal::scorpiofs_backend::BackendLifecycle::RecoverableError; + desired.last_error = Some( + "Libra-owned ScorpioFS worker stopped; reattach this job to recover" + .to_string(), + ); + } + } + state.worker = None; + state + .save(storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + } + + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).map_err(|source| { + WorktreeError::IoWrite(format!("failed to reserve ScorpioFS worker port: {source}")) + })?; + let port = listener + .local_addr() + .map_err(|source| { + WorktreeError::IoRead(format!( + "failed to inspect reserved ScorpioFS worker port: {source}" + )) + })? + .port(); + drop(listener); + + let runtime_key = { + use std::hash::{Hash, Hasher}; + + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + storage.to_string_lossy().hash(&mut hasher); + format!("{:016x}", hasher.finish()) + }; + let runtime_root = env::temp_dir().join("libra-scorpiofs").join(runtime_key); + fs::create_dir_all(&runtime_root).map_err(|source| { + WorktreeError::IoWrite(format!( + "failed to create ScorpioFS runtime directory '{}': {source}", + runtime_root.display() + )) + })?; + let config_path = if config_path.is_absolute() { + config_path.to_path_buf() + } else { + env::current_dir() + .map_err(|source| { + WorktreeError::IoRead(format!( + "failed to resolve ScorpioFS config path: {source}" + )) + })? + .join(config_path) + }; + let endpoint = format!("http://127.0.0.1:{port}"); + let log_root = storage.join("scorpiofs"); + fs::create_dir_all(&log_root).map_err(|source| { + WorktreeError::IoWrite(format!( + "failed to create ScorpioFS log directory '{}': {source}", + log_root.display() + )) + })?; + let log_path = log_root.join("worker.log"); + let stdout = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .map_err(|source| { + WorktreeError::IoWrite(format!( + "failed to open ScorpioFS worker log '{}': {source}", + log_path.display() + )) + })?; + let stderr = stdout.try_clone().map_err(|source| { + WorktreeError::IoWrite(format!("failed to clone ScorpioFS worker log: {source}")) + })?; + let executable = env::current_exe().map_err(|source| { + WorktreeError::IoRead(format!("failed to locate the Libra executable: {source}")) + })?; + let child = std::process::Command::new(executable) + .arg("scorpiofs-worker") + .arg("--config-path") + .arg(&config_path) + .arg("--bind") + .arg(format!("127.0.0.1:{port}")) + .arg("--upper-root") + .arg(runtime_root.join("upper")) + .arg("--cl-root") + .arg(runtime_root.join("cl")) + .arg("--mount-root") + .arg(runtime_root.join("mounts")) + .arg("--runtime-state-file") + .arg(log_root.join("ignored-runtime-state.toml")) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::from(stdout)) + .stderr(std::process::Stdio::from(stderr)) + .spawn() + .map_err(|source| { + WorktreeError::IoWrite(format!( + "failed to start Libra-owned ScorpioFS worker: {source}" + )) + })?; + + state.worker = Some(ManagedWorkerRecord { + pid: child.id(), + endpoint: endpoint.clone(), + config_path: config_path.to_string_lossy().into_owned(), + }); + state + .save(storage) + .map_err(|error| WorktreeError::IoWrite(error.to_string()))?; + + let client = HttpScorpioFsClient::new(&endpoint) + .map_err(|error| WorktreeError::IoRead(error.to_string()))?; + let deadline = tokio::time::Instant::now() + timeout; + loop { + if let Ok(service) = client.service_info().await { + for capability in [ + CAPABILITY_MOUNT_V1, + CAPABILITY_READY_V1, + CAPABILITY_CHANGES_V1, + ] { + service + .require(capability) + .map_err(|error| WorktreeError::OperationBlocked(error.to_string()))?; + } + return Ok(endpoint); + } + if tokio::time::Instant::now() >= deadline { + stop_managed_scorpiofs_worker(&mut state); + let _ = state.save(storage); + return Err(WorktreeError::IoRead(format!( + "Libra-owned ScorpioFS worker did not become ready; inspect '{}'", + log_path.display() + ))); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } +} + +fn stop_managed_scorpiofs_worker( + state: &mut crate::internal::scorpiofs_backend::LibraScorpioFsState, +) { + let Some(worker) = state.worker.take() else { + return; + }; + #[cfg(unix)] + unsafe { + libc::kill(worker.pid as i32, libc::SIGINT); + } +} + +fn attach_worktree_pointer(pointer: &Path, gitdir: &Path) -> io::Result { + let expected = format!("gitdir: {}\n", gitdir.display()); + match fs::read_to_string(pointer) { + Ok(existing) if existing == expected => Ok(false), + Ok(_) => Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!( + "worktree already contains a different .libra pointer at '{}'", + pointer.display() + ), + )), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + fs::write(pointer, expected)?; + Ok(true) + } + Err(error) => Err(error), + } +} + +fn read_worktree_id(gitdir: &Path) -> WorktreeResult { + let path = gitdir.join("worktree_id"); + let worktree_id = fs::read_to_string(&path).map_err(|source| { + WorktreeError::IoRead(format!( + "failed to read ScorpioFS worktree id '{}': {source}", + path.display() + )) + })?; + let worktree_id = worktree_id.trim(); + if worktree_id.is_empty() { + return Err(WorktreeError::IoRead(format!( + "ScorpioFS worktree id '{}' is empty", + path.display() + ))); + } + Ok(worktree_id.to_string()) +} + +fn scorpiofs_worktree_id(endpoint: &str, job_id: &str, remote_path: &str) -> String { + let key = format!("{endpoint}\0{job_id}\0{remote_path}"); + let mut hash: u64 = 0xcbf29ce484222325; + for byte in key.as_bytes() { + hash ^= *byte as u64; + hash = hash.wrapping_mul(0x100000001b3); + } + let label: String = job_id + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() { + character + } else { + '-' + } + }) + .collect(); + format!("{label}-{hash:016x}") +} + +fn render_scorpiofs_attach(result: &ScorpioFsAttachOutput, output: &OutputConfig) -> CliResult<()> { + if output.is_json() { + return emit_json_data("worktree.scorpiofs.attach", result, output); + } + if !output.quiet { + println!("{}", result.path); + } + Ok(()) +} + +fn render_scorpiofs_detach(result: &ScorpioFsDetachOutput, output: &OutputConfig) -> CliResult<()> { + if output.is_json() { + return emit_json_data("worktree.scorpiofs.detach", result, output); + } + if !output.quiet { + println!("Detached ScorpioFS worktree '{}'.", result.path); + } + Ok(()) +} + fn render_add_worktree(result: &WorktreeAddOutput, output: &OutputConfig) -> CliResult<()> { if output.is_json() { return emit_json_data("worktree.add", result, output); @@ -2958,7 +3960,9 @@ fn render_add_worktree(result: &WorktreeAddOutput, output: &OutputConfig) -> Cli /// its `.libra/worktree_id` file if present, else recompute deterministically /// from the canonical path (lore.md 2.1). fn resolve_worktree_id(target: &Path) -> Option { - if let Ok(id) = fs::read_to_string(target.join(util::ROOT_DIR).join("worktree_id")) { + if let Ok(gitdir) = util::try_get_worktree_gitdir(Some(target.to_path_buf())) + && let Ok(id) = fs::read_to_string(gitdir.join("worktree_id")) + { let id = id.trim(); if !id.is_empty() { return Some(id.to_string()); @@ -3397,7 +4401,7 @@ pub(crate) fn resolve_entry_worktree_id(path: &str, is_main: bool) -> Option String { + "scorpiofs".to_string() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum BackendTransport { + ManagedCrate, + #[default] + ExternalHttp, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScorpioFsBackendRecord { + pub schema_version: u32, + pub backend: String, + #[serde(default)] + pub transport: BackendTransport, + pub endpoint: String, + pub mount_id: String, + pub job_id: String, + pub remote_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_oid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cl: Option, +} + +impl ScorpioFsBackendRecord { + pub fn new( + endpoint: &Url, + mount: &MountResponse, + request: &MountRequest, + ) -> Result { + Self::new_with_transport(endpoint, mount, request, BackendTransport::ExternalHttp) + } + + pub fn new_with_transport( + endpoint: &Url, + mount: &MountResponse, + request: &MountRequest, + transport: BackendTransport, + ) -> Result { + validate_identifier("mount_id", &mount.mount_id)?; + validate_identifier("job_id", &request.job_id)?; + validate_remote_path(&request.path)?; + + Ok(Self { + schema_version: BACKEND_SCHEMA_VERSION, + backend: "scorpiofs".to_string(), + transport, + endpoint: endpoint.as_str().trim_end_matches('/').to_string(), + mount_id: mount.mount_id.clone(), + job_id: request.job_id.clone(), + remote_path: request.path.clone(), + base_oid: mount.base_oid.clone().or_else(|| request.base_oid.clone()), + cl: request.cl.clone(), + }) + } + + pub fn validate(&self) -> Result<(), BackendError> { + if self.schema_version != BACKEND_SCHEMA_VERSION { + return Err(BackendError::UnsupportedRecordVersion { + found: self.schema_version, + supported: BACKEND_SCHEMA_VERSION, + }); + } + if self.backend != "scorpiofs" { + return Err(BackendError::InvalidRecord(format!( + "expected backend 'scorpiofs', found '{}'", + self.backend + ))); + } + + validate_endpoint(&self.endpoint)?; + validate_identifier("mount_id", &self.mount_id)?; + validate_identifier("job_id", &self.job_id)?; + validate_remote_path(&self.remote_path) + } + + pub fn load(gitdir: &Path) -> Result { + let path = gitdir.join(BACKEND_RECORD_FILE); + let data = fs::read(&path).map_err(|source| BackendError::RecordIo { + path: path.clone(), + source, + })?; + let record = + serde_json::from_slice::(&data).map_err(|source| BackendError::RecordDecode { + path: path.clone(), + source, + })?; + record.validate()?; + Ok(record) + } + + pub fn save(&self, gitdir: &Path) -> Result<(), BackendError> { + self.validate()?; + fs::create_dir_all(gitdir).map_err(|source| BackendError::RecordIo { + path: gitdir.to_path_buf(), + source, + })?; + + let path = gitdir.join(BACKEND_RECORD_FILE); + let temporary = gitdir.join(format!(".{BACKEND_RECORD_FILE}.tmp-{}", std::process::id())); + let data = serde_json::to_vec_pretty(self).map_err(BackendError::RecordEncode)?; + fs::write(&temporary, data).map_err(|source| BackendError::RecordIo { + path: temporary.clone(), + source, + })?; + if let Err(source) = replace_file(&temporary, &path) { + let _ = fs::remove_file(&temporary); + return Err(BackendError::RecordIo { path, source }); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedWorkerRecord { + pub pid: u32, + pub endpoint: String, + pub config_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DesiredMountRecord { + pub request: MountRequest, + pub transport: BackendTransport, + pub lifecycle: BackendLifecycle, + pub endpoint: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mount_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mountpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worktree_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LibraScorpioFsState { + pub schema_version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker: Option, + #[serde(default)] + pub mounts: BTreeMap, +} + +pub struct ScorpioFsStateLock { + file: fs::File, +} + +impl ScorpioFsStateLock { + pub fn acquire(storage: &Path) -> Result { + let path = storage.join(DESIRED_STATE_LOCK_FILE); + let parent = path.parent().ok_or_else(|| { + BackendError::InvalidRecord("ScorpioFS lock path has no parent".to_string()) + })?; + fs::create_dir_all(parent).map_err(|source| BackendError::StateIo { + path: parent.to_path_buf(), + source, + })?; + let file = fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&path) + .map_err(|source| BackendError::StateIo { + path: path.clone(), + source, + })?; + fs2::FileExt::lock_exclusive(&file) + .map_err(|source| BackendError::StateIo { path, source })?; + Ok(Self { file }) + } +} + +impl Drop for ScorpioFsStateLock { + fn drop(&mut self) { + let _ = fs2::FileExt::unlock(&self.file); + } +} + +impl Default for LibraScorpioFsState { + fn default() -> Self { + Self { + schema_version: 1, + worker: None, + mounts: BTreeMap::new(), + } + } +} + +impl LibraScorpioFsState { + pub fn load(storage: &Path) -> Result { + let path = storage.join(DESIRED_STATE_FILE); + let data = match fs::read(&path) { + Ok(data) => data, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Self::default()), + Err(source) => return Err(BackendError::StateIo { path, source }), + }; + let state = + serde_json::from_slice::(&data).map_err(|source| BackendError::StateDecode { + path: path.clone(), + source, + })?; + if state.schema_version != 1 { + return Err(BackendError::InvalidRecord(format!( + "unsupported Libra ScorpioFS state version {}", + state.schema_version + ))); + } + Ok(state) + } + + pub fn save(&self, storage: &Path) -> Result<(), BackendError> { + let path = storage.join(DESIRED_STATE_FILE); + let parent = path.parent().ok_or_else(|| { + BackendError::InvalidRecord("ScorpioFS state path has no parent".to_string()) + })?; + fs::create_dir_all(parent).map_err(|source| BackendError::StateIo { + path: parent.to_path_buf(), + source, + })?; + let temporary = parent.join(format!(".state.json.tmp-{}", std::process::id())); + let data = serde_json::to_vec_pretty(self).map_err(BackendError::RecordEncode)?; + fs::write(&temporary, data).map_err(|source| BackendError::StateIo { + path: temporary.clone(), + source, + })?; + if let Err(source) = replace_file(&temporary, &path) { + let _ = fs::remove_file(&temporary); + return Err(BackendError::StateIo { path, source }); + } + Ok(()) + } + + pub fn begin_mount( + &mut self, + request: MountRequest, + transport: BackendTransport, + endpoint: String, + ) -> Result<(), BackendError> { + request.validate()?; + if let Some(existing) = self.mounts.get(&request.job_id) + && (existing.request.path != request.path || existing.request.cl != request.cl) + { + return Err(BackendError::InvalidRecord(format!( + "ScorpioFS job '{}' is already assigned to '{}' with CL {:?}", + request.job_id, existing.request.path, existing.request.cl + ))); + } + self.mounts.insert( + request.job_id.clone(), + DesiredMountRecord { + request, + transport, + lifecycle: BackendLifecycle::Mounting, + endpoint, + mount_id: None, + mountpoint: None, + worktree_id: None, + last_error: None, + }, + ); + Ok(()) + } + + pub fn mark_ready( + &mut self, + job_id: &str, + mount: &MountResponse, + worktree_id: &str, + ) -> Result<(), BackendError> { + let desired = self.mounts.get_mut(job_id).ok_or_else(|| { + BackendError::InvalidRecord(format!("missing desired mount for job '{job_id}'")) + })?; + desired.lifecycle = BackendLifecycle::Ready; + desired.mount_id = Some(mount.mount_id.clone()); + desired.mountpoint = Some(mount.mountpoint.clone()); + desired.worktree_id = Some(worktree_id.to_string()); + desired.last_error = None; + Ok(()) + } + + pub fn mark_error(&mut self, job_id: &str, error: impl Into) { + if let Some(desired) = self.mounts.get_mut(job_id) { + desired.lifecycle = BackendLifecycle::RecoverableError; + desired.last_error = Some(error.into()); + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ServiceInfo { + pub protocol_version: u32, + pub service: String, + #[serde(default)] + pub service_version: Option, + #[serde(default)] + pub capabilities: Vec, +} + +impl ServiceInfo { + pub fn validate(&self) -> Result<(), BackendError> { + if self.protocol_version != PROTOCOL_VERSION { + return Err(BackendError::UnsupportedProtocolVersion { + found: self.protocol_version, + supported: PROTOCOL_VERSION, + }); + } + if self.service != "scorpiofs" { + return Err(BackendError::UnexpectedService(self.service.clone())); + } + Ok(()) + } + + pub fn supports(&self, capability: &str) -> bool { + self.capabilities.iter().any(|item| item == capability) + } + + pub fn require(&self, capability: &'static str) -> Result<(), BackendError> { + if self.supports(capability) { + Ok(()) + } else { + Err(BackendError::MissingCapability(capability)) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MountRequest { + pub job_id: String, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cl: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_oid: Option, +} + +impl MountRequest { + pub fn validate(&self) -> Result<(), BackendError> { + validate_identifier("job_id", &self.job_id)?; + validate_remote_path(&self.path)?; + if let Some(base_oid) = self.base_oid.as_deref() { + validate_object_id(base_oid)?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MountResponse { + #[serde(alias = "id")] + pub mount_id: String, + pub mountpoint: String, + #[serde(default)] + pub base_oid: Option, + #[serde(default)] + pub ready: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReadyResponse { + #[serde(default)] + pub mount_id: Option, + pub ready: bool, + #[serde(default)] + pub status: Option, + #[serde(default)] + pub detail: Option, +} + +#[derive(Debug, Error)] +pub enum BackendError { + #[error("invalid ScorpioFS endpoint: {0}")] + InvalidEndpoint(String), + #[error("invalid ScorpioFS backend record: {0}")] + InvalidRecord(String), + #[error("failed to discover ScorpioFS worktree metadata: {0}")] + MetadataDiscovery(#[source] io::Error), + #[error( + "unsupported ScorpioFS backend record version {found}; this Libra supports version {supported}" + )] + UnsupportedRecordVersion { found: u32, supported: u32 }, + #[error( + "unsupported ScorpioFS protocol version {found}; this Libra supports version {supported}" + )] + UnsupportedProtocolVersion { found: u32, supported: u32 }, + #[error("unexpected ScorpioFS control service '{0}'")] + UnexpectedService(String), + #[error("failed to read or write ScorpioFS backend record '{}': {source}", path.display())] + RecordIo { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("failed to encode ScorpioFS backend record: {0}")] + RecordEncode(serde_json::Error), + #[error("failed to decode ScorpioFS backend record '{}': {source}", path.display())] + RecordDecode { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("failed to read or write Libra ScorpioFS state '{}': {source}", path.display())] + StateIo { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("failed to decode Libra ScorpioFS state '{}': {source}", path.display())] + StateDecode { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("invalid {field}: {message}")] + InvalidIdentifier { + field: &'static str, + message: String, + }, + #[error("invalid ScorpioFS remote path: {0}")] + InvalidRemotePath(String), + #[error("invalid ScorpioFS changed path: {0}")] + InvalidChangedPath(String), + #[error("invalid base object id: {0}")] + InvalidObjectId(String), + #[error("ScorpioFS does not advertise required capability '{0}'")] + MissingCapability(&'static str), + #[error("ScorpioFS request '{operation}' failed: {source}")] + Request { + operation: &'static str, + #[source] + source: anyhow::Error, + }, + #[error("ScorpioFS request '{operation}' returned HTTP {status}: {message}")] + HttpStatus { + operation: &'static str, + status: StatusCode, + message: String, + }, + #[error("ScorpioFS mount '{mount_id}' did not become ready within {timeout:?}")] + ReadinessTimeout { mount_id: String, timeout: Duration }, +} + +fn replace_file(temporary: &Path, destination: &Path) -> io::Result<()> { + #[cfg(windows)] + if destination.exists() { + fs::remove_file(destination)?; + } + fs::rename(temporary, destination) +} + +#[async_trait] +pub trait ScorpioFsControl: Send + Sync { + async fn service_info(&self) -> Result; + async fn mount(&self, request: &MountRequest) -> Result; + async fn ready(&self, mount_id: &str) -> Result; + async fn changes(&self, mount_id: &str) -> Result; + async fn delete_by_job(&self, job_id: &str) -> Result<(), BackendError>; +} + +#[derive(Debug, Clone)] +pub struct HttpScorpioFsClient { + endpoint: Url, + client: Client, +} + +impl HttpScorpioFsClient { + pub fn new(endpoint: &str) -> Result { + let endpoint = validate_endpoint(endpoint)?; + let client = Client::builder() + .timeout(DEFAULT_REQUEST_TIMEOUT) + .build() + .map_err(|source| BackendError::Request { + operation: "create client", + source: source.into(), + })?; + Ok(Self { endpoint, client }) + } + + pub fn endpoint(&self) -> &Url { + &self.endpoint + } + + pub async fn wait_until_ready( + &self, + mount_id: &str, + timeout: Duration, + ) -> Result { + validate_identifier("mount_id", mount_id)?; + let deadline = Instant::now() + timeout; + + loop { + let response = self.ready(mount_id).await?; + if response.ready { + return Ok(response); + } + if Instant::now() >= deadline { + return Err(BackendError::ReadinessTimeout { + mount_id: mount_id.to_string(), + timeout, + }); + } + sleep(DEFAULT_READY_POLL_INTERVAL).await; + } + } + + fn url(&self, segments: &[&str]) -> Result { + let mut url = self.endpoint.clone(); + { + let mut path = url.path_segments_mut().map_err(|_| { + BackendError::InvalidEndpoint( + "the endpoint cannot be used as a hierarchical URL".to_string(), + ) + })?; + path.pop_if_empty(); + for segment in segments { + path.push(segment); + } + } + Ok(url) + } + + async fn status_error(operation: &'static str, response: reqwest::Response) -> BackendError { + let status = response.status(); + let message = response + .text() + .await + .unwrap_or_else(|_| "response body was unreadable".to_string()); + BackendError::HttpStatus { + operation, + status, + message: truncate_message(&message), + } + } +} + +#[async_trait] +impl ScorpioFsControl for HttpScorpioFsClient { + async fn service_info(&self) -> Result { + let operation = "service info"; + let response = self + .client + .get(self.url(&["health"])?) + .send() + .await + .with_context(|| "failed to reach the ScorpioFS health endpoint") + .map_err(|source| BackendError::Request { operation, source })?; + if !response.status().is_success() { + return Err(Self::status_error(operation, response).await); + } + + #[derive(Deserialize)] + struct Health { + #[serde(default = "default_service_name")] + service: String, + #[serde(default, alias = "version")] + service_version: Option, + #[serde(default)] + protocol_version: Option, + #[serde(default)] + capabilities: Vec, + } + + let health: Health = response + .json() + .await + .with_context(|| "ScorpioFS health returned invalid JSON") + .map_err(|source| BackendError::Request { operation, source })?; + let info = ServiceInfo { + protocol_version: health.protocol_version.unwrap_or(PROTOCOL_VERSION), + service: health.service, + service_version: health.service_version, + capabilities: health.capabilities, + }; + info.validate()?; + Ok(info) + } + + async fn mount(&self, request: &MountRequest) -> Result { + request.validate()?; + let operation = "mount"; + let response = self + .client + .post(self.url(&["mounts"])?) + .json(request) + .send() + .await + .with_context(|| "failed to send the ScorpioFS mount request") + .map_err(|source| BackendError::Request { operation, source })?; + if !response.status().is_success() { + return Err(Self::status_error(operation, response).await); + } + let mount: MountResponse = response + .json() + .await + .with_context(|| "ScorpioFS mount returned invalid JSON") + .map_err(|source| BackendError::Request { operation, source })?; + validate_identifier("mount_id", &mount.mount_id)?; + if mount.mountpoint.trim().is_empty() { + return Err(BackendError::InvalidRecord( + "ScorpioFS returned an empty mountpoint".to_string(), + )); + } + Ok(mount) + } + + async fn ready(&self, mount_id: &str) -> Result { + validate_identifier("mount_id", mount_id)?; + let operation = "mount readiness"; + let response = self + .client + .get(self.url(&["mounts", mount_id, "ready"])?) + .send() + .await + .with_context(|| format!("failed to query ScorpioFS mount '{mount_id}' readiness")) + .map_err(|source| BackendError::Request { operation, source })?; + if !response.status().is_success() { + return Err(Self::status_error(operation, response).await); + } + response + .json() + .await + .with_context(|| "ScorpioFS readiness returned invalid JSON") + .map_err(|source| BackendError::Request { operation, source }) + } + + async fn changes(&self, mount_id: &str) -> Result { + validate_identifier("mount_id", mount_id)?; + let operation = "changed paths"; + let response = self + .client + .get(self.url(&["mounts", mount_id, "changes"])?) + .send() + .await + .with_context(|| format!("failed to query ScorpioFS mount '{mount_id}' changes")) + .map_err(|source| BackendError::Request { operation, source })?; + if !response.status().is_success() { + return Err(Self::status_error(operation, response).await); + } + let changes: ChangeSet = response + .json() + .await + .with_context(|| "ScorpioFS changed paths returned invalid JSON") + .map_err(|source| BackendError::Request { operation, source })?; + validate_identifier("mount_id", &changes.mount_id)?; + changes + .validate() + .map_err(|error| BackendError::InvalidChangedPath(error.to_string()))?; + Ok(changes) + } + + async fn delete_by_job(&self, job_id: &str) -> Result<(), BackendError> { + validate_identifier("job_id", job_id)?; + let operation = "delete mount"; + let response = self + .client + .delete(self.url(&["mounts", "by-job", job_id])?) + .send() + .await + .with_context(|| format!("failed to delete ScorpioFS job '{job_id}'")) + .map_err(|source| BackendError::Request { operation, source })?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(()); + } + if !response.status().is_success() { + return Err(Self::status_error(operation, response).await); + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct ScorpioFsDriver { + client: HttpScorpioFsClient, +} + +impl ScorpioFsDriver { + pub fn new(client: HttpScorpioFsClient) -> Self { + Self { client } + } +} + +#[async_trait] +impl WorktreeBackendDriver for ScorpioFsDriver { + fn descriptor(&self) -> BackendDescriptor { + BackendDescriptor::scorpiofs(true) + } + + async fn mount( + &self, + request: &BackendMountRequest, + ) -> Result { + let (remote_path, base_oid, change_layer) = match &request.source { + BackendMountSource::RemoteProjection { + remote_path, + base_oid, + change_layer, + } => (remote_path, base_oid, change_layer), + BackendMountSource::LocalDirectory { .. } => { + return Err(WorktreeBackendError::UnsupportedSource { + backend: BackendKind::ScorpioFs, + detail: "local_directory", + }); + } + BackendMountSource::PersistentVolume { .. } => { + return Err(WorktreeBackendError::UnsupportedSource { + backend: BackendKind::ScorpioFs, + detail: "persistent_volume", + }); + } + }; + + let service = self.client.service_info().await.map_err(|error| { + WorktreeBackendError::operation(BackendKind::ScorpioFs, "service_info", error) + })?; + for capability in [ + CAPABILITY_MOUNT_V1, + CAPABILITY_READY_V1, + CAPABILITY_CHANGES_V1, + ] { + service.require(capability).map_err(|error| { + WorktreeBackendError::operation( + BackendKind::ScorpioFs, + "capability_negotiation", + error, + ) + })?; + } + + let scorpio_request = MountRequest { + job_id: request.instance_id.clone(), + path: remote_path.clone(), + cl: change_layer.clone(), + base_oid: base_oid.clone(), + }; + let mount = self.client.mount(&scorpio_request).await.map_err(|error| { + WorktreeBackendError::operation(BackendKind::ScorpioFs, "mount", error) + })?; + if mount.ready != Some(true) + && let Err(error) = self + .client + .wait_until_ready( + &mount.mount_id, + Duration::from_secs(request.ready_timeout_secs), + ) + .await + { + let _ = self.client.delete_by_job(&request.instance_id).await; + return Err(WorktreeBackendError::operation( + BackendKind::ScorpioFs, + "wait_until_ready", + error, + )); + } + + Ok(BackendMountSession { + backend: BackendKind::ScorpioFs, + session_id: mount.mount_id, + mountpoint: PathBuf::from(mount.mountpoint), + cleanup_key: request.instance_id.clone(), + base_oid: mount.base_oid.or_else(|| base_oid.clone()), + }) + } + + async fn health( + &self, + session: &BackendMountSession, + ) -> Result { + let response = self + .client + .ready(&session.session_id) + .await + .map_err(|error| { + WorktreeBackendError::operation(BackendKind::ScorpioFs, "health", error) + })?; + Ok(BackendHealth { + ready: response.ready, + detail: response.detail.or(response.status), + }) + } + + async fn changed_paths( + &self, + session: &BackendMountSession, + ) -> Result, WorktreeBackendError> { + self.client + .changes(&session.session_id) + .await + .map(Some) + .map_err(|error| { + WorktreeBackendError::operation(BackendKind::ScorpioFs, "changed_paths", error) + }) + } + + async fn unmount(&self, session: &BackendMountSession) -> Result<(), WorktreeBackendError> { + self.client + .delete_by_job(&session.cleanup_key) + .await + .map_err(|error| { + WorktreeBackendError::operation(BackendKind::ScorpioFs, "unmount", error) + }) + } +} + +/// Return the changed-path set for the current worktree when it is backed by +/// ScorpioFS. Ordinary Libra worktrees return `None` without making a request. +pub async fn current_worktree_changes() -> Result, BackendError> { + let gitdir = crate::utils::util::try_get_worktree_gitdir(None) + .map_err(BackendError::MetadataDiscovery)?; + let record_path = gitdir.join(BACKEND_RECORD_FILE); + if !record_path.exists() { + return Ok(None); + } + + let record = ScorpioFsBackendRecord::load(&gitdir)?; + let client = HttpScorpioFsClient::new(&record.endpoint)?; + let driver = ScorpioFsDriver::new(client); + let session = BackendMountSession { + backend: BackendKind::ScorpioFs, + session_id: record.mount_id, + mountpoint: PathBuf::new(), + cleanup_key: record.job_id, + base_oid: record.base_oid, + }; + driver + .changed_paths(&session) + .await + .map_err(|error| BackendError::InvalidRecord(error.to_string())) +} + +fn validate_endpoint(endpoint: &str) -> Result { + let mut url = + Url::parse(endpoint).map_err(|error| BackendError::InvalidEndpoint(error.to_string()))?; + if !matches!(url.scheme(), "http" | "https") { + return Err(BackendError::InvalidEndpoint( + "only http and https are supported by the initial transport".to_string(), + )); + } + if url.host_str().is_none() { + return Err(BackendError::InvalidEndpoint( + "the endpoint must include a host".to_string(), + )); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(BackendError::InvalidEndpoint( + "credentials must not be embedded in the endpoint URL".to_string(), + )); + } + if url.query().is_some() || url.fragment().is_some() { + return Err(BackendError::InvalidEndpoint( + "query strings and fragments are not allowed".to_string(), + )); + } + + let normalized = url.path().trim_end_matches('/').to_string(); + url.set_path(&normalized); + Ok(url) +} + +fn validate_identifier(field: &'static str, value: &str) -> Result<(), BackendError> { + if value.is_empty() { + return Err(BackendError::InvalidIdentifier { + field, + message: "value cannot be empty".to_string(), + }); + } + if value.len() > 255 { + return Err(BackendError::InvalidIdentifier { + field, + message: "value exceeds 255 bytes".to_string(), + }); + } + if !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + { + return Err(BackendError::InvalidIdentifier { + field, + message: "only ASCII letters, digits, '-', '_', and '.' are allowed".to_string(), + }); + } + Ok(()) +} + +fn validate_remote_path(path: &str) -> Result<(), BackendError> { + if !path.starts_with('/') { + return Err(BackendError::InvalidRemotePath( + "path must be absolute within the monorepo".to_string(), + )); + } + if path.contains('\0') || path.split('/').any(|part| part == "..") { + return Err(BackendError::InvalidRemotePath( + "path must not contain NUL or parent traversal".to_string(), + )); + } + Ok(()) +} + +fn validate_object_id(object_id: &str) -> Result<(), BackendError> { + if !matches!(object_id.len(), 40 | 64) + || !object_id.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(BackendError::InvalidObjectId( + "expected a 40- or 64-character hexadecimal object id".to_string(), + )); + } + Ok(()) +} + +fn truncate_message(message: &str) -> String { + const MAX_CHARS: usize = 2048; + let message = message.trim(); + if message.chars().count() <= MAX_CHARS { + message.to_string() + } else { + format!("{}...", message.chars().take(MAX_CHARS).collect::()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_rejects_credentials_and_query_data() { + assert!(matches!( + HttpScorpioFsClient::new("http://user:secret@localhost:2725/antares"), + Err(BackendError::InvalidEndpoint(_)) + )); + assert!(matches!( + HttpScorpioFsClient::new("http://localhost:2725/antares?token=secret"), + Err(BackendError::InvalidEndpoint(_)) + )); + } + + #[test] + fn endpoint_appends_antares_routes_without_replacing_prefix() { + let client = + HttpScorpioFsClient::new("http://127.0.0.1:2725/antares/").expect("valid endpoint"); + let url = client + .url(&["mounts", "mount-1", "ready"]) + .expect("valid route"); + assert_eq!( + url.as_str(), + "http://127.0.0.1:2725/antares/mounts/mount-1/ready" + ); + } + + #[test] + fn changed_paths_reject_absolute_and_parent_paths() { + for path in ["/absolute", "../outside", "src/../outside", "src//lib.rs"] { + let changed = ChangedPath { + kind: ChangeKind::Modified, + path: path.to_string(), + source_path: None, + }; + assert!(changed.validate().is_err(), "{path} must be rejected"); + } + } + + #[test] + fn backend_record_round_trips_without_credentials() { + let endpoint = Url::parse("http://127.0.0.1:2725/antares").expect("valid URL"); + let request = MountRequest { + job_id: "build-123".to_string(), + path: "/project/aardvark-dns".to_string(), + cl: Some("1XFJ4PGK".to_string()), + base_oid: None, + }; + let mount = MountResponse { + mount_id: "mount-123".to_string(), + mountpoint: "/var/lib/scorpiofs/antares/mnt/mount-123".to_string(), + base_oid: None, + ready: Some(false), + }; + let record = + ScorpioFsBackendRecord::new(&endpoint, &mount, &request).expect("valid record"); + let encoded = serde_json::to_string(&record).expect("serialize record"); + let decoded: ScorpioFsBackendRecord = + serde_json::from_str(&encoded).expect("deserialize record"); + + assert_eq!(decoded, record); + assert!(!encoded.contains("secret")); + decoded.validate().expect("record remains valid"); + } + + #[test] + fn renamed_change_requires_a_source_path() { + let change = ChangedPath { + kind: ChangeKind::Renamed, + path: "src/new.rs".to_string(), + source_path: None, + }; + assert!(matches!( + change.validate(), + Err(WorktreeBackendError::InvalidChangedPath(_)) + )); + } + + #[test] + fn change_set_candidates_include_rename_sources_and_are_deduplicated() { + let changes = ChangeSet { + mount_id: "mount-1".to_string(), + generation: 1, + changes: vec![ + ChangedPath { + kind: ChangeKind::Modified, + path: "src/new.rs".to_string(), + source_path: None, + }, + ChangedPath { + kind: ChangeKind::Renamed, + path: "src/new.rs".to_string(), + source_path: Some("src/old.rs".to_string()), + }, + ], + }; + + assert_eq!( + changes.candidate_paths(), + vec![PathBuf::from("src/new.rs"), PathBuf::from("src/old.rs")] + ); + } + + #[test] + fn libra_state_owns_mount_lifecycle_transitions() { + let temp = tempfile::tempdir().expect("temporary state root"); + let request = MountRequest { + job_id: "build-123".to_string(), + path: "/project/aardvark-dns".to_string(), + cl: None, + base_oid: None, + }; + let mount = MountResponse { + mount_id: "mount-123".to_string(), + mountpoint: "/mnt/aardvark-dns".to_string(), + base_oid: None, + ready: Some(true), + }; + + let mut state = LibraScorpioFsState::default(); + state + .begin_mount( + request, + BackendTransport::ManagedCrate, + "http://127.0.0.1:2725".to_string(), + ) + .expect("begin mount"); + assert_eq!( + state.mounts["build-123"].lifecycle, + BackendLifecycle::Mounting + ); + state + .mark_ready("build-123", &mount, "scorpiofs-worktree") + .expect("mark ready"); + state.save(temp.path()).expect("save state"); + + let loaded = LibraScorpioFsState::load(temp.path()).expect("load state"); + let desired = &loaded.mounts["build-123"]; + assert_eq!(desired.lifecycle, BackendLifecycle::Ready); + assert_eq!(desired.transport, BackendTransport::ManagedCrate); + assert_eq!(desired.mount_id.as_deref(), Some("mount-123")); + assert_eq!(desired.worktree_id.as_deref(), Some("scorpiofs-worktree")); + } + + #[test] + fn libra_state_lock_serializes_updates() { + let temp = tempfile::tempdir().unwrap(); + let first = ScorpioFsStateLock::acquire(temp.path()).unwrap(); + let storage = temp.path().to_path_buf(); + let acquired = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let acquired_in_thread = acquired.clone(); + + let waiter = std::thread::spawn(move || { + let _second = ScorpioFsStateLock::acquire(&storage).unwrap(); + acquired_in_thread.store(true, std::sync::atomic::Ordering::SeqCst); + }); + + std::thread::sleep(Duration::from_millis(50)); + assert!(!acquired.load(std::sync::atomic::Ordering::SeqCst)); + drop(first); + waiter.join().unwrap(); + assert!(acquired.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[test] + fn libra_state_rejects_reusing_a_job_for_another_path() { + let mut state = LibraScorpioFsState::default(); + state + .begin_mount( + MountRequest { + job_id: "build-123".to_string(), + path: "/project/a".to_string(), + cl: None, + base_oid: None, + }, + BackendTransport::ManagedCrate, + "http://127.0.0.1:2725".to_string(), + ) + .expect("first mount"); + + assert!(matches!( + state.begin_mount( + MountRequest { + job_id: "build-123".to_string(), + path: "/project/b".to_string(), + cl: None, + base_oid: None, + }, + BackendTransport::ManagedCrate, + "http://127.0.0.1:2725".to_string(), + ), + Err(BackendError::InvalidRecord(_)) + )); + } +} diff --git a/src/internal/tui/app.rs b/src/internal/tui/app.rs index 13024d310..bbe74fcf2 100644 --- a/src/internal/tui/app.rs +++ b/src/internal/tui/app.rs @@ -555,10 +555,10 @@ impl ProcessTerminateGate { /// Instant the first terminate signal was observed, if any. pub fn signaled_at(&self) -> Option { - self.signaled_at + *self + .signaled_at .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) - .clone() } pub async fn wait(&self) { diff --git a/src/internal/worktree_backend.rs b/src/internal/worktree_backend.rs new file mode 100644 index 000000000..da8e1a4c4 --- /dev/null +++ b/src/internal/worktree_backend.rs @@ -0,0 +1,379 @@ +//! Backend-neutral contracts for mounted Libra worktrees. +//! +//! Libra owns Git semantics and durable desired state. A backend driver only +//! prepares a POSIX-visible worktree, reports health and optional changed-path +//! candidates, flushes backend data when required, and tears the worktree down. +//! The contract deliberately does not mirror POSIX operations: filesystem +//! crates already provide those APIs, while build tools consume their mounts. + +use std::path::PathBuf; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub const BACKEND_CONTROL_PROTOCOL_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BackendKind { + Local, + ScorpioFs, + BrewFs, +} + +impl BackendKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Local => "local", + Self::ScorpioFs => "scorpiofs", + Self::BrewFs => "brewfs", + } + } +} + +impl std::fmt::Display for BackendKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackendCapabilities { + pub posix_mount: bool, + pub revision_projection: bool, + pub native_change_detection: bool, + pub persistent_volume: bool, + pub multi_client: bool, + pub flush_before_commit: bool, +} + +impl BackendCapabilities { + pub const fn local() -> Self { + Self { + posix_mount: true, + revision_projection: false, + native_change_detection: false, + persistent_volume: false, + multi_client: false, + flush_before_commit: false, + } + } + + pub const fn scorpiofs() -> Self { + Self { + posix_mount: true, + revision_projection: true, + native_change_detection: true, + persistent_volume: false, + multi_client: false, + flush_before_commit: false, + } + } + + pub const fn brewfs() -> Self { + Self { + posix_mount: true, + revision_projection: false, + native_change_detection: false, + persistent_volume: true, + multi_client: true, + flush_before_commit: true, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BackendDescriptor { + pub kind: BackendKind, + pub display_name: &'static str, + pub protocol_version: u32, + pub capabilities: BackendCapabilities, + pub available: bool, + pub unavailable_reason: Option<&'static str>, +} + +impl BackendDescriptor { + pub const fn local() -> Self { + Self { + kind: BackendKind::Local, + display_name: "Local filesystem", + protocol_version: BACKEND_CONTROL_PROTOCOL_VERSION, + capabilities: BackendCapabilities::local(), + available: true, + unavailable_reason: None, + } + } + + pub const fn scorpiofs(available: bool) -> Self { + Self { + kind: BackendKind::ScorpioFs, + display_name: "ScorpioFS remote projection", + protocol_version: BACKEND_CONTROL_PROTOCOL_VERSION, + capabilities: BackendCapabilities::scorpiofs(), + available, + unavailable_reason: if available { + None + } else { + Some("requires Linux and the scorpiofs-direct feature") + }, + } + } + + pub const fn brewfs(available: bool) -> Self { + Self { + kind: BackendKind::BrewFs, + display_name: "BrewFS persistent volume", + protocol_version: BACKEND_CONTROL_PROTOCOL_VERSION, + capabilities: BackendCapabilities::brewfs(), + available, + unavailable_reason: if available { + None + } else { + Some("requires a BrewFS SDK runtime implementation") + }, + } + } +} + +pub struct BackendRegistry; + +impl BackendRegistry { + pub fn builtins() -> Vec { + vec![ + BackendDescriptor::local(), + BackendDescriptor::scorpiofs(cfg!(all( + target_os = "linux", + feature = "scorpiofs-direct" + ))), + // The driver boundary is implemented, but BrewFS 0.1.2 does not + // yet export the complete mount constructor used by its binary. + BackendDescriptor::brewfs(false), + ] + } + + pub fn descriptor(kind: BackendKind) -> Option { + Self::builtins() + .into_iter() + .find(|descriptor| descriptor.kind == kind) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum BackendMountSource { + LocalDirectory { + path: PathBuf, + }, + RemoteProjection { + remote_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + base_oid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + change_layer: Option, + }, + PersistentVolume { + volume: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + subpath: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackendMountRequest { + pub instance_id: String, + pub worktree_id: String, + pub source: BackendMountSource, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mountpoint_hint: Option, + #[serde(default = "default_ready_timeout_secs")] + pub ready_timeout_secs: u64, +} + +fn default_ready_timeout_secs() -> u64 { + 120 +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackendMountSession { + pub backend: BackendKind, + pub session_id: String, + pub mountpoint: PathBuf, + pub cleanup_key: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_oid: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackendHealth { + pub ready: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BackendLifecycle { + Detached, + Mounting, + Ready, + SwitchingBase, + Unmounting, + RecoverableError, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChangeKind { + Added, + Modified, + Deleted, + Renamed, + ModeChanged, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangedPath { + pub kind: ChangeKind, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_path: Option, +} + +impl ChangedPath { + pub fn validate(&self) -> Result<(), WorktreeBackendError> { + validate_relative_path(&self.path)?; + if let Some(source_path) = self.source_path.as_deref() { + validate_relative_path(source_path)?; + } + if matches!(self.kind, ChangeKind::Renamed) && self.source_path.is_none() { + return Err(WorktreeBackendError::InvalidChangedPath( + "a renamed path must include source_path".to_string(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangeSet { + pub mount_id: String, + pub generation: u64, + #[serde(default)] + pub changes: Vec, +} + +impl ChangeSet { + pub fn validate(&self) -> Result<(), WorktreeBackendError> { + if self.mount_id.trim().is_empty() { + return Err(WorktreeBackendError::InvalidRequest( + "change set mount_id cannot be empty".to_string(), + )); + } + for change in &self.changes { + change.validate()?; + } + Ok(()) + } + + pub fn candidate_paths(&self) -> Vec { + let mut paths = Vec::with_capacity(self.changes.len() * 2); + for change in &self.changes { + paths.push(PathBuf::from(&change.path)); + if let Some(source_path) = change.source_path.as_deref() { + paths.push(PathBuf::from(source_path)); + } + } + paths.sort(); + paths.dedup(); + paths + } +} + +#[derive(Debug, Error)] +pub enum WorktreeBackendError { + #[error("invalid worktree backend request: {0}")] + InvalidRequest(String), + #[error("backend '{backend}' does not support source type '{detail}'")] + UnsupportedSource { + backend: BackendKind, + detail: &'static str, + }, + #[error("worktree backend '{backend}' is unavailable: {reason}")] + Unavailable { + backend: BackendKind, + reason: String, + }, + #[error("worktree backend '{backend}' operation '{operation}' failed: {source}")] + Operation { + backend: BackendKind, + operation: &'static str, + #[source] + source: anyhow::Error, + }, + #[error("invalid backend changed path: {0}")] + InvalidChangedPath(String), +} + +impl WorktreeBackendError { + pub fn operation( + backend: BackendKind, + operation: &'static str, + source: impl Into, + ) -> Self { + Self::Operation { + backend, + operation, + source: source.into(), + } + } +} + +#[async_trait] +pub trait WorktreeBackendDriver: Send + Sync { + fn descriptor(&self) -> BackendDescriptor; + + async fn mount( + &self, + request: &BackendMountRequest, + ) -> Result; + + async fn health( + &self, + session: &BackendMountSession, + ) -> Result; + + async fn changed_paths( + &self, + _session: &BackendMountSession, + ) -> Result, WorktreeBackendError> { + Ok(None) + } + + async fn flush(&self, _session: &BackendMountSession) -> Result<(), WorktreeBackendError> { + Ok(()) + } + + async fn unmount(&self, session: &BackendMountSession) -> Result<(), WorktreeBackendError>; + + async fn recover( + &self, + request: &BackendMountRequest, + ) -> Result { + self.mount(request).await + } +} + +fn validate_relative_path(path: &str) -> Result<(), WorktreeBackendError> { + if path.is_empty() || path.starts_with('/') || path.contains('\0') { + return Err(WorktreeBackendError::InvalidChangedPath( + "path must be a non-empty relative path".to_string(), + )); + } + if path.split('/').any(|part| matches!(part, "" | "." | "..")) { + return Err(WorktreeBackendError::InvalidChangedPath( + "path must be normalized and must not contain traversal".to_string(), + )); + } + Ok(()) +} diff --git a/src/internal/worktree_scope.rs b/src/internal/worktree_scope.rs index 76fd2b54b..013f504c0 100644 --- a/src/internal/worktree_scope.rs +++ b/src/internal/worktree_scope.rs @@ -377,12 +377,37 @@ mod tests { #[test] #[serial_test::serial] fn a_pinned_scope_survives_a_cwd_change() { + // A REAL repository, because `pin_scope_for_test` resolves its paths + // through `RequestScope::resolve` and installs nothing when the + // workdir is not inside one — pinning the ambient cwd only appeared to + // work when another test had parked it in someone else's fixture. + // + // Built BEFORE the cwd lock below: `setup_with_new_libra_in` takes and + // releases that lock itself, and running a whole `libra init` while + // holding it would block every other fixture in the suite for the + // duration. + let repo = tempfile::tempdir().expect("repo"); + { + let _cd = crate::utils::test::ChangeDirGuard::new(repo.path()); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime") + .block_on(crate::utils::test::setup_with_new_libra_in(repo.path())); + } + + // Moving the process cwd is not this test's business alone: every + // `ChangeDirGuard` in the suite reads the same one. `#[serial]` only + // orders this against other `#[serial]` tests, so hold the cwd lock + // the guard uses — otherwise the `set_current_dir` below yanks the + // directory out from under whatever repository fixture is mid-flight. + let _cwd_lock = crate::utils::test::cwd_lock_guard(); let original = std::env::current_dir().expect("cwd"); let elsewhere = std::env::temp_dir(); let _pin = WorktreeScope::pin_scope_for_test( WorktreeScope::Linked("wt-pinned".to_string()), - original.clone(), + repo.path().to_path_buf(), ); assert!(WorktreeScope::request_scope_is_pinned()); assert_eq!(WorktreeScope::for_request().storage_key(), "wt-pinned"); @@ -432,6 +457,10 @@ mod tests { .expect("runtime") .block_on(crate::utils::test::setup_with_new_libra_in(repo.path())); } + // See `a_pinned_scope_survives_a_cwd_change`: the raw `set_current_dir` + // below is process-wide, so it has to hold `ChangeDirGuard`'s lock — + // taken only now, so the `libra init` above does not run under it. + let _cwd_lock = crate::utils::test::cwd_lock_guard(); let original = std::env::current_dir().expect("cwd"); // Pin a SUBDIRECTORY, which is what a command invoked from one does. @@ -483,6 +512,10 @@ mod tests { .expect("runtime") .block_on(crate::utils::test::setup_with_new_libra_in(repo.path())); } + // See `a_pinned_scope_survives_a_cwd_change`: the raw `set_current_dir` + // below is process-wide, so it has to hold `ChangeDirGuard`'s lock — + // taken only now, so the `libra init` above does not run under it. + let _cwd_lock = crate::utils::test::cwd_lock_guard(); let original = std::env::current_dir().expect("cwd"); let canonical_repo = std::fs::canonicalize(repo.path()).unwrap_or_else(|_| repo.path().to_path_buf()); @@ -523,11 +556,15 @@ mod tests { async fn the_request_database_follows_the_pin_not_the_cwd() { let repo_a = tempfile::tempdir().expect("repo a"); let repo_b = tempfile::tempdir().expect("repo b"); - let original = std::env::current_dir().expect("cwd"); for repo in [repo_a.path(), repo_b.path()] { let _cd = crate::utils::test::ChangeDirGuard::new(repo); crate::utils::test::setup_with_new_libra_in(repo).await; } + // See `a_pinned_scope_survives_a_cwd_change`: the raw `set_current_dir` + // below is process-wide, so it has to hold `ChangeDirGuard`'s lock — + // taken only now, so the two `libra init`s above do not run under it. + let _cwd_lock = crate::utils::test::cwd_lock_guard(); + let original = std::env::current_dir().expect("cwd"); let _pin = WorktreeScope::pin_request_scope(repo_a.path().to_path_buf()); // The cwd is repository B; the pin is repository A. @@ -576,11 +613,15 @@ mod tests { let outer = tempfile::tempdir().expect("the enclosing repository"); let ambient = tempfile::tempdir().expect("the repository the cwd is in"); let nowhere = tempfile::tempdir().expect("not a repository"); - let original = std::env::current_dir().expect("cwd"); for repo in [outer.path(), ambient.path()] { let _cd = crate::utils::test::ChangeDirGuard::new(repo); crate::utils::test::setup_with_new_libra_in(repo).await; } + // See `a_pinned_scope_survives_a_cwd_change`: the raw `set_current_dir` + // below is process-wide, so it has to hold `ChangeDirGuard`'s lock — + // taken only now, so the two `libra init`s above do not run under it. + let _cwd_lock = crate::utils::test::cwd_lock_guard(); + let original = std::env::current_dir().expect("cwd"); let _outer_pin = WorktreeScope::pin_request_scope(outer.path().to_path_buf()); std::env::set_current_dir(ambient.path()).expect("move the cwd"); diff --git a/src/utils/path.rs b/src/utils/path.rs index 11d9e6266..2e8f1c025 100644 --- a/src/utils/path.rs +++ b/src/utils/path.rs @@ -98,6 +98,11 @@ mod tests { let linked = root.path().join("linked"); let linked_gitdir = linked.join(".libra"); fs::create_dir_all(common.join("objects")).expect("create shared object store"); + // The commondir target must look like TERMINAL common storage + // (`util::is_terminal_common_storage`), which an object store alone + // does not: a real main gitdir also carries the repository database, + // and without it the pointer is refused as corruption. + fs::write(common.join(crate::utils::util::DATABASE), b"").expect("create repository db"); fs::create_dir_all(&linked_gitdir).expect("create linked worktree gitdir"); fs::write( linked_gitdir.join("commondir"), diff --git a/src/utils/util.rs b/src/utils/util.rs index b47dc3aa5..43defe1e4 100644 --- a/src/utils/util.rs +++ b/src/utils/util.rs @@ -180,6 +180,75 @@ fn read_gitdir_file(path: &Path, worktree: &Path) -> Option { }) } +/// Resolve a Libra linked-worktree pointer file. +/// +/// External worktree backends such as ScorpioFS cannot keep their private +/// index and HEAD inside an ephemeral mount. They place a regular `.libra` +/// file in the mounted worktree containing `gitdir: `, while the real +/// worktree gitdir remains under the main repository's persistent storage. +/// +/// Unlike the best-effort Git repository discovery helper above, Libra +/// repository discovery is fail-closed: a present but malformed pointer is a +/// corrupt repository and must not be silently ignored. +fn read_libra_gitdir_file(path: &Path, worktree: &Path) -> io::Result { + let contents = fs::read_to_string(path).map_err(|error| { + io::Error::new( + error.kind(), + format!( + "cannot read Libra worktree pointer '{}': {error}", + path.display() + ), + ) + })?; + let line = contents.lines().next().map(str::trim).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Libra worktree pointer '{}' is empty", path.display()), + ) + })?; + let raw = line + .strip_prefix("gitdir:") + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Libra worktree pointer '{}' must contain 'gitdir: '", + path.display() + ), + ) + })?; + + let configured = Path::new(raw); + let resolved = if configured.is_absolute() { + configured.to_path_buf() + } else { + worktree.join(configured) + }; + let resolved = fs::canonicalize(&resolved).map_err(|error| { + io::Error::new( + error.kind(), + format!( + "Libra worktree pointer '{}' targets unavailable gitdir '{}': {error}", + path.display(), + resolved.display() + ), + ) + })?; + if !resolved.is_dir() || !is_valid_storage_dir(&resolved) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Libra worktree pointer '{}' targets invalid gitdir '{}'", + path.display(), + resolved.display() + ), + )); + } + Ok(resolved) +} + fn resolve_dot_git_dir(worktree: &Path) -> Option { let dot_git = worktree.join(".git"); let metadata = fs::metadata(&dot_git).ok()?; @@ -590,6 +659,12 @@ fn try_get_paths_full(path: Option) -> Result<(PathBuf, PathBuf, PathBu return Ok((common, path.clone(), gitdir)); } + if standard_repo.is_file() { + let gitdir = read_libra_gitdir_file(&standard_repo, &path)?; + let common = worktree_common_storage(&gitdir)?; + return Ok((common, path.clone(), gitdir)); + } + if path.join(DATABASE).exists() && path.join("objects").exists() { return Ok((path.clone(), path.clone(), path.clone())); } @@ -3497,6 +3572,68 @@ mod test { assert!(!location.is_bare); } + #[test] + #[serial] + fn test_libra_worktree_pointer_resolves_external_gitdir() { + let temp = tempdir().unwrap(); + let main_storage = temp.path().join("main").join(".libra"); + let external_gitdir = main_storage + .join("worktrees") + .join("scorpiofs") + .join("workspace-1"); + let mounted_worktree = temp.path().join("mount"); + fs::create_dir_all(main_storage.join("objects")).unwrap(); + fs::create_dir_all(main_storage.join("hooks")).unwrap(); + fs::create_dir_all(main_storage.join("info")).unwrap(); + fs::write(main_storage.join(DATABASE), b"repo db").unwrap(); + fs::create_dir_all(&external_gitdir).unwrap(); + fs::write( + external_gitdir.join("commondir"), + format!("{}\n", main_storage.display()), + ) + .unwrap(); + fs::write(external_gitdir.join("worktree_id"), b"workspace-1\n").unwrap(); + fs::create_dir_all(&mounted_worktree).unwrap(); + fs::write( + mounted_worktree.join(ROOT_DIR), + format!("gitdir: {}\n", external_gitdir.display()), + ) + .unwrap(); + + let _guard = test::ChangeDirGuard::new(&mounted_worktree); + + assert_eq!( + try_get_storage_path(None).unwrap(), + main_storage.canonicalize().unwrap() + ); + assert_eq!( + try_get_worktree_gitdir(None).unwrap(), + external_gitdir.canonicalize().unwrap() + ); + assert_eq!(current_worktree_id().as_deref(), Some("workspace-1")); + } + + #[test] + #[serial] + fn test_libra_worktree_pointer_fails_closed_when_target_is_missing() { + let temp = tempdir().unwrap(); + let mounted_worktree = temp.path().join("mount"); + fs::create_dir_all(&mounted_worktree).unwrap(); + fs::write( + mounted_worktree.join(ROOT_DIR), + "gitdir: /definitely/missing/libra-worktree\n", + ) + .unwrap(); + + let _guard = test::ChangeDirGuard::new(&mounted_worktree); + let error = try_get_storage_path(None).expect_err("missing gitdir must fail"); + + assert!( + error.to_string().contains("targets unavailable gitdir"), + "unexpected error: {error}" + ); + } + /// W0 §C.4.1.1 (plan line 2262): info files belong to the WORKTREE VIEW. /// A Git-layout linked worktree resolves its OWN gitdir's `info/`, /// not the commondir target's — the pre-W0 behavior (follow `commondir`) diff --git a/tests/command/worktree_test.rs b/tests/command/worktree_test.rs index f6cb42331..e0e197b9d 100644 --- a/tests/command/worktree_test.rs +++ b/tests/command/worktree_test.rs @@ -6,6 +6,7 @@ use std::fs; #[cfg(unix)] use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; +use axum::{Json, Router}; use clap::Parser; use libra::{ command::{ @@ -127,6 +128,145 @@ fn assert_worktree_error(output: &std::process::Output, error_code: &str) -> Cli report } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn test_scorpiofs_attach_is_persistent_idempotent_and_detachable() { + let repo_dir = tempdir().unwrap(); + let mount_dir = tempdir().unwrap(); + test::setup_with_new_libra_in(repo_dir.path()).await; + let _guard = test::ChangeDirGuard::new(repo_dir.path()); + + let mountpoint = mount_dir.path().canonicalize().unwrap(); + fs::write(mountpoint.join("hello.txt"), "hello from ScorpioFS\n").unwrap(); + fs::write( + mountpoint.join("unreported-local-artifact.txt"), + "not part of the ScorpioFS change set\n", + ) + .unwrap(); + let mountpoint_json = mountpoint.to_string_lossy().into_owned(); + let mount_response_path = mountpoint_json.clone(); + let app = Router::new() + .route( + "/health", + axum::routing::get(|| async { + Json(serde_json::json!({ + "protocol_version": 1, + "service": "scorpiofs", + "service_version": "test", + "capabilities": ["mount.v1", "ready.v1", "changes.v1"], + "status": "healthy", + "mount_count": 1, + "uptime_secs": 0 + })) + }), + ) + .route( + "/mounts", + axum::routing::post(move || { + let mountpoint = mount_response_path.clone(); + async move { + Json(serde_json::json!({ + "mount_id": "11111111-1111-4111-8111-111111111111", + "mountpoint": mountpoint, + "ready": true + })) + } + }), + ) + .route( + "/mounts/11111111-1111-4111-8111-111111111111/changes", + axum::routing::get(|| async { + Json(serde_json::json!({ + "mount_id": "11111111-1111-4111-8111-111111111111", + "generation": 0, + "changes": [{ + "kind": "modified", + "path": "hello.txt" + }] + })) + }), + ) + .route( + "/mounts/by-job/dev-project", + axum::routing::delete(|| async { Json(serde_json::json!({"deleted": true})) }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + exec_worktree(&[ + "scorpiofs", + "attach", + "--endpoint", + &endpoint, + "--remote-path", + "/project", + "--job-id", + "dev-project", + ]) + .await + .expect("ScorpioFS attach should succeed"); + exec_worktree(&[ + "scorpiofs", + "attach", + "--endpoint", + &endpoint, + "--remote-path", + "/project", + "--job-id", + "dev-project", + ]) + .await + .expect("repeated ScorpioFS attach should be idempotent"); + + let pointer = mountpoint.join(util::ROOT_DIR); + assert!( + pointer.is_file(), + "mount root should contain a .libra pointer" + ); + let gitdir = util::try_get_worktree_gitdir(Some(mountpoint.clone())).unwrap(); + assert!(gitdir.join("backend.json").is_file()); + assert_eq!( + read_worktree_state() + .entries + .iter() + .filter(|entry| entry.path == mountpoint_json) + .count(), + 1 + ); + + let status = run_libra_command(&["status", "--porcelain"], &mountpoint); + assert_cli_success(&status, "ScorpioFS worktree status"); + assert!( + String::from_utf8_lossy(&status.stdout).contains("hello.txt"), + "changed-path candidate should be visible to status" + ); + let add = run_libra_command(&["add", "hello.txt"], &mountpoint); + assert_cli_success(&add, "ScorpioFS worktree add"); + let commit = run_libra_command( + &["commit", "-m", "test ScorpioFS-backed commit"], + &mountpoint, + ); + assert_cli_success(&commit, "ScorpioFS worktree commit"); + let status = run_libra_command(&["status", "--porcelain"], &mountpoint); + assert_cli_success(&status, "clean ScorpioFS worktree status"); + assert!( + status.stdout.is_empty(), + "committed ScorpioFS worktree should be clean: {}", + String::from_utf8_lossy(&status.stdout) + ); + + exec_worktree(&["scorpiofs", "detach", &mountpoint_json]) + .await + .expect("ScorpioFS detach should succeed"); + assert!(!pointer.exists()); + assert!(!gitdir.exists()); + + server.abort(); +} + /// §C.8 (W4 review): the `worktree list` JSON data half carries its OWN /// `schema_version` (the global `--json` flag never selects a schema), the /// v2 fields are present, `--schema-version 2` is the explicit spelling of