From fd9c763e55351c893d88250eb16cee4a45faa98b Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 20:05:53 +0200 Subject: [PATCH 01/10] Extract elfuse_launch into src/core/launch.c elfuse_launch owns guest bring-up: guest_bootstrap_prepare, the FUSE-temp unlink, the sysroot casefold probe, vCPU creation, GDB init/sync/wait, the run loop, gdb_stub_shutdown, the shim counter and syscall histogram dumps, and guest_destroy. main() retains the original CLI argv (proctitle rewriting), option parsing, sysroot provisioning, the shebang loop, the --gdb x86_64 guard, host cwd, and the heap resource cleanup, and now hands off through launch_args_t so other launchers (the OCI run helper) can share one bring-up path. The launch_args_t envp field generalizes the old hard-coded environ: NULL keeps the host environment, so main()'s behavior is unchanged. Bring-up failures unwind through a single fail label instead of repeating the guest_destroy-plus-unlink tail at every error site. Ownership of the FUSE-materialized temp ELF moves with the bring-up: elfuse_launch owns the unlink from the prepare call onward (teardown and the post-prepare error paths), and main() drops its claim before handing off, so main's shared goto unwind cannot double-unlink a path whose ownership has been transferred. The embedded shim blob include moves along with its only consumer, so shim_bin has a single object definition site. --- Makefile | 1 + src/core/launch.c | 203 ++++++++++++++++++++++++++++++++++++++++++++++ src/core/launch.h | 94 +++++++++++++++++++++ src/main.c | 161 ++++++++++-------------------------- 4 files changed, 340 insertions(+), 119 deletions(-) create mode 100644 src/core/launch.c create mode 100644 src/core/launch.h diff --git a/Makefile b/Makefile index d50e35d3..41634c03 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ SRCS := \ core/vdso.c \ core/shim-globals.c \ core/bootstrap.c \ + core/launch.c \ core/rosetta.c \ core/sysroot.c \ runtime/thread.c \ diff --git a/src/core/launch.c b/src/core/launch.c new file mode 100644 index 00000000..d92e30dc --- /dev/null +++ b/src/core/launch.c @@ -0,0 +1,203 @@ +/* elfuse VM launch: bring-up + GDB + run loop + teardown + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Implementation of elfuse_launch (contract and caller/callee ownership in + * launch.h). Extracted from src/main.c so the positional-ELF CLI and other + * launchers (e.g. the OCI run helper) share one bring-up path. + * + * shim_blob.h is included here, not in src/main.c, so the static + * shim_bin / shim_bin_len blob has a single object definition site. + */ + +#include "launch.h" + +#include +#include +#include +#include +#include +#include + +#include "core/bootstrap.h" +#include "core/guest.h" +#include "core/shim-globals.h" +#include "core/sysroot.h" + +#include "runtime/futex.h" /* futex_interrupt_request */ +#include "runtime/thread.h" +#include "syscall/poll.h" /* wakeup_pipe_signal */ +#include "syscall/proc.h" + +#include "debug/gdbstub.h" +#include "debug/log.h" +#include "debug/syscall-hist.h" + +/* Embedded shim binary (generated by xxd -i from shim.bin). */ +#include "shim_blob.h" + +/* The shim code slot in the infra reserve is sized tight (INFRA_SHIM_SLOT, a + * few x the current blob) so the rest of the reserve goes to the page-table + * pool. If the shim ever outgrows the slot it would overlap the shim-data + * block; fail the build loudly rather than corrupt memory at boot. Enlarge + * INFRA_SHIM_SLOT (and shrink the pool to match) if this fires. + */ +_Static_assert(sizeof(shim_bin) <= INFRA_SHIM_SLOT, + "shim blob exceeds its infra slot; bump INFRA_SHIM_SLOT"); + +int elfuse_launch(const launch_args_t *args) +{ + if (!args) { + log_error("elfuse_launch: NULL args"); + return 1; + } + + extern char **environ; + char **envp_use = args->envp ? args->envp : environ; + + guest_t g; + bool guest_initialized = false; + guest_bootstrap_t boot; + /* Track the temp flag locally: elfuse_launch owns the post-prepare + * unlink of a FUSE-materialized temp, but the caller's launch_args_t + * is const and its copy is discarded after the call anyway. + */ + bool elf_host_temp = args->elf_host_temp; + /* The guest-visible entrypoint path is argv[0]; elf_path is the + * resolved host path to that binary. They differ when path + * translation or a FUSE-materialized temp is involved. + */ + const char *elf_guest_path = (args->guest_argc > 0 && args->guest_argv) + ? args->guest_argv[0] + : args->elf_path; + + if (guest_bootstrap_prepare( + &g, args->elf_path, elf_host_temp, elf_guest_path, args->sysroot, + args->guest_argc, args->guest_argv, envp_use, shim_bin, + shim_bin_len, args->verbose, &guest_initialized, &boot) < 0) + goto fail; + + /* A FUSE-materialized temp has been loaded; drop it once the guest + * has its own mapping, unless Rosetta still needs the reopenable + * host path. + */ + if (elf_host_temp && !g.is_rosetta) { + unlink(args->elf_path); + elf_host_temp = false; + } + + if (args->sysroot) { + bool case_sensitive = true; + bool case_preserving = true; + if (sysroot_probe_case_sensitivity(args->sysroot, &case_sensitive, + &case_preserving) == 0) + proc_set_sysroot_casefold(case_preserving && !case_sensitive); + else + proc_set_sysroot_casefold(false); + } else { + proc_set_sysroot_casefold(false); + } + + /* fork_child / vfork_notify dispatch stays in main() (early return + * before reaching elfuse_launch). The fields are kept on + * launch_args_t so callers route through one launch struct shape + * even when the IPC plumbing changes. + */ + (void) args->fork_child_fd; + (void) args->vfork_notify_fd; + + hv_vcpu_t vcpu; + hv_vcpu_exit_t *vexit; + if (guest_bootstrap_create_vcpu(&g, &boot, args->verbose, &vcpu, &vexit) < + 0) + goto fail; + + /* GDB setup must happen before the first run so entry-stop and + * hardware breakpoints can affect the initial vCPU. + */ + if (args->gdb_port > 0) { + if (gdb_stub_init(args->gdb_port, &g) < 0) { + log_error("failed to initialize GDB stub"); + goto fail; + } + gdb_stub_sync_debug_regs(vcpu); + if (args->gdb_stop_on_entry) + gdb_stub_wait_for_attach(); + } + + /* vcpu_run_loop owns guest execution until exit, fatal signal, or timeout. + */ + int exit_code = vcpu_run_loop(vcpu, vexit, &g, args->verbose, + args->timeout_sec, NULL); + + /* Tear down debugger state before joining workers: a worker parked in + * gdb_stub_handle_stop() stays active (not deactivated) until this + * broadcasts resume_cond, so joining first would just time out and + * detach it while it is still paused. + */ + gdb_stub_shutdown(); + + /* Join worker vCPU threads before guest_destroy unmaps the guest slab: a + * sibling still mid-iteration in its own run loop would fault on freed + * guest memory and crash the host with SIGSEGV, masking the real exit + * code. The join is a no-op once workers have wound down (the common + * single-threaded case). + * + * vcpu_run_loop can also return via a bare break (alarm timeout 124, a + * fatal default-disposition signal, or ELR_EL1==0) with no one having + * requested exit_group or kicked the siblings out of hv_vcpu_run. Mirror + * guest_destroy's request-interrupt prefix here first; otherwise this join + * burns its full poll cap and detaches every worker, and guest_destroy's + * own interrupt-join skips them (it honors join_abandoned), leaving live + * pthreads to fault on the imminent unmap. + */ + if (!proc_exit_group_requested()) + proc_request_exit_group(0); + futex_interrupt_request(); + wakeup_pipe_signal(); + thread_interrupt_all(); + /* Workers parked on internal condvars (fork barrier, ptrace stop/wait) + * see neither the pipe nor the vCPU kick; broadcast so they re-check the + * exit-group flag and terminate before the join below gives up on them. + */ + thread_wake_exit_waiters(); + thread_join_workers(); + + /* Diagnostic counter dump runs before guest_destroy so the + * shim_data mapping is still valid. ELFUSE_SHIM_STATS is the gate; + * an unset variable produces no output. + */ + if (shim_globals_stats_enabled()) + shim_globals_counters_dump(&g); + + /* Dump the startup histogram before guest_destroy so any + * cleanup-path syscalls (closing host fds, unmapping the slab) do + * not appear in the captured set. The dump is a no-op when + * ELFUSE_STARTUP_TRACE=syscalls was not requested. + */ + syscall_hist_dump(); + + if (guest_initialized) + guest_destroy(&g); + + /* Rosetta guests keep the FUSE-materialized temp alive for the whole run + * (the translator reopens the host path); drop it now that the guest is + * gone so repeated Rosetta launches do not accumulate temp files. + */ + if (elf_host_temp) + unlink(args->elf_path); + + return exit_code; + +fail: + /* Bring-up failed: unwind whatever exists so far. The caller owns + * pre-prepare failures; from the prepare call onward the temp unlink + * is ours. + */ + if (guest_initialized) + guest_destroy(&g); + if (elf_host_temp) + unlink(args->elf_path); + return 1; +} diff --git a/src/core/launch.h b/src/core/launch.h new file mode 100644 index 00000000..0f4e20e5 --- /dev/null +++ b/src/core/launch.h @@ -0,0 +1,94 @@ +/* elfuse VM launch entry: post-CLI bring-up + run loop + teardown + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * elfuse_launch is the single entry point for "run a guest binary in a + * fresh HVF VM until it exits". It is shared between main() (legacy + * positional-ELF CLI) and future launchers such as the OCI run helper. + * + * The function owns the guest_t, the vCPU, the GDB stub, the run loop, + * the diagnostic dumps, and guest teardown; it does NOT own the + * elf_path / sysroot / guest_argv heap copies or the sysroot_mount the + * host CLI may have provisioned; those stay with the caller so + * behaviors that need the original CLI argv (proctitle rewriting, + * --create-sysroot detach on exit, host cwd save+restore) remain + * coherent regardless of how the launch was kicked off. + * + * Lifetime / ownership contract: + * + * - The caller owns every pointer in launch_args_t. elfuse_launch reads + * them and does not free them; const-qualified pointers stay valid + * for the duration of the call. + * - envp may be NULL; the host process environ is used in that case. + * - guest_argv is the string array the guest sees as its argv. + * guest_argv[0] is the guest-visible entrypoint path (what the guest + * reads back via /proc/self/exe and argv[0]); elf_path is the + * resolved host path to that binary. The two differ when path + * translation or a FUSE-materialized temp is involved. + * - elf_host_temp is true when elf_path is a FUSE-materialized temp + * that must be unlinked once guest_bootstrap_prepare has loaded it + * (skipped for Rosetta guests, which reopen the path). The caller + * owns the unlink for any pre-prepare failure; elfuse_launch owns it + * from the prepare call onward. + * - fork_child_fd / vfork_notify_fd are forwarded for fork-child-routed + * launches; main() dispatches the fork-child path before reaching + * elfuse_launch and so passes -1. + */ + +#pragma once + +#include +#include + +typedef struct { + /* Host filesystem path to the guest ELF (resolved; may be a + * FUSE-materialized temp when elf_host_temp is true). + */ + const char *elf_path; + + /* True when elf_path is a temp to unlink after guest_bootstrap_prepare + * loads it. + */ + bool elf_host_temp; + + /* Host filesystem path to the sysroot the guest sees as / (absolute), + * or NULL when the guest runs without a sysroot. + */ + const char *sysroot; + + /* String array the guest sees as its argv. guest_argv[0] is the + * guest-visible entrypoint path. + */ + int guest_argc; + const char **guest_argv; + + /* NULL-terminated guest environ. NULL means "use host environ". envp is + * char** (not const) to match the environ/guest_bootstrap_prepare + * convention: guest programs may mutate their environment. + */ + char **envp; + + /* GDB Remote Serial Protocol port (0 disables the stub) and whether + * to halt before the first guest instruction. + */ + int gdb_port; + bool gdb_stop_on_entry; + + /* Per-iteration vCPU run timeout. 0 disables (no alarm()). */ + int timeout_sec; + + /* Fork-child IPC handles. -1 means "not a fork child". main()'s + * --fork-child dispatch handles the >= 0 case before reaching + * elfuse_launch. + */ + int fork_child_fd, vfork_notify_fd; + + bool verbose; +} launch_args_t; + +/* Bring up the guest VM, run it to exit / signal / timeout, tear down, + * return the exit code. Returns 1 on bring-up failure (with a log + * message) and the guest's exit status otherwise. + */ +int elfuse_launch(const launch_args_t *args); diff --git a/src/main.c b/src/main.c index 3168797c..9a83db99 100644 --- a/src/main.c +++ b/src/main.c @@ -31,21 +31,17 @@ #include "core/bootstrap.h" #include "core/guest.h" +#include "core/launch.h" #include "core/rosetta.h" -#include "core/shim-globals.h" #include "core/sysroot.h" #include "runtime/forkipc.h" -#include "runtime/futex.h" /* futex_interrupt_request */ #include "runtime/proctitle.h" -#include "runtime/thread.h" #include "syscall/fuse.h" #include "syscall/path.h" -#include "syscall/poll.h" /* wakeup_pipe_signal */ #include "syscall/proc.h" -#include "debug/gdbstub.h" #include "debug/log.h" #include "debug/syscall-hist.h" @@ -128,17 +124,11 @@ static void cleanup_main_resources(guest_t *g, free((void *) sysroot_path); } -/* Embedded shim binary (generated by xxd -i from shim.bin) */ -#include "shim_blob.h" - -/* The shim code slot in the infra reserve is sized tight (INFRA_SHIM_SLOT, a - * few x the current blob) so the rest of the reserve goes to the page-table - * pool. If the shim ever outgrows the slot it would overlap the shim-data - * block; fail the build loudly rather than corrupt memory at boot. Enlarge - * INFRA_SHIM_SLOT (and shrink the pool to match) if this fires. +/* The embedded shim binary (shim_blob.h, generated by xxd -i from shim.bin) + * is now included by src/core/launch.c, the single site that hands the blob to + * guest_bootstrap_prepare. main() no longer references shim_bin/shim_bin_len + * directly, so the static blob has one object definition site. */ -_Static_assert(sizeof(shim_bin) <= INFRA_SHIM_SLOT, - "shim blob exceeds its infra slot; bump INFRA_SHIM_SLOT"); /* The infra-reserve layout invariants documented in guest.h are derived from * raw offset constants, so a future edit that grows the pool by shifting one @@ -606,119 +596,52 @@ int main(int argc, char **argv) } } - guest_bootstrap_t boot; - extern char **environ; - - if (guest_bootstrap_prepare(&g, elf_host_path, elf_host_temp, elf_path, - sysroot, guest_argc, guest_argv, environ, - shim_bin, shim_bin_len, verbose, - &guest_initialized, &boot) < 0) - goto fail; - if (elf_host_temp && !g.is_rosetta) { - unlink(elf_host_path); - elf_host_temp = false; - } - - if (have_sysroot) { - bool case_sensitive = true; - bool case_preserving = true; - if (sysroot_probe_case_sensitivity(sysroot, &case_sensitive, - &case_preserving) == 0) { - proc_set_sysroot_casefold(case_preserving && !case_sensitive); - } else { - proc_set_sysroot_casefold(false); - } - } else { - proc_set_sysroot_casefold(false); - } - - runtime_set_process_title(argc, argv, elf_path); - - hv_vcpu_t vcpu; - hv_vcpu_exit_t *vexit; - if (guest_bootstrap_create_vcpu(&g, &boot, verbose, &vcpu, &vexit) < 0) - goto fail; - - /* GDB setup must happen before the first run so entry-stop and hardware - * breakpoints can affect the initial vCPU. - */ - if (gdb_port > 0) { - if (gdb_stub_init(gdb_port, &g) < 0) { - log_error("failed to initialize GDB stub"); - goto fail; - } - /* Mirror any preconfigured breakpoints/watchpoints into this vCPU. */ - gdb_stub_sync_debug_regs(vcpu); - - if (gdb_stop_on_entry) - gdb_stub_wait_for_attach(); - } - - /* vcpu_run_loop owns guest execution until exit, fatal signal, or timeout. - */ - exit_code = vcpu_run_loop(vcpu, vexit, &g, verbose, timeout_sec, NULL); - - /* Tear down debugger state before joining workers: a worker parked in - * gdb_stub_handle_stop() stays active (not deactivated) until this - * broadcasts resume_cond, so joining first would just time out and - * detach it while it is still paused. */ - gdb_stub_shutdown(); - - /* Wait for worker vCPU threads to stop before tearing down guest memory. - * The main thread leaves the run loop as soon as it observes the - * exit_group flag, but sibling vCPU threads may still be mid-iteration in - * their own run loops (e.g. touching shim_globals). cleanup_main_resources - * unmaps the guest slab via guest_destroy, so a still-running worker would - * fault on freed guest memory and crash the host with SIGSEGV, masking the - * real exit code. thread_join_workers() is a no-op once the workers have - * already wound down (the common single-threaded case). - * - * vcpu_run_loop can also return here without anyone having requested - * exit_group or kicked the siblings out of hv_vcpu_run: the alarm timeout - * (exit_code 124), a fatal default-disposition signal, or ELR_EL1==0 all - * bail out with a bare break. On those paths siblings are still spinning - * in the guest, so mirror guest_destroy's request-interrupt prefix before - * joining -- otherwise this call burns its full poll cap, detaches every - * worker, and guest_destroy's own request-interrupt-join (which honors - * join_abandoned) skips them, leaving live pthreads to fault on the - * imminent unmap. + /* Rewrite the host-visible process title from the guest entrypoint. This + * clobbers the original argv block (already snapshotted into the heap + * elf_path / guest_argv above), so it must run before elfuse_launch hands + * control to the guest but after the shebang loop has fixed elf_path. The + * call only touches the original argv and the kernel procname; it does not + * depend on guest_bootstrap_prepare having run, so hoisting it out of the + * bring-up (where it previously sat between prepare and create_vcpu) is + * behavior-preserving. */ - if (!proc_exit_group_requested()) - proc_request_exit_group(0); - futex_interrupt_request(); - wakeup_pipe_signal(); - thread_interrupt_all(); - /* Workers parked on internal condvars (fork barrier, ptrace stop/wait) - * see neither the pipe nor the vCPU kick; broadcast so they re-check the - * exit-group flag and terminate before the join below gives up on them. - */ - thread_wake_exit_waiters(); - thread_join_workers(); + runtime_set_process_title(argc, argv, elf_path); - /* Diagnostic counter dump runs before guest_destroy so the shim_data - * mapping is still valid. ELFUSE_SHIM_STATS is the gate; an unset variable - * produces no output. + /* Hand the bring-up, run loop, and guest teardown to elfuse_launch. main() + * retains ownership of the original argv (proctitle above), the sysroot + * mount (detached in cleanup_main_resources after the guest exits so the + * mount stays live for the whole run), host cwd, and the heap elf_path / + * sysroot_path / guest_argv copies. */ - if (shim_globals_stats_enabled()) - shim_globals_counters_dump(&g); - - /* Dump the startup histogram before guest_destroy so any cleanup-path - * syscalls (closing host fds, unmapping the slab) do not appear in the - * captured set. The dump is a no-op when ELFUSE_STARTUP_TRACE=syscalls was - * not requested. + launch_args_t largs = { + .elf_path = elf_host_path, + .elf_host_temp = elf_host_temp, + .sysroot = sysroot, + .guest_argc = guest_argc, + .guest_argv = guest_argv, + .gdb_port = gdb_port, + .gdb_stop_on_entry = gdb_stop_on_entry, + .timeout_sec = timeout_sec, + .fork_child_fd = fork_child_fd, + .vfork_notify_fd = vfork_notify_fd, + .verbose = verbose, + }; + /* elfuse_launch owns the temp unlink from the prepare call onward, so + * drop main()'s claim before handing off: the shared cleanup below must + * not unlink a path whose ownership has been transferred. */ - syscall_hist_dump(); + elf_host_temp = false; + exit_code = elfuse_launch(&largs); goto cleanup; fail: exit_code = 1; cleanup: - /* Single unwind for every exit past the heap-copy allocations: frees the - * caller-owned heap copies (guest_destroy included, via - * cleanup_main_resources, once the guest came up), detaches the sysroot - * mount, restores the host cwd, and drops a still-owned FUSE-materialized - * temp ELF, which the post-prepare error paths and a Rosetta guest's - * teardown previously leaked. + /* Single unwind for every exit past the heap-copy allocations. On the + * success path elfuse_launch has already run guest_destroy + * (guest_initialized never becomes true in main), so this frees the + * caller-owned heap copies, detaches the sysroot mount, restores the + * host cwd, and drops a still-owned FUSE-materialized temp ELF. */ cleanup_main_resources(&g, guest_initialized, &sysroot_mount, have_host_cwd ? host_cwd : NULL, guest_argv, From 8034a5c4d1a24290609ccd0d893be743b257a6ec Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:10:43 +0200 Subject: [PATCH 02/10] Add --user, --workdir, and --env launch flags An OCI image front end needs to set the guest identity, working directory, and environment without patching the runtime; the new flags map onto launch_args_t fields and `elfuse-oci run` drives exactly this interface. The --user identity is staged before bring-up (proc_set_initial_ids) so the auxv AT_UID/AT_GID snapshot taken by build_linux_stack matches what getuid()/getgid() later report. --workdir rejects non-absolute paths up front instead of silently resolving them against the host cwd, and is applied by elfuse_launch after the casefold probe so the translation sees the sysroot's real case behavior. --env/--clear-env build the guest environment with env(1) semantics: KEY=VAL sets, bare KEY inherits from the host environ, --clear-env starts empty; with neither flag given envp stays NULL and the host environ is used unchanged. The new heap resources join main()'s shared goto unwind: envp, workdir, and the raw --env override array are released at the single cleanup label on every exit path. --- src/core/launch.c | 49 ++++++ src/core/launch.h | 66 ++++---- src/main.c | 327 +++++++++++++++++++++++++++++++++--- src/syscall/proc-identity.c | 21 +++ src/syscall/proc.h | 7 + 5 files changed, 412 insertions(+), 58 deletions(-) diff --git a/src/core/launch.c b/src/core/launch.c index d92e30dc..26577b26 100644 --- a/src/core/launch.c +++ b/src/core/launch.c @@ -15,9 +15,11 @@ #include #include +#include #include #include #include +#include #include #include "core/bootstrap.h" @@ -27,6 +29,7 @@ #include "runtime/futex.h" /* futex_interrupt_request */ #include "runtime/thread.h" +#include "syscall/path.h" #include "syscall/poll.h" /* wakeup_pipe_signal */ #include "syscall/proc.h" @@ -72,6 +75,14 @@ int elfuse_launch(const launch_args_t *args) ? args->guest_argv[0] : args->elf_path; + /* Stage --user before bring-up: prepare's proc_init re-seeds the identity + * state, and build_linux_stack snapshots it into auxv AT_UID/AT_GID. + * Setting the ids after prepare would leave getauxval() reporting the + * default identity while getuid() reports the requested one. + */ + if (args->has_creds) + proc_set_initial_ids(args->uid, args->gid); + if (guest_bootstrap_prepare( &g, args->elf_path, elf_host_temp, elf_guest_path, args->sysroot, args->guest_argc, args->guest_argv, envp_use, shim_bin, @@ -87,6 +98,18 @@ int elfuse_launch(const launch_args_t *args) elf_host_temp = false; } + /* Reject GDB for a Rosetta (x86_64) guest here, not just in main(): the + * stub exposes the aarch64 shim's register/memory view, which is the wrong + * architecture for a Rosetta-translated x86_64 guest. main() rejects it up + * front via a static ELF probe, but enforcing it in elfuse_launch (once + * bring-up has set g.is_rosetta) makes every caller inherit the constraint, + * including the planned OCI run helper. + */ + if (args->gdb_port > 0 && g.is_rosetta) { + log_error("--gdb is not supported for x86_64 (Rosetta) guests"); + goto fail; + } + if (args->sysroot) { bool case_sensitive = true; bool case_preserving = true; @@ -99,6 +122,32 @@ int elfuse_launch(const launch_args_t *args) proc_set_sysroot_casefold(false); } + /* Apply the guest's initial working directory. The guest cwd IS the host + * process cwd (sys_chdir translates a guest path and calls host chdir), + * so --workdir DIR does the same: translate DIR against the sysroot (now + * that casefold is configured above) and chdir to the resulting host path, + * then refresh the cached guest-visible cwd so the first getcwd sees DIR. + * This mirrors the plain real-directory branch of sys_chdir; FUSE-mounted + * or /proc-virtual workdirs are not supported through this flag (neither + * is a realistic image WorkingDir). + */ + if (args->cwd_guest && args->cwd_guest[0] != '\0') { + path_translation_t tx; + if (path_translate_at(LINUX_AT_FDCWD, args->cwd_guest, PATH_TR_NONE, + &tx) < 0) { + log_error("failed to resolve working directory %s: %s", + args->cwd_guest, strerror(errno)); + goto fail; + } + if (chdir(tx.host_path) < 0) { + log_error("failed to set working directory %s: %s", args->cwd_guest, + strerror(errno)); + goto fail; + } + if (proc_cwd_refresh() < 0) + proc_cwd_invalidate(); + } + /* fork_child / vfork_notify dispatch stays in main() (early return * before reaching elfuse_launch). The fields are kept on * launch_args_t so callers route through one launch struct shape diff --git a/src/core/launch.h b/src/core/launch.h index 0f4e20e5..003b19e2 100644 --- a/src/core/launch.h +++ b/src/core/launch.h @@ -7,33 +7,16 @@ * fresh HVF VM until it exits". It is shared between main() (legacy * positional-ELF CLI) and future launchers such as the OCI run helper. * - * The function owns the guest_t, the vCPU, the GDB stub, the run loop, - * the diagnostic dumps, and guest teardown; it does NOT own the - * elf_path / sysroot / guest_argv heap copies or the sysroot_mount the - * host CLI may have provisioned; those stay with the caller so - * behaviors that need the original CLI argv (proctitle rewriting, - * --create-sysroot detach on exit, host cwd save+restore) remain - * coherent regardless of how the launch was kicked off. + * The function owns the guest_t, the vCPU, the GDB stub, the run loop, the + * diagnostic dumps, and guest teardown; it does NOT own the elf_path / + * sysroot / guest_argv heap copies or the sysroot_mount the host CLI may + * have provisioned. Those stay with the caller so behaviors that need the + * original CLI argv (proctitle rewriting, --create-sysroot detach on exit, + * host cwd save+restore) stay coherent however the launch was kicked off. * - * Lifetime / ownership contract: - * - * - The caller owns every pointer in launch_args_t. elfuse_launch reads - * them and does not free them; const-qualified pointers stay valid - * for the duration of the call. - * - envp may be NULL; the host process environ is used in that case. - * - guest_argv is the string array the guest sees as its argv. - * guest_argv[0] is the guest-visible entrypoint path (what the guest - * reads back via /proc/self/exe and argv[0]); elf_path is the - * resolved host path to that binary. The two differ when path - * translation or a FUSE-materialized temp is involved. - * - elf_host_temp is true when elf_path is a FUSE-materialized temp - * that must be unlinked once guest_bootstrap_prepare has loaded it - * (skipped for Rosetta guests, which reopen the path). The caller - * owns the unlink for any pre-prepare failure; elfuse_launch owns it - * from the prepare call onward. - * - fork_child_fd / vfork_notify_fd are forwarded for fork-child-routed - * launches; main() dispatches the fork-child path before reaching - * elfuse_launch and so passes -1. + * The caller owns every pointer in launch_args_t for the duration of the + * call; elfuse_launch reads but never frees them. Per-field lifetime and + * ownership notes live on the struct members below. */ #pragma once @@ -42,13 +25,15 @@ #include typedef struct { - /* Host filesystem path to the guest ELF (resolved; may be a - * FUSE-materialized temp when elf_host_temp is true). + /* Host path to the guest ELF; may be a FUSE-materialized temp when + * elf_host_temp is set. */ const char *elf_path; - /* True when elf_path is a temp to unlink after guest_bootstrap_prepare - * loads it. + /* elf_path is a FUSE-materialized temp to unlink once + * guest_bootstrap_prepare has loaded it (kept for Rosetta guests, which + * reopen the path). The caller owns the unlink on any pre-prepare + * failure; elfuse_launch owns it from the prepare call onward. */ bool elf_host_temp; @@ -57,8 +42,10 @@ typedef struct { */ const char *sysroot; - /* String array the guest sees as its argv. guest_argv[0] is the - * guest-visible entrypoint path. + /* Argv the guest sees. guest_argv[0] is the guest-visible entrypoint + * path (what the guest reads back via /proc/self/exe and argv[0]); it + * differs from elf_path (the resolved host path) under path translation + * or a FUSE-materialized temp. */ int guest_argc; const char **guest_argv; @@ -69,6 +56,21 @@ typedef struct { */ char **envp; + /* When true, stage uid/gid as the guest identity before bring-up so the + * auxv AT_UID/AT_GID snapshot and getuid()/getgid() agree. When false, + * uid/gid are ignored and the guest runs under the compile-time default + * GUEST_UID/GUEST_GID (0 under fakeroot), NOT the host identity; a + * launcher that wants the host identity must set has_creds and pass + * getuid()/getgid(). + */ + bool has_creds; + uint32_t uid, gid; + + /* Guest-absolute initial working directory. NULL inherits the host + * cwd (the caller may chdir first to control it). + */ + const char *cwd_guest; + /* GDB Remote Serial Protocol port (0 disables the stub) and whether * to halt before the first guest instruction. */ diff --git a/src/main.c b/src/main.c index 9a83db99..4db3fbdc 100644 --- a/src/main.c +++ b/src/main.c @@ -104,6 +104,140 @@ static void free_guest_argv(const char **guest_argv, int guest_argc) free((void *) guest_argv); } +/* Free a guest envp vector produced by build_guest_env. Each entry is a + * heap "KEY=VAL" string owned by us (never a borrowed environ pointer), so + * free every slot then the array. A NULL envp (meaning "use host environ") + * is a no-op. + */ +static void free_envp(char **envp) +{ + if (!envp) + return; + for (char **e = envp; *e; e++) + free(*e); + free(envp); +} + +/* Free the raw --env override array collected during option parsing. These + * strings are distinct from build_guest_env's output (which strdups its own + * copies), so both must be freed. + */ +static void free_env_overrides(char **env_overrides, int n) +{ + if (!env_overrides) + return; + for (int i = 0; i < n; i++) + free(env_overrides[i]); + free(env_overrides); +} + +/* Build the guest environment vector, mirroring `env(1)` semantics. Returns 0 + * and sets *out_envp to either: + * - NULL when no --env/--clear-env was given, meaning "use the host environ + * as-is" (the pre-flag behavior, preserved exactly), or + * - a malloc'd, NULL-terminated char** of strdup'd "KEY=VAL" strings (caller + * frees with free_envp): the base is the host environ, or empty under + * clear_env, and each override replaces a matching KEY= in place or + * appends. "KEY=VAL" sets; a bare "KEY" inherits KEY from the host environ, + * skipped when unset so it can never create an empty-string variable. Returns + * -1 on allocation failure (out_envp untouched). + */ +static int build_guest_env(char *const *overrides, + int n_overrides, + bool clear_env, + char ***out_envp) +{ + if (n_overrides == 0 && !clear_env) { + *out_envp = NULL; + return 0; + } + + extern char **environ; + int cap = 1; /* NULL terminator */ + if (!clear_env) + for (char **e = environ; *e; e++) + cap++; + cap += n_overrides; + + char **envp = (char **) calloc((size_t) cap, sizeof(char *)); + if (!envp) + return -1; + int n = 0; + + if (!clear_env) { + for (char **e = environ; *e; e++) { + envp[n] = strdup(*e); + if (!envp[n]) + goto fail; + n++; + } + } + + for (int i = 0; i < n_overrides; i++) { + const char *ov = overrides[i]; + const char *eq = strchr(ov, '='); + /* Reject an empty variable name ("--env =VAL", or a bare "--env ""): + * eq == ov (or an empty ov) means a zero-length key. setenv(3), whose + * semantics --env mirrors, rejects an empty name, and appending + * "=VAL" verbatim would hand the guest a malformed environ entry the + * dedup scan cannot match. */ + if ((size_t) (eq ? eq - ov : strlen(ov)) == 0) { + log_error("invalid --env entry \"%s\": empty variable name", ov); + goto fail; + } + char *entry; + if (eq) { + entry = strdup(ov); + } else { + const char *val = getenv(ov); + if (!val) + continue; /* bare KEY, unset on host: skip */ + size_t need = strlen(ov) + 1 + strlen(val) + 1; + entry = (char *) malloc(need); + if (entry) + snprintf(entry, need, "%s=%s", ov, val); + } + if (!entry) + goto fail; + + size_t klen = (size_t) (eq ? eq - ov : strlen(ov)); + int found = -1; + for (int j = 0; j < n; j++) { + if (envp[j] && strncmp(envp[j], ov, klen) == 0 && + envp[j][klen] == '=') { + found = j; + break; + } + } + if (found >= 0) { + free(envp[found]); + envp[found] = entry; + } else { + if (n + 1 >= cap) { + int ncap = cap + 4; + char **grown = + (char **) realloc(envp, (size_t) ncap * sizeof(char *)); + if (!grown) { + free(entry); + goto fail; + } + for (int j = cap; j < ncap; j++) + grown[j] = NULL; + envp = grown; + cap = ncap; + } + envp[n++] = entry; + } + } + envp[n] = NULL; + *out_envp = envp; + return 0; + +fail: + free_envp(envp); + return -1; +} + static void cleanup_main_resources(guest_t *g, bool guest_initialized, sysroot_mount_t *sysroot_mount, @@ -124,12 +258,6 @@ static void cleanup_main_resources(guest_t *g, free((void *) sysroot_path); } -/* The embedded shim binary (shim_blob.h, generated by xxd -i from shim.bin) - * is now included by src/core/launch.c, the single site that hands the blob to - * guest_bootstrap_prepare. main() no longer references shim_bin/shim_bin_len - * directly, so the static blob has one object definition site. - */ - /* The infra-reserve layout invariants documented in guest.h are derived from * raw offset constants, so a future edit that grows the pool by shifting one * offset without the others would silently overlap two regions. Enforce them at @@ -242,6 +370,17 @@ int main(int argc, char **argv) int gdb_port = 0; bool gdb_stop_on_entry = false; bool fakeroot = false; + /* Launch flags driven by `elfuse-oci run` (and usable directly). They + * map onto launch_args_t fields; --user overrides the guest identity, + * --workdir sets the guest's initial cwd, --env/--clear-env build the + * guest environment. All are additive: existing flags are unchanged. + */ + bool has_creds = false; + uint32_t uid = 0, gid = 0; + char *workdir = NULL; + char **env_overrides = NULL; + int n_env_overrides = 0, env_cap = 0; + bool clear_env = false; int arg_start = 1; /* 'elfuse rosettad translate ' runs the real Apple rosettad @@ -275,6 +414,8 @@ int main(int argc, char **argv) " [--create-sysroot PATH]\n" " [--no-rosetta] [--fakeroot]\n" " [--gdb PORT] [--gdb-stop-on-entry]\n" + " [--user UID[:GID]] [--workdir DIR]\n" + " [--env KEY=VAL] [--clear-env]\n" " [args...]\n" "\n" "Options:\n" @@ -295,7 +436,17 @@ int main(int argc, char **argv) " --gdb PORT Listen for GDB Remote Serial " "Protocol on PORT\n" " --gdb-stop-on-entry Halt before the first guest " - "instruction\n"); + "instruction\n" + " --user UID[:GID] Run the guest as UID (and GID; " + "defaults to UID). Numeric; elfuse-oci resolves symbolic " + "names\n" + " --workdir DIR Guest-absolute initial working " + "directory (resolved under --sysroot)\n" + " --env KEY=VAL Set a guest environment variable; " + "repeatable. 'KEY' (no '=') inherits from the host environ\n" + " --clear-env Start the guest environment empty " + "(only --env entries apply); default inherits the host " + "environ\n"); return 0; } } @@ -358,6 +509,83 @@ int main(int argc, char **argv) } else if (!strcmp(argv[arg_start], "--gdb-stop-on-entry")) { gdb_stop_on_entry = true; arg_start++; + } else if (!strcmp(argv[arg_start], "--user") && arg_start + 1 < argc) { + /* Numeric UID[:GID]; elfuse-oci resolves symbolic User + * against the image /etc/passwd+group and passes numbers. A bare + * UID sets gid=uid (typical single-user image). + */ + const char *spec = argv[arg_start + 1]; + char *end; + errno = 0; + unsigned long u = strtoul(spec, &end, 10); + if (errno || end == spec || u > UINT32_MAX) { + log_error("invalid --user UID: %s", spec); + goto fail_parse; + } + unsigned long gg = u; + if (*end == ':') { + errno = 0; + char *end2; + gg = strtoul(end + 1, &end2, 10); + if (errno || end2 == end + 1 || *end2 != '\0' || + gg > UINT32_MAX) { + log_error("invalid --user UID:GID: %s", spec); + goto fail_parse; + } + } else if (*end != '\0') { + log_error("invalid --user spec: %s", spec); + goto fail_parse; + } + uid = (uint32_t) u; + gid = (uint32_t) gg; + has_creds = true; + arg_start += 2; + } else if (!strcmp(argv[arg_start], "--workdir") && + arg_start + 1 < argc) { + /* Guest-absolute working directory; elfuse_launch translates it + * against the sysroot and chdirs there. Reject relative paths up + * front: translation would resolve them against the host cwd, + * silently starting the guest outside the intended tree. strdup + * now because runtime_set_process_title clobbers the original + * argv block. + */ + if (argv[arg_start + 1][0] != '/') { + log_error("--workdir requires a guest-absolute path, got %s", + argv[arg_start + 1]); + goto fail_parse; + } + free(workdir); + workdir = strdup(argv[arg_start + 1]); + if (!workdir) { + log_error("out of memory"); + goto fail_parse; + } + arg_start += 2; + } else if (!strcmp(argv[arg_start], "--env") && arg_start + 1 < argc) { + /* "KEY=VAL" sets; "KEY" inherits from the host environ (resolved + * in build_guest_env). strdup now; argv is clobbered later. + */ + if (n_env_overrides == env_cap) { + int ncap = env_cap ? env_cap * 2 : 8; + char **grown = (char **) realloc( + env_overrides, (size_t) ncap * sizeof(char *)); + if (!grown) { + log_error("out of memory"); + goto fail_parse; + } + env_overrides = grown; + env_cap = ncap; + } + env_overrides[n_env_overrides] = strdup(argv[arg_start + 1]); + if (!env_overrides[n_env_overrides]) { + log_error("out of memory"); + goto fail_parse; + } + n_env_overrides++; + arg_start += 2; + } else if (!strcmp(argv[arg_start], "--clear-env")) { + clear_env = true; + arg_start++; } else if (!strcmp(argv[arg_start], "--")) { arg_start++; break; @@ -366,16 +594,17 @@ int main(int argc, char **argv) log_error( "usage: elfuse [--verbose] [--timeout N] " "[--sysroot PATH] [--create-sysroot PATH] [--no-rosetta] " - "[--fakeroot] [--gdb PORT] " - "[--gdb-stop-on-entry] [args...]"); - return 1; + "[--fakeroot] [--gdb PORT] [--gdb-stop-on-entry] " + "[--user UID[:GID]] [--workdir DIR] [--env KEY=VAL] " + "[--clear-env] [args...]"); + goto fail_parse; } } if (sysroot && create_sysroot) { log_error( "use either --sysroot PATH or --create-sysroot PATH, not both"); - return 1; + goto fail_parse; } /* ELFUSE_NO_ROSETTA=1 mirrors --no-rosetta for environments where passing @@ -404,14 +633,14 @@ int main(int argc, char **argv) * internal host reserve. */ if (host_nofile_ensure_capacity() < 0) - return 1; + goto fail_parse; /* Block the vCPU-preemption signals and start the sigwait thread before any * vCPU thread exists, so both the normal path and the fork-child path below * inherit the block on every thread they spawn. */ if (proc_preempt_init() < 0) - return 1; + goto fail_parse; /* Fork-child mode: receive VM state over IPC and run */ if (fork_child_fd >= 0) @@ -422,7 +651,21 @@ int main(int argc, char **argv) log_error( "usage: elfuse [--verbose] [--timeout N] " "[--sysroot PATH] [--create-sysroot PATH] [--no-rosetta] " - "[--fakeroot] [args...]"); + "[--fakeroot] [--user UID[:GID]] [--workdir DIR] [--env KEY=VAL] " + "[--clear-env] [args...]"); + goto fail_parse; + } + + /* Shared unwind for argument-parsing errors past the point where --env / + * --workdir may have allocated: frees those two (the only heap state owned + * before the elf_path/guest_argv copies below) and exits 1. Placed before + * those later declarations so no goto crosses into their scope; post-copy + * paths use the cleanup: label instead. + */ + if (0) { + fail_parse: + free_env_overrides(env_overrides, n_env_overrides); + free(workdir); return 1; } @@ -444,6 +687,8 @@ int main(int argc, char **argv) src_len, LINUX_PATH_MAX - 1, sysroot_src); free(elf_path); free(sysroot_path); + free_env_overrides(env_overrides, n_env_overrides); + free(workdir); return 1; } } @@ -459,6 +704,10 @@ int main(int argc, char **argv) char elf_host_path[LINUX_PATH_MAX]; bool elf_host_temp = false; bool have_host_cwd = (getcwd(host_cwd, sizeof(host_cwd)) != NULL); + /* Declared (and NULL-initialized) before the first `goto fail` so the + * shared cleanup below never frees an uninitialized pointer. + */ + char **envp = NULL; int exit_code; memset(&sysroot_mount, 0, sizeof(sysroot_mount)); if (!elf_path || (have_sysroot && !sysroot_path) || !guest_argv) { @@ -596,22 +845,38 @@ int main(int argc, char **argv) } } + /* Build the guest environment vector late (after the shebang loop and the + * --gdb guard) so only this block's OOM path and the post-launch cleanup + * must free it. With neither --env nor --clear-env given, envp stays NULL + * and elfuse_launch uses the host environ (pre-flag behavior, preserved). + */ + if (n_env_overrides > 0 || clear_env) { + if (build_guest_env(env_overrides, n_env_overrides, clear_env, &envp) < + 0) { + /* build_guest_env has already logged the specific reason (OOM or a + * malformed --env entry). */ + goto fail; + } + } + /* build_guest_env strdups its own copies, so the raw override array and + * its strings are no longer needed. + */ + free_env_overrides(env_overrides, n_env_overrides); + env_overrides = NULL; + n_env_overrides = 0; + /* Rewrite the host-visible process title from the guest entrypoint. This * clobbers the original argv block (already snapshotted into the heap * elf_path / guest_argv above), so it must run before elfuse_launch hands - * control to the guest but after the shebang loop has fixed elf_path. The - * call only touches the original argv and the kernel procname; it does not - * depend on guest_bootstrap_prepare having run, so hoisting it out of the - * bring-up (where it previously sat between prepare and create_vcpu) is - * behavior-preserving. + * control to the guest but after the shebang loop has fixed elf_path. */ runtime_set_process_title(argc, argv, elf_path); - /* Hand the bring-up, run loop, and guest teardown to elfuse_launch. main() - * retains ownership of the original argv (proctitle above), the sysroot - * mount (detached in cleanup_main_resources after the guest exits so the - * mount stays live for the whole run), host cwd, and the heap elf_path / - * sysroot_path / guest_argv copies. + /* Hand bring-up, run loop, and guest teardown to elfuse_launch. main() + * keeps ownership of the original argv (proctitle above), the sysroot mount + * (detached in cleanup_main_resources after the guest exits so it stays + * live for the whole run), host cwd, and the heap elf_path / sysroot_path / + * guest_argv / envp / workdir copies. */ launch_args_t largs = { .elf_path = elf_host_path, @@ -619,6 +884,11 @@ int main(int argc, char **argv) .sysroot = sysroot, .guest_argc = guest_argc, .guest_argv = guest_argv, + .envp = envp, + .has_creds = has_creds, + .uid = uid, + .gid = gid, + .cwd_guest = workdir, .gdb_port = gdb_port, .gdb_stop_on_entry = gdb_stop_on_entry, .timeout_sec = timeout_sec, @@ -640,9 +910,14 @@ int main(int argc, char **argv) /* Single unwind for every exit past the heap-copy allocations. On the * success path elfuse_launch has already run guest_destroy * (guest_initialized never becomes true in main), so this frees the - * caller-owned heap copies, detaches the sysroot mount, restores the - * host cwd, and drops a still-owned FUSE-materialized temp ELF. + * caller-owned heap copies, detaches the sysroot mount, restores the host + * cwd, and drops a still-owned FUSE-materialized temp ELF. A successful + * build_guest_env already freed and reset the override array, so freeing it + * here is then a no-op. */ + free_env_overrides(env_overrides, n_env_overrides); + free_envp(envp); + free(workdir); cleanup_main_resources(&g, guest_initialized, &sysroot_mount, have_host_cwd ? host_cwd : NULL, guest_argv, guest_argc, elf_path, sysroot_path); diff --git a/src/syscall/proc-identity.c b/src/syscall/proc-identity.c index 9af7e005..8215b524 100644 --- a/src/syscall/proc-identity.c +++ b/src/syscall/proc-identity.c @@ -30,6 +30,10 @@ static _Atomic int32_t guest_has_ctty = 1; static _Atomic bool fakeroot_enabled = false; +static _Atomic bool initial_ids_staged = false; +static _Atomic uint32_t initial_uid = GUEST_UID; +static _Atomic uint32_t initial_gid = GUEST_GID; + void proc_set_fakeroot_enabled(bool enabled) { atomic_store(&fakeroot_enabled, enabled); @@ -40,6 +44,13 @@ bool proc_fakeroot_enabled(void) return atomic_load(&fakeroot_enabled); } +void proc_set_initial_ids(uint32_t uid, uint32_t gid) +{ + atomic_store(&initial_uid, uid); + atomic_store(&initial_gid, gid); + atomic_store(&initial_ids_staged, true); +} + void proc_identity_init(void) { guest_pid = 1; @@ -54,6 +65,16 @@ void proc_identity_init(void) gid = 0; } + /* An explicit --user request wins over the defaults and over fakeroot. + * It is staged before init rather than applied afterwards because + * build_linux_stack snapshots these values into auxv AT_UID/AT_GID; a + * post-init override would leave getauxval() disagreeing with getuid(). + */ + if (atomic_load(&initial_ids_staged)) { + uid = atomic_load(&initial_uid); + gid = atomic_load(&initial_gid); + } + emu_uid = uid; emu_euid = uid; emu_suid = uid; diff --git a/src/syscall/proc.h b/src/syscall/proc.h index 4ca8c166..ce9e21ab 100644 --- a/src/syscall/proc.h +++ b/src/syscall/proc.h @@ -108,6 +108,13 @@ bool proc_rosetta_active(void); void proc_set_fakeroot_enabled(bool enabled); bool proc_fakeroot_enabled(void); +/* Stage the initial guest credentials (--user) before proc_init. + * proc_identity_init applies them in place of the GUEST_UID/GUEST_GID + * defaults, so the auxv AT_UID/AT_GID snapshot taken by build_linux_stack + * matches what getuid()/getgid() later report. + */ +void proc_set_initial_ids(uint32_t uid, uint32_t gid); + /* Store the guest command line for /proc/self/cmdline emulation. argv is a * NULL-terminated array of strings. */ From 3879599f2bac3b6d949ad4bac629afc8be715147 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:22:51 +0200 Subject: [PATCH 03/10] Add elfuse-oci store with pull and inspect elfuse-oci is a standalone Go binary that owns the OCI image pipeline; elfuse itself stays a pure Linux syscall-to-Darwin runtime with no OCI commands. This first slice is the acquisition half: an OCI image-layout store plus the pull and inspect commands, built on go-containerregistry ($ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci by default). The store is a spec-shape image layout other tools can read, with a refs.json pin table mapping references to manifest digests. An exclusive flock serializes refs.json/index.json updates so concurrent pulls cannot lose pins, pin persistence syncs the temp file and directory around the rename, and a nil-object refs.json is rejected as corrupt instead of treated as empty. addImage distinguishes genuinely-absent from unreadable descriptors by scanning index membership, so store corruption surfaces rather than duplicating entries. digestFor returns a distinct errNotPulled for a missing ref so later callers can tell "not pulled" from "store broken". Two helpers keep the plumbing in one place: every subcommand opens the store through commonFlags.openResolvedStore (resolve the store path, then open the layout), and store.withLock scopes lock-held sections; pin and addImage wrap their load-modify-save cycles in it so a critical section cannot leak its lock on an error path. pull resolves the requested platform (default linux/arm64) and validates --platform shape up front; inspect prints the manifest and config summary (or --json) and propagates digest/size and writer errors instead of reporting partial output as success. Tests cover the pin table, store locking, error kinds, flag parsing, and the command dispatch, with cranePull as a swappable seam so no test touches the network. --- Makefile | 19 +- cmd/elfuse-oci/commands.go | 59 ++++++ cmd/elfuse-oci/common.go | 177 ++++++++++++++++++ cmd/elfuse-oci/common_test.go | 90 ++++++++++ cmd/elfuse-oci/inspect.go | 82 +++++++++ cmd/elfuse-oci/inspect_test.go | 163 +++++++++++++++++ cmd/elfuse-oci/main.go | 83 +++++++++ cmd/elfuse-oci/main_command_test.go | 84 +++++++++ cmd/elfuse-oci/pull.go | 42 +++++ cmd/elfuse-oci/store.go | 270 ++++++++++++++++++++++++++++ cmd/elfuse-oci/store_test.go | 195 ++++++++++++++++++++ cmd/elfuse-oci/test_helpers_test.go | 157 ++++++++++++++++ go.mod | 17 ++ go.sum | 30 ++++ mk/toolchain.mk | 5 + 15 files changed, 1472 insertions(+), 1 deletion(-) create mode 100644 cmd/elfuse-oci/commands.go create mode 100644 cmd/elfuse-oci/common.go create mode 100644 cmd/elfuse-oci/common_test.go create mode 100644 cmd/elfuse-oci/inspect.go create mode 100644 cmd/elfuse-oci/inspect_test.go create mode 100644 cmd/elfuse-oci/main.go create mode 100644 cmd/elfuse-oci/main_command_test.go create mode 100644 cmd/elfuse-oci/pull.go create mode 100644 cmd/elfuse-oci/store.go create mode 100644 cmd/elfuse-oci/store_test.go create mode 100644 cmd/elfuse-oci/test_helpers_test.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/Makefile b/Makefile index 41634c03..7323716a 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ endef .PHONY: all elfuse .PHONY: gen-syscall-dispatch check-syscall-dispatch -all: elfuse +all: elfuse elfuse-oci ## Regenerate build/dispatch.h from src/syscall/dispatch.tbl gen-syscall-dispatch: @@ -126,6 +126,23 @@ elfuse: $(ELFUSE_BIN) $(ELFUSE_BIN): $(OBJS) | $(BUILD_DIR) $(call link-and-sign,$@,$(OBJS)) +# OCI image CLI (Go). Pure Go, no HVF entitlement or codesigning required, +# so it also builds under Linux for spec-conformance / interop CI. The version +# is stamped from the same VERSION string the C binary uses. +OCI_BIN := $(BUILD_DIR)/elfuse-oci +OCI_SRCS := $(shell find cmd/elfuse-oci -type f -name '*.go' 2>/dev/null) + +.PHONY: elfuse-oci +elfuse-oci: $(OCI_BIN) + +# rm -f first: `go build -o` follows an existing symlink at the output path, +# so a stale build/elfuse-oci symlink would clobber build/elfuse. +$(OCI_BIN): go.mod $(OCI_SRCS) | $(BUILD_DIR) + @echo " GO $@" + $(Q)rm -f $@ + $(Q)cd $(CURDIR) && $(GO) build -ldflags "-X main.version=$(VERSION)" \ + -o $@ ./cmd/elfuse-oci + # Native test binaries (macOS, Hypervisor.framework) ## Build the multi-vCPU HVF validation test (native macOS binary) diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go new file mode 100644 index 00000000..4d59efe1 --- /dev/null +++ b/cmd/elfuse-oci/commands.go @@ -0,0 +1,59 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" +) + +// The subcommands share common-flag parsing (common.go) and the OCI +// image-layout store (store.go). pull and inspect are pure store ops. + +// cmdPull implements `elfuse-oci pull [--store] [--platform] `. +func cmdPull(args []string) error { + cf, ref, err := parsePullArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + return pullImage(cf, s, ref) +} + +func parsePullArgs(args []string) (commonFlags, string, error) { + var cf commonFlags + fs := newCommandFlagSet("pull", &cf) + if err := fs.Parse(args); err != nil { + return cf, "", err + } + ref, err := oneArg("pull", fs.Args(), "") + return cf, ref, err +} + +// cmdInspect implements `elfuse-oci inspect [--store] [--json] `. +func cmdInspect(args []string) error { + cf, asJSON, ref, err := parseInspectArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + return inspect(os.Stdout, s, ref, asJSON) +} + +func parseInspectArgs(args []string) (commonFlags, bool, string, error) { + var cf commonFlags + var asJSON bool + fs := newCommandFlagSet("inspect", &cf) + fs.BoolVar(&asJSON, "json", false, "") + if err := fs.Parse(args); err != nil { + return cf, false, "", err + } + ref, err := oneArg("inspect", fs.Args(), "") + return cf, asJSON, ref, err +} diff --git a/cmd/elfuse-oci/common.go b/cmd/elfuse-oci/common.go new file mode 100644 index 00000000..925f2812 --- /dev/null +++ b/cmd/elfuse-oci/common.go @@ -0,0 +1,177 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "flag" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" +) + +// version is stamped at build time via -ldflags "-X main.version=...". The +// default "dev" is what `go build` without the stamp produces. +var version = "dev" + +// Platform is an OCI platform triple. Variant is optional (e.g. "v8" for +// arm64); empty means "the default variant for this arch". +type Platform struct { + OS string + Arch string + Variant string +} + +func (p Platform) String() string { + if p.Variant != "" { + return p.OS + "/" + p.Arch + "/" + p.Variant + } + return p.OS + "/" + p.Arch +} + +// Set implements flag.Value for --platform. +func (p *Platform) Set(s string) error { + parsed, err := parsePlatform(s) + if err != nil { + return err + } + *p = parsed + return nil +} + +// defaultPlatform is linux/arm64: elfuse runs aarch64-linux guests natively via +// HVF, and x86_64 guests via Rosetta. elfuse-oci targets arm64 by default; +// --platform selects another (e.g. linux/amd64 for an x86_64 image run under +// Rosetta). +var defaultPlatform = Platform{OS: "linux", Arch: "arm64"} + +// parsePlatform parses "os/arch" or "os/arch/variant". The value must have +// exactly two or three slash-separated components, each non-empty: "linux//", +// "/arm64", "linux/arm64/", and "linux/arm64/v8/extra" are all rejected rather +// than riding through to the registry client as a nonsense platform. +func parsePlatform(s string) (Platform, error) { + parts := strings.Split(s, "/") + if len(parts) < 2 || len(parts) > 3 || slices.Contains(parts, "") { + return Platform{}, fmt.Errorf("invalid --platform %q (want os/arch[/variant])", s) + } + if len(parts) == 3 { + return Platform{OS: parts[0], Arch: parts[1], Variant: parts[2]}, nil + } + return Platform{OS: parts[0], Arch: parts[1]}, nil +} + +// defaultStore returns the OCI store directory: $ELFUSE_OCI_STORE if set, +// otherwise ~/.local/share/elfuse/oci. The store is an OCI image-layout +// (blobs/, index.json) plus a ref->digest pin table (see store.go). +func defaultStore() (string, error) { + if s := os.Getenv("ELFUSE_OCI_STORE"); s != "" { + return s, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("no --store given and $HOME unset: %w", err) + } + return filepath.Join(home, ".local", "share", "elfuse", "oci"), nil +} + +// commonFlags holds the flags shared by every subcommand. +type commonFlags struct { + store string + platform Platform + // platformSet records an explicit --platform: run uses it to validate the + // pinned image, while the default platform stays advisory so a ref pulled + // for another platform still runs without re-specifying --platform. + platformSet bool +} + +// platformFlag adapts commonFlags.platform to flag.Value while recording +// that the flag was set explicitly. +type platformFlag struct{ cf *commonFlags } + +func (pf platformFlag) String() string { + if pf.cf == nil { + return "" + } + return pf.cf.platform.String() +} + +func (pf platformFlag) Set(s string) error { + if err := pf.cf.platform.Set(s); err != nil { + return err + } + pf.cf.platformSet = true + return nil +} + +// resolveStore fills cf.store with the default when unset and ensures the +// directory exists. +func (cf *commonFlags) resolveStore() error { + if cf.store == "" { + s, err := defaultStore() + if err != nil { + return err + } + cf.store = s + } + return os.MkdirAll(cf.store, 0o755) +} + +// openResolvedStore is every subcommand's store preamble: resolve the store +// path (defaulting and creating it) and open the layout. +func (cf *commonFlags) openResolvedStore() (*store, error) { + if err := cf.resolveStore(); err != nil { + return nil, err + } + return openStore(cf.store) +} + +// newCommandFlagSet creates a FlagSet whose parse errors are returned (not +// exited on) so main reports them uniformly, while ` -h` and a bad flag +// still print that subcommand's own flag list. The FlagSet's own error line is +// discarded (main prints the returned error); the Usage closure writes the flag +// list straight to stderr so it survives regardless. +func newCommandFlagSet(name string, cf *commonFlags) *flag.FlagSet { + *cf = commonFlags{platform: defaultPlatform} + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: elfuse-oci %s [flags]\n", name) + fs.SetOutput(os.Stderr) + fs.PrintDefaults() + fs.SetOutput(io.Discard) + } + fs.StringVar(&cf.store, "store", "", "OCI store directory (default $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci)") + fs.Var(platformFlag{cf}, "platform", "target platform os/arch[/variant]") + return fs +} + +type repeatedStringFlag []string + +func (f *repeatedStringFlag) String() string { + if f == nil { + return "" + } + return strings.Join(*f, ",") +} + +func (f *repeatedStringFlag) Set(s string) error { + *f = append(*f, s) + return nil +} + +func oneArg(cmd string, args []string, what string) (string, error) { + if len(args) != 1 { + return "", fmt.Errorf("%s: expected one %s, got %d", cmd, what, len(args)) + } + return args[0], nil +} + +func noArgs(cmd string, args []string) error { + if len(args) != 0 { + return fmt.Errorf("%s: takes no argument", cmd) + } + return nil +} diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go new file mode 100644 index 00000000..8bf083ee --- /dev/null +++ b/cmd/elfuse-oci/common_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "reflect" + "testing" +) + +func TestParsePlatform(t *testing.T) { + cases := []struct { + in string + want Platform + wantErr bool + }{ + {"linux/arm64", Platform{OS: "linux", Arch: "arm64"}, false}, + {"linux/amd64/v8", Platform{OS: "linux", Arch: "amd64", Variant: "v8"}, false}, + {"darwin/arm64", Platform{OS: "darwin", Arch: "arm64"}, false}, + {"linux", Platform{}, true}, + {"", Platform{}, true}, + {"linux//", Platform{}, true}, + {"/arm64", Platform{}, true}, + {"linux//v8", Platform{}, true}, + {"linux/arm64/", Platform{}, true}, + {"linux/arm64/v8/extra", Platform{}, true}, + {"//", Platform{}, true}, + } + for _, c := range cases { + got, err := parsePlatform(c.in) + if (err != nil) != c.wantErr { + t.Errorf("parsePlatform(%q): err=%v, wantErr=%v", c.in, err, c.wantErr) + continue + } + if c.wantErr { + continue + } + if !reflect.DeepEqual(got, c.want) { + t.Errorf("parsePlatform(%q): got %+v, want %+v", c.in, got, c.want) + } + if got.String() != c.in { + t.Errorf("Platform(%q).String() = %q, want %q", c.in, got.String(), c.in) + } + } +} + +func TestParsePullArgs(t *testing.T) { + cf, ref, err := parsePullArgs([]string{"--store", "/s", "--platform", "linux/amd64", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if ref != "alpine:3" { + t.Fatalf("ref = %q, want alpine:3", ref) + } + if cf.store != "/s" { + t.Errorf("store = %q, want /s", cf.store) + } + if !reflect.DeepEqual(cf.platform, Platform{OS: "linux", Arch: "amd64"}) { + t.Errorf("platform = %+v, want linux/amd64", cf.platform) + } +} + +func TestParseInspectArgs(t *testing.T) { + _, asJSON, ref, err := parseInspectArgs([]string{"--json", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if !asJSON { + t.Error("asJSON = false, want true") + } + if ref != "alpine:3" { + t.Errorf("ref = %q, want alpine:3", ref) + } +} + +func TestParseCommandFlagErrors(t *testing.T) { + cases := []struct { + name string + err error + }{ + {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, + {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, + {"missing ref", func() error { _, _, err := parsePullArgs(nil); return err }()}, + } + for _, tc := range cases { + if tc.err == nil { + t.Errorf("%s: got nil error", tc.name) + } + } +} diff --git a/cmd/elfuse-oci/inspect.go b/cmd/elfuse-oci/inspect.go new file mode 100644 index 00000000..9679925b --- /dev/null +++ b/cmd/elfuse-oci/inspect.go @@ -0,0 +1,82 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io" +) + +// inspect prints a stored image's manifest + config. With --json the raw config +// JSON is emitted (one object); otherwise a human-readable summary. +func inspect(w io.Writer, s *store, ref string, asJSON bool) error { + img, err := s.image(ref) + if err != nil { + return err + } + d, err := img.Digest() + if err != nil { + return err + } + cfg, err := img.ConfigFile() + if err != nil { + return err + } + cf := cfg.Config + + if asJSON { + b, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + // A failed write (closed pipe, full disk on a redirect) must not + // exit 0: callers would consume truncated JSON as success. + if _, err := fmt.Fprintf(w, "%s\n", b); err != nil { + return fmt.Errorf("inspect: write: %w", err) + } + return nil + } + + // bufio latches the first write error and reports it at Flush, so the + // summary can print unconditionally without an if around every line. + bw := bufio.NewWriter(w) + w = bw + + fmt.Fprintf(w, "%-12s %s\n", "Ref:", ref) + fmt.Fprintf(w, "%-12s %s\n", "Digest:", d) + fmt.Fprintf(w, "%-12s %s/%s\n", "Platform:", cfg.OS, cfg.Architecture) + if !cfg.Created.IsZero() { + fmt.Fprintf(w, "%-12s %s\n", "Created:", cfg.Created.UTC().Format("2006-01-02T15:04:05Z")) + } + fmt.Fprintf(w, "%-12s %v\n", "Entrypoint:", cf.Entrypoint) + fmt.Fprintf(w, "%-12s %v\n", "Cmd:", cf.Cmd) + fmt.Fprintf(w, "%-12s %s\n", "WorkingDir:", cf.WorkingDir) + fmt.Fprintf(w, "%-12s %s\n", "User:", cf.User) + fmt.Fprintf(w, "Env (%d):\n", len(cf.Env)) + for _, e := range cf.Env { + fmt.Fprintf(w, " %s\n", e) + } + layers, err := img.Layers() + if err != nil { + return err + } + fmt.Fprintf(w, "Layers (%d):\n", len(layers)) + for i, l := range layers { + ld, err := l.Digest() + if err != nil { + return fmt.Errorf("inspect: layer %d digest: %w", i, err) + } + ls, err := l.Size() + if err != nil { + return fmt.Errorf("inspect: layer %d size: %w", i, err) + } + fmt.Fprintf(w, " %2d %s %d bytes\n", i, ld, ls) + } + if err := bw.Flush(); err != nil { + return fmt.Errorf("inspect: write: %w", err) + } + return nil +} diff --git a/cmd/elfuse-oci/inspect_test.go b/cmd/elfuse-oci/inspect_test.go new file mode 100644 index 00000000..da7cd956 --- /dev/null +++ b/cmd/elfuse-oci/inspect_test.go @@ -0,0 +1,163 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" +) + +// TestInspectHuman asserts the human-readable summary surfaces the ref, +// platform, and command. Substring checks (not column spacing) keep it robust +// to formatting tweaks. +func TestInspectHuman(t *testing.T) { + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + ref := "local:tiny" + if _, err := s.addImage(ref, tinyImage(t)); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := inspect(&buf, s, ref, false); err != nil { + t.Fatal(err) + } + out := buf.String() + for _, want := range []string{"local:tiny", "Platform:", "linux/arm64", "Cmd:", "/hello"} { + if !strings.Contains(out, want) { + t.Errorf("inspect output missing %q:\n%s", want, out) + } + } +} + +// TestInspectJSON asserts --json emits a valid v1.ConfigFile with the tiny +// image's architecture/os/cmd. +func TestInspectJSON(t *testing.T) { + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + ref := "local:tiny" + if _, err := s.addImage(ref, tinyImage(t)); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := inspect(&buf, s, ref, true); err != nil { + t.Fatal(err) + } + var cf v1.ConfigFile + if err := json.Unmarshal(buf.Bytes(), &cf); err != nil { + t.Fatalf("json unmarshal: %v (raw %q)", err, buf.String()) + } + if cf.Architecture != "arm64" { + t.Errorf("Architecture: got %q, want arm64", cf.Architecture) + } + if cf.OS != "linux" { + t.Errorf("OS: got %q, want linux", cf.OS) + } + if len(cf.Config.Cmd) != 1 || cf.Config.Cmd[0] != "/hello" { + t.Errorf("Cmd: got %v, want [/hello]", cf.Config.Cmd) + } +} + +func imageWithRichConfig(t *testing.T) v1.Image { + t.Helper() + img := tinyImage(t) + cfg, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + cfg.Created = v1.Time{Time: time.Date(2026, 7, 9, 12, 34, 56, 0, time.FixedZone("test", 2*60*60))} + cfg.Config.Entrypoint = []string{"/entry"} + cfg.Config.Cmd = []string{"arg"} + cfg.Config.Env = []string{"A=1", "B=2"} + cfg.Config.WorkingDir = "/work" + cfg.Config.User = "1000:1000" + img, err = mutate.ConfigFile(img, cfg) + if err != nil { + t.Fatal(err) + } + return img +} + +func TestInspectHumanIncludesCreatedEnvAndLayers(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:rich", imageWithRichConfig(t)); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := inspect(&buf, s, "local:rich", false); err != nil { + t.Fatal(err) + } + out := buf.String() + for _, want := range []string{ + "Ref: local:rich", + "Created: 2026-07-09T10:34:56Z", + "Entrypoint: [/entry]", + "Cmd: [arg]", + "WorkingDir: /work", + "User: 1000:1000", + "Env (2):", + "A=1", + "Layers (1):", + } { + if !strings.Contains(out, want) { + t.Fatalf("inspect output missing %q:\n%s", want, out) + } + } +} + +func TestInspectMissingRefAndConfigErrors(t *testing.T) { + s := openTestStore(t) + if err := inspect(&bytes.Buffer{}, s, "local:missing", false); err == nil || !strings.Contains(err.Error(), "not pulled") { + t.Fatalf("inspect missing ref err = %v, want not pulled", err) + } + + img := tinyImage(t) + config, err := img.ConfigName() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tiny", img); err != nil { + t.Fatal(err) + } + if err := os.Remove(blobPath(s.root, config.String())); err != nil { + t.Fatal(err) + } + if err := inspect(&bytes.Buffer{}, s, "local:tiny", false); err == nil { + t.Fatal("inspect with missing config succeeded, want error") + } +} + +// failingWriter fails every write, standing in for a closed pipe or a full +// filesystem behind a redirect. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("sink failed") } + +func TestInspectPropagatesWriteErrors(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:tiny", tinyImage(t)); err != nil { + t.Fatal(err) + } + if err := inspect(failingWriter{}, s, "local:tiny", true); err == nil { + t.Fatal("inspect --json exited clean on a failed write, want error") + } + if err := inspect(failingWriter{}, s, "local:tiny", false); err == nil { + t.Fatal("inspect summary exited clean on a failed write, want error") + } +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go new file mode 100644 index 00000000..d9e77c3a --- /dev/null +++ b/cmd/elfuse-oci/main.go @@ -0,0 +1,83 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +// elfuse-oci is the OCI image CLI for elfuse. +// +// It owns the OCI image pipeline (pull, store, and inspect for now) +// using go-containerregistry. elfuse itself stays a pure Linux +// syscall-to-Darwin runtime with no OCI awareness. +// +// Usage: +// +// elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] +// elfuse-oci inspect [--store DIR] [--json] +// +// is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., +// localhost:5000/foo:tag, or name@sha256:...). The default store is +// $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci. +package main + +import ( + "errors" + "flag" + "fmt" + "os" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "elfuse-oci: %s\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `usage: elfuse-oci [flags] [args...] + +commands: + pull Pull an image reference into the local OCI store + inspect Print a stored image's manifest + config + help Show this help + version Print the elfuse-oci version + +common flags: + --store DIR OCI store directory (default $ELFUSE_OCI_STORE or + ~/.local/share/elfuse/oci) + --platform os/arch[/variant] Target platform (default linux/arm64) +`) +} + +func run(args []string) error { + if len(args) == 0 { + usage() + return fmt.Errorf("no command given") + } + cmd, rest := args[0], args[1:] + // A ` -h`/`--help` makes the subcommand FlagSet print its own flag list + // and return flag.ErrHelp; treat that as success rather than an error. + if err := dispatch(cmd, rest); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + return nil +} + +func dispatch(cmd string, rest []string) error { + switch cmd { + case "help", "-h", "--help": + usage() + return nil + case "version", "-V", "--version": + fmt.Println("elfuse-oci " + version) + return nil + case "pull": + return cmdPull(rest) + case "inspect": + return cmdInspect(rest) + default: + usage() + return fmt.Errorf("unknown command: %s", cmd) + } +} diff --git a/cmd/elfuse-oci/main_command_test.go b/cmd/elfuse-oci/main_command_test.go new file mode 100644 index 00000000..9a8636da --- /dev/null +++ b/cmd/elfuse-oci/main_command_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os/exec" + "strings" + "testing" +) + +func TestRunDispatchHelpVersionAndErrors(t *testing.T) { + stdout, stderr, err := captureOutput(t, func() error { return run([]string{"help"}) }) + if err != nil { + t.Fatalf("run help: %v", err) + } + if stdout != "" { + t.Fatalf("help stdout = %q, want empty", stdout) + } + if !strings.Contains(stderr, "usage: elfuse-oci") || !strings.Contains(stderr, "commands:") { + t.Fatalf("help stderr missing usage:\n%s", stderr) + } + + stdout, stderr, err = captureOutput(t, func() error { return run([]string{"--version"}) }) + if err != nil { + t.Fatalf("run --version: %v", err) + } + if strings.TrimSpace(stdout) != "elfuse-oci "+version { + t.Fatalf("version stdout = %q, want elfuse-oci %s", stdout, version) + } + if stderr != "" { + t.Fatalf("version stderr = %q, want empty", stderr) + } + + _, stderr, err = captureOutput(t, func() error { return run(nil) }) + if err == nil || !strings.Contains(err.Error(), "no command") { + t.Fatalf("run nil err = %v, want no command", err) + } + if !strings.Contains(stderr, "usage: elfuse-oci") { + t.Fatalf("no-arg stderr missing usage:\n%s", stderr) + } + + _, stderr, err = captureOutput(t, func() error { return run([]string{"bogus"}) }) + if err == nil || !strings.Contains(err.Error(), "unknown command: bogus") { + t.Fatalf("run bogus err = %v, want unknown command", err) + } + if !strings.Contains(stderr, "usage: elfuse-oci") { + t.Fatalf("unknown-command stderr missing usage:\n%s", stderr) + } +} + +func TestMainSubprocessExitBehavior(t *testing.T) { + stdout, stderr, err := runMainSubprocess(t, "version") + if err != nil { + t.Fatalf("main version err = %v, stdout=%q stderr=%q", err, stdout, stderr) + } + if strings.TrimSpace(stdout) != "elfuse-oci "+version { + t.Fatalf("main version stdout = %q", stdout) + } + if stderr != "" { + t.Fatalf("main version stderr = %q, want empty", stderr) + } + + stdout, stderr, err = runMainSubprocess(t, "bogus") + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + t.Fatalf("main bogus err = %T %v, want exit 1", err, err) + } + if stdout != "" { + t.Fatalf("main bogus stdout = %q, want empty", stdout) + } + if !strings.Contains(stderr, "elfuse-oci: unknown command: bogus") { + t.Fatalf("main bogus stderr = %q, want formatted error", stderr) + } + + _, stderr, err = runMainSubprocess(t) + exit, ok = err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + t.Fatalf("main no-arg err = %T %v, want exit 1", err, err) + } + if !strings.Contains(stderr, "elfuse-oci: no command given") { + t.Fatalf("main no-arg stderr = %q, want formatted error", stderr) + } +} diff --git a/cmd/elfuse-oci/pull.go b/cmd/elfuse-oci/pull.go new file mode 100644 index 00000000..639716d1 --- /dev/null +++ b/cmd/elfuse-oci/pull.go @@ -0,0 +1,42 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1" +) + +var cranePull = crane.Pull + +// platformOption converts the parsed --platform into a crane option. +func platformOption(cf commonFlags) crane.Option { + p := v1.Platform{ + OS: cf.platform.OS, + Architecture: cf.platform.Arch, + Variant: cf.platform.Variant, + } + return crane.WithPlatform(&p) +} + +// pullImage fetches ref from a registry into the store, pinning ref to the +// image's manifest digest. Re-pulling the same digest is a no-op on the +// layout index (dedup by digest); only the pin table is refreshed. +func pullImage(cf commonFlags, s *store, ref string) error { + img, err := cranePull(ref, platformOption(cf)) + if err != nil { + return fmt.Errorf("pull %s: %w", ref, err) + } + digest, err := s.addImage(ref, img) + if err != nil { + return err + } + // Progress goes to stderr: pullImage also runs on the `run` path, where + // stdout belongs to the guest and callers capture it ($(run ...)). + fmt.Fprintf(os.Stderr, "Pulled %s -> %s\n", ref, digest) + return nil +} diff --git a/cmd/elfuse-oci/store.go b/cmd/elfuse-oci/store.go new file mode 100644 index 00000000..5dc4ab2c --- /dev/null +++ b/cmd/elfuse-oci/store.go @@ -0,0 +1,270 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "syscall" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" +) + +// The store is a real OCI image-layout on disk: an `oci-layout` version +// file, a `blobs/sha256/` tree, and an `index.json` image index (managed by +// go-containerregistry's layout package). Multiple pulled images coexist as +// separate manifest descriptors in the one index, distinguished by digest. +// +// On top of the spec layout we keep a ref->manifest-digest pin table +// (refs.json) so `unpack`/`inspect`/`run` can resolve an image by its +// original reference. This is elfuse-specific lookup metadata; OCI readers can +// still parse the layout through index.json and the content-addressed blobs. +// Keeping it separate lets us preserve the exact pull reference, including +// `docker.io/library/alpine:3` or `name@sha256:...`. + +const ( + ociLayoutFile = `{"imageLayoutVersion":"1.0.0"}` + emptyIndex = `{"schemaVersion":2,"manifests":[]}` +) + +type store struct { + path layout.Path + root string +} + +// openStore ensures the layout scaffolding exists and returns a handle. +// Creating an empty layout (oci-layout + empty index.json + blobs/sha256/) +// here, rather than via layout.Write, lets the first pull go through the same +// Append path as every subsequent one. +func openStore(root string) (*store, error) { + for _, d := range []string{root, filepath.Join(root, "blobs"), filepath.Join(root, "blobs", "sha256")} { + if err := os.MkdirAll(d, 0o755); err != nil { + return nil, err + } + } + if err := writeIfAbsent(filepath.Join(root, "oci-layout"), []byte(ociLayoutFile)); err != nil { + return nil, err + } + if err := writeIfAbsent(filepath.Join(root, "index.json"), []byte(emptyIndex)); err != nil { + return nil, err + } + return &store{path: layout.Path(root), root: root}, nil +} + +func writeIfAbsent(path string, data []byte) error { + if _, err := os.Stat(path); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + return os.WriteFile(path, data, 0o644) +} + +// refPins maps an image reference to its manifest digest ("sha256:..."). +type refPins map[string]string + +func (s *store) loadPins() (refPins, error) { + b, err := os.ReadFile(filepath.Join(s.root, "refs.json")) + if os.IsNotExist(err) { + return refPins{}, nil + } else if err != nil { + return nil, err + } + var p refPins + if err := json.Unmarshal(b, &p); err != nil { + return nil, fmt.Errorf("store: corrupt refs.json: %w", err) + } + if p == nil { + return nil, fmt.Errorf("store: corrupt refs.json: expected object") + } + return p, nil +} + +func (s *store) savePins(p refPins) error { + b, err := json.MarshalIndent(p, "", " ") + if err != nil { + return err + } + // Unique temp name: a fixed name would let two writers clobber each + // other's half-written temp even before the rename race. + tmp, err := os.CreateTemp(s.root, ".refs.json.*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) // no-op once the rename succeeds + if _, err := tmp.Write(b); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(0o644); err != nil { + tmp.Close() + return err + } + // Durability, not just atomicity: rmi's crash-ordering argument (drop the + // pin before dropping the descriptor) only holds if the new refs.json + // cannot revert to an old pin after a crash. Sync the temp file before the + // rename and the directory after it, so the rename is durable once this + // returns. + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmp.Name(), filepath.Join(s.root, "refs.json")); err != nil { + return err + } + return fsyncDir(s.root) +} + +// fsyncDir flushes a directory's entries (renames, unlinks) to stable +// storage. +func fsyncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} + +// lock takes an exclusive advisory flock on /.lock and returns the +// unlock func. It serializes read-modify-write cycles on refs.json and +// index.json across concurrent elfuse-oci processes (parallel pulls, or +// a pull racing an rmi); without it, last-writer-wins on refs.json can drop a +// just-recorded pin. +func (s *store) lock() (func(), error) { + f, err := os.OpenFile(filepath.Join(s.root, ".lock"), os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return nil, fmt.Errorf("store: open lock: %w", err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + f.Close() + return nil, fmt.Errorf("store: lock: %w", err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} + +// withLock runs fn while holding the store lock. Results cross the closure +// boundary by capture; the lock is released before withLock returns, so +// callers can keep post-lock work (reporting, other stores) outside the +// critical section. +func (s *store) withLock(fn func() error) error { + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + return fn() +} + +// pin records ref->digest in the pin table. +func (s *store) pin(ref, digest string) error { + return s.withLock(func() error { return s.pinLocked(ref, digest) }) +} + +// pinLocked is pin's load-modify-save cycle; the caller holds the store lock. +func (s *store) pinLocked(ref, digest string) error { + p, err := s.loadPins() + if err != nil { + return err + } + p[ref] = digest + return s.savePins(p) +} + +// errNotPulled marks the ref-simply-missing case, distinguishing it from +// store corruption or IO failures: `run` auto-pulls only on this error. +var errNotPulled = fmt.Errorf("not pulled") + +// digestFor returns the manifest digest pinned for ref, or an error wrapping +// errNotPulled if the ref has not been pulled into this store. +func (s *store) digestFor(ref string) (string, error) { + p, err := s.loadPins() + if err != nil { + return "", err + } + d, ok := p[ref] + if !ok { + return "", fmt.Errorf("store: %q %w (run `elfuse-oci pull %s` first)", ref, errNotPulled, ref) + } + return d, nil +} + +// addImage appends img to the layout index if its manifest is not already +// present (dedup by digest), and pins ref to that digest. Returns the digest. +// The store lock covers the whole check-append-pin sequence: index.json is +// itself updated by read-modify-write inside the layout package, so two +// concurrent pulls could otherwise duplicate or drop descriptors. +func (s *store) addImage(ref string, img v1.Image) (string, error) { + d, err := img.Digest() + if err != nil { + return "", fmt.Errorf("store: compute manifest digest: %w", err) + } + h, err := v1.NewHash(d.String()) + if err != nil { + return "", err + } + err = s.withLock(func() error { + // Re-pulling the same digest must not append a duplicate descriptor + // to the index. + present, err := s.hasImageLocked(h) + if err != nil { + return fmt.Errorf("store: read layout index: %w", err) + } + if !present { + if err := s.path.AppendImage(img); err != nil { + return fmt.Errorf("store: append image: %w", err) + } + } + return s.pinLocked(ref, d.String()) + }) + if err != nil { + return "", err + } + return d.String(), nil +} + +// hasImageLocked reports whether the layout index already carries a manifest +// descriptor for h. The caller holds the store lock. This is a positive +// membership scan rather than a probe via s.path.Image(h): the layout package +// returns an untyped error for both "not found" and a corrupt or unreadable +// index.json, and treating the latter as "absent" would silently append into +// a broken store, masking the corruption. +func (s *store) hasImageLocked(h v1.Hash) (bool, error) { + ii, err := s.path.ImageIndex() + if err != nil { + return false, err + } + im, err := ii.IndexManifest() + if err != nil { + return false, err + } + for _, desc := range im.Manifests { + if desc.Digest == h { + return true, nil + } + } + return false, nil +} + +// image returns the v1.Image pinned for ref. +func (s *store) image(ref string) (v1.Image, error) { + d, err := s.digestFor(ref) + if err != nil { + return nil, err + } + h, err := v1.NewHash(d) + if err != nil { + return nil, err + } + return s.path.Image(h) +} diff --git a/cmd/elfuse-oci/store_test.go b/cmd/elfuse-oci/store_test.go new file mode 100644 index 00000000..243af730 --- /dev/null +++ b/cmd/elfuse-oci/store_test.go @@ -0,0 +1,195 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// TestDigestForErrorKinds pins the distinction cmdRun's auto-pull relies on: +// a merely-absent ref is errNotPulled (triggers the pull), while a corrupt +// refs.json is a different error that must surface instead of being masked by +// a network pull. +func TestDigestForErrorKinds(t *testing.T) { + s := openTestStore(t) + if _, err := s.digestFor("local:absent"); !errors.Is(err, errNotPulled) { + t.Fatalf("missing ref err = %v, want errNotPulled", err) + } + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte("{corrupt"), 0o644); err != nil { + t.Fatal(err) + } + _, err := s.digestFor("local:absent") + if err == nil || errors.Is(err, errNotPulled) { + t.Fatalf("corrupt refs.json err = %v, must not be errNotPulled", err) + } +} + +// TestAddImageCorruptIndexSurfaces pins that addImage distinguishes "image +// not in the layout" from "layout index unreadable": appending into a corrupt +// store would mask the corruption behind a fresh descriptor. +func TestAddImageCorruptIndexSurfaces(t *testing.T) { + s := openTestStore(t) + if err := os.WriteFile(filepath.Join(s.root, "index.json"), []byte("{corrupt"), 0o644); err != nil { + t.Fatal(err) + } + _, err := s.addImage("local:corrupt", tinyImage(t)) + if err == nil || !strings.Contains(err.Error(), "read layout index") { + t.Fatalf("addImage with corrupt index.json err = %v, want read-layout-index error", err) + } + // The corrupt index must be left as-is for diagnosis, not clobbered by an + // append. + b, rerr := os.ReadFile(filepath.Join(s.root, "index.json")) + if rerr != nil || string(b) != "{corrupt" { + t.Fatalf("index.json after failed addImage = %q, err=%v; want untouched", b, rerr) + } +} + +// TestPinConcurrentWritersKeepAllEntries pins the store-lock behavior: N +// concurrent pin calls (as parallel `pull` processes would issue) must all +// survive into refs.json. Without the flock around the load-modify-save +// cycle, last-writer-wins drops entries. +func TestPinConcurrentWritersKeepAllEntries(t *testing.T) { + s := openTestStore(t) + const n = 16 + var wg sync.WaitGroup + errs := make(chan error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + errs <- s.pin(fmt.Sprintf("local:ref%d", i), fmt.Sprintf("sha256:%064d", i)) + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + pins, err := s.loadPins() + if err != nil { + t.Fatal(err) + } + if len(pins) != n { + t.Fatalf("refs.json has %d pins after %d concurrent writers, want %d", len(pins), n, n) + } +} + +func TestDefaultStoreFromEnvAndResolveStore(t *testing.T) { + want := filepath.Join(t.TempDir(), "store") + t.Setenv("ELFUSE_OCI_STORE", want) + got, err := defaultStore() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("defaultStore = %q, want %q", got, want) + } + + var cf commonFlags + if err := cf.resolveStore(); err != nil { + t.Fatalf("resolveStore: %v", err) + } + if cf.store != want { + t.Fatalf("resolved store = %q, want %q", cf.store, want) + } + if fi, err := os.Stat(want); err != nil || !fi.IsDir() { + t.Fatalf("resolved store dir = %v, err=%v; want directory", fi, err) + } + + fileStore := filepath.Join(t.TempDir(), "store-file") + if err := os.WriteFile(fileStore, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + cf = commonFlags{store: fileStore} + if err := cf.resolveStore(); err == nil { + t.Fatal("resolveStore on file path succeeded, want error") + } + + home := t.TempDir() + t.Setenv("ELFUSE_OCI_STORE", "") + t.Setenv("HOME", home) + got, err = defaultStore() + if err != nil { + t.Fatal(err) + } + want = filepath.Join(home, ".local", "share", "elfuse", "oci") + if got != want { + t.Fatalf("defaultStore without env = %q, want %q", got, want) + } +} + +func TestRepeatedStringFlag(t *testing.T) { + var nilFlag *repeatedStringFlag + if got := nilFlag.String(); got != "" { + t.Fatalf("nil repeatedStringFlag String = %q, want empty", got) + } + + var f repeatedStringFlag + if err := f.Set("A=1"); err != nil { + t.Fatal(err) + } + if err := f.Set("B=2"); err != nil { + t.Fatal(err) + } + if got := f.String(); got != "A=1,B=2" { + t.Fatalf("repeatedStringFlag String = %q, want A=1,B=2", got) + } +} + +func TestOpenStoreAndWriteIfAbsentErrorCases(t *testing.T) { + rootFile := filepath.Join(t.TempDir(), "store-file") + if err := os.WriteFile(rootFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := openStore(rootFile); err == nil { + t.Fatal("openStore on file path succeeded, want error") + } + + p := filepath.Join(t.TempDir(), "existing") + if err := os.WriteFile(p, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeIfAbsent(p, []byte("new")); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if string(b) != "old" { + t.Fatalf("writeIfAbsent overwrote existing file with %q, want old", b) + } +} + +func TestLoadPinsCorruptNullAndPinError(t *testing.T) { + s := openTestStore(t) + for _, tc := range []struct { + name string + data string + want string + }{ + {"malformed", "{", "corrupt refs.json"}, + {"null", "null", "expected object"}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte(tc.data), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.loadPins(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("loadPins err = %v, want %q", err, tc.want) + } + if err := s.pin("local:a", "sha256:"+strings.Repeat("1", 64)); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("pin err = %v, want %q", err, tc.want) + } + }) + } +} diff --git a/cmd/elfuse-oci/test_helpers_test.go b/cmd/elfuse-oci/test_helpers_test.go new file mode 100644 index 00000000..fcb0c7af --- /dev/null +++ b/cmd/elfuse-oci/test_helpers_test.go @@ -0,0 +1,157 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +func TestMain(m *testing.M) { + if raw, ok := os.LookupEnv("ELFUSE_OCI_MAIN_TEST_ARGS"); ok { + var args []string + if raw != "" { + args = strings.Split(raw, "\x00") + } + os.Args = append([]string{os.Args[0]}, args...) + main() + return + } + os.Exit(m.Run()) +} + +func sliceContains(xs []string, want string) bool { + return slices.Contains(xs, want) +} + +func captureOutput(t *testing.T, fn func() error) (string, string, error) { + t.Helper() + + oldStdout, oldStderr := os.Stdout, os.Stderr + stdoutR, stdoutW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + stderrR, stderrW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + + stdoutCh := make(chan string, 1) + stderrCh := make(chan string, 1) + go func() { + b, _ := io.ReadAll(stdoutR) + stdoutCh <- string(b) + }() + go func() { + b, _ := io.ReadAll(stderrR) + stderrCh <- string(b) + }() + + os.Stdout, os.Stderr = stdoutW, stderrW + defer func() { + os.Stdout, os.Stderr = oldStdout, oldStderr + }() + + fnErr := fn() + _ = stdoutW.Close() + _ = stderrW.Close() + stdout := <-stdoutCh + stderr := <-stderrCh + _ = stdoutR.Close() + _ = stderrR.Close() + return stdout, stderr, fnErr +} + +func openTestStore(t *testing.T) *store { + t.Helper() + s, err := openStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return s +} + +// buildImage builds a one-layer in-memory image whose layer content is fixed +// ("hello"="world") but whose Cmd is cmd, so two calls with different cmds +// produce the same layer blob but distinct config/manifest digests. This is +// what TestRmiKeepsSharedBlobs needs to exercise reachability GC: two pinned +// refs sharing one layer, with distinct manifests. +func buildImage(t *testing.T, cmd []string) v1.Image { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + if err := tw.WriteHeader(&tar.Header{Name: "hello", Mode: 0o644, Size: 5, Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("world")); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + // LayerFromOpener calls the opener per read so digest/diffid can be queried + // repeatedly (LayerFromReader is deprecated and single-shot). + opener := func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(buf.Bytes())), nil + } + layer, err := tarball.LayerFromOpener(opener) + if err != nil { + t.Fatal(err) + } + img, err := mutate.AppendLayers(empty.Image, layer) + if err != nil { + t.Fatal(err) + } + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + img, err = mutate.ConfigFile(img, &v1.ConfigFile{ + Architecture: "arm64", + OS: "linux", + Config: v1.Config{Cmd: cmd}, + RootFS: v1.RootFS{Type: "layers", DiffIDs: []v1.Hash{diffID}}, + }) + if err != nil { + t.Fatal(err) + } + return img +} + +// tinyImage is the canonical single-layer offline test image. +func tinyImage(t *testing.T) v1.Image { + t.Helper() + return buildImage(t, []string{"/hello"}) +} + +func blobPath(root, digest string) string { + return filepath.Join(root, "blobs", "sha256", strings.TrimPrefix(digest, "sha256:")) +} + +func runMainSubprocess(t *testing.T, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^$") + cmd.Env = append(os.Environ(), "ELFUSE_OCI_MAIN_TEST_ARGS="+strings.Join(args, "\x00")) + // Buffers instead of pipes: exec.Cmd drains both concurrently, so a child + // filling one stream can't deadlock against a sequential reader of the + // other (the pattern the os/exec docs warn about). + var outB, errB bytes.Buffer + cmd.Stdout = &outB + cmd.Stderr = &errB + waitErr := cmd.Run() + return outB.String(), errB.String(), waitErr +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..d004d0b6 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/sysprog21/elfuse + +go 1.26.4 + +require github.com/google/go-containerregistry v0.21.7 + +require ( + github.com/docker/cli v29.5.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/klauspost/compress v1.18.7 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + gotest.tools/v3 v3.5.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..627c493a --- /dev/null +++ b/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v29.5.3+incompatible h1:nbEFfz774vBwQ5KRYv7c/AghjReqnGISvrRhzjV0evs= +github.com/docker/cli v29.5.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.7 h1:/vPFuVXDjtFREsVArW+0h1CIl5urnOhzei4X2DMW9IU= +github.com/google/go-containerregistry v0.21.7/go.mod h1:kjSbt7/zMsKLWfnHrIvKvhXHUw91jbe9DNjPPJ32gXE= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/mk/toolchain.mk b/mk/toolchain.mk index e0f6be4d..b701e5e4 100644 --- a/mk/toolchain.mk +++ b/mk/toolchain.mk @@ -42,3 +42,8 @@ SHIM_ASFLAGS ?= -arch arm64 # clang-format CLANG_FORMAT ?= clang-format + +# Go toolchain for the OCI image CLI (build/elfuse-oci). It is a +# pure Go program with no HVF dependency, so it builds and runs on Linux CI +# too (for spec-conformance / interop tests). `go` from PATH by default. +GO ?= go From 240b7f18b0b908dff77eae58f04a37f14c586161 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:23:35 +0200 Subject: [PATCH 04/10] Add elfuse-oci unpack of OCI layers Apply a stored image's layers into a rootfs directory, whiteouts and all, so `unpack` (and later `run`) can materialize a filesystem tree from the store without any external tool. Layer application is hardened against hostile archives: extraction is bounded by os.Root so no entry, symlink, or hardlink escapes the destination; parent components are Lstat'd and a symlinked or non-dir intermediate is replaced with a real directory (containerd/Docker behavior); opaque whiteouts clear through a real directory only, are order-independent within a layer, and an invalid bare .wh. entry fails extraction instead of deleting its parent; a directory entry replaces a lower-layer non-directory. File bodies are bounded by the tar header size and short bodies are an error, and permissions are finalized with an explicit chmod so the host umask cannot skew modes. setuid/setgid/sticky bits are re-applied where the host allows it, and degrade gracefully where it does not. An unprivileged chmod that sets setuid/setgid is rejected with EPERM on macOS when the unpacked file's inherited group is one the invoking user is not in (a new file takes its parent directory's group under BSD semantics, e.g. wheel under /tmp, not the tar's root/shadow owner). Since the rootfs is owned by the invoking user those bits could not be honored at runtime there anyway, so they are dropped with a warning naming the lost bit rather than aborting the whole unpack, which is what lets Debian-family images and their shadow suite (chage, passwd, ...) unpack at all. Reported at https://github.com/sysprog21/elfuse/pull/191#issuecomment-5011520776 Unpack stages into a temp sibling directory and renames into the final cache path (keyed by manifest digest under /rootfs/), so a concurrent reader never observes a partial tree and the loser of a rename race adopts the winner's complete one. A failed unpack removes only what it created. Tests drive whiteouts, opaque ordering, parent-symlink replacement, hardlink identity, exact-size reads, mode preservation, the special- bit degrade decision, and the staged-rename semantics over synthetic layer tarballs. --- cmd/elfuse-oci/cache_key.go | 53 +++ cmd/elfuse-oci/commands.go | 43 +- cmd/elfuse-oci/common_test.go | 18 +- cmd/elfuse-oci/main.go | 8 +- cmd/elfuse-oci/store_test.go | 13 + cmd/elfuse-oci/unpack.go | 433 ++++++++++++++++++++ cmd/elfuse-oci/unpack_image_test.go | 597 ++++++++++++++++++++++++++++ cmd/elfuse-oci/unpack_test.go | 417 +++++++++++++++++++ 8 files changed, 1578 insertions(+), 4 deletions(-) create mode 100644 cmd/elfuse-oci/cache_key.go create mode 100644 cmd/elfuse-oci/unpack.go create mode 100644 cmd/elfuse-oci/unpack_image_test.go create mode 100644 cmd/elfuse-oci/unpack_test.go diff --git a/cmd/elfuse-oci/cache_key.go b/cmd/elfuse-oci/cache_key.go new file mode 100644 index 00000000..a59597c8 --- /dev/null +++ b/cmd/elfuse-oci/cache_key.go @@ -0,0 +1,53 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// Store subdirectories holding unpacked caches, keyed by cacheKeyForDigest: +// plain rootfs trees and (darwin) case-sensitive sparsebundle bundles. +const ( + rootfsCacheDirName = "rootfs" + csCacheDirName = "cs" +) + +// legacyCacheNameForRef reproduces the pre-digest cache naming scheme, which +// flattened the ref itself into a single (intentionally lossy) path +// component. New caches are keyed by digest (cacheKeyForDigest); this helper +// survives to document the old layout. +func legacyCacheNameForRef(ref string) string { + return strings.NewReplacer("/", "_", ":", "_", "@", "_").Replace(ref) +} + +// cacheKeyForDigest returns the relative cache key used under rootfs/ and cs/. +// The current store writes sha256 blobs only; keep the algorithm component in +// the path so the layout remains explicit and non-lossy. +func cacheKeyForDigest(digest string) (string, error) { + h, err := v1.NewHash(digest) + if err != nil { + return "", err + } + if h.Algorithm != "sha256" || h.Hex == "" { + return "", fmt.Errorf("unsupported cache digest %q", digest) + } + return filepath.Join(h.Algorithm, h.Hex), nil +} + +func defaultRootfsForDigest(store, digest string) (string, error) { + key, err := cacheKeyForDigest(digest) + if err != nil { + return "", err + } + return filepath.Join(store, rootfsCacheDirName, key), nil +} + +func legacyRootfsForRef(store, ref string) string { + return filepath.Join(store, rootfsCacheDirName, legacyCacheNameForRef(ref)) +} diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index 4d59efe1..125b7471 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -4,11 +4,12 @@ package main import ( + "fmt" "os" ) // The subcommands share common-flag parsing (common.go) and the OCI -// image-layout store (store.go). pull and inspect are pure store ops. +// image-layout store (store.go). pull/unpack/inspect are pure store ops. // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { @@ -33,6 +34,46 @@ func parsePullArgs(args []string) (commonFlags, string, error) { return cf, ref, err } +// cmdUnpack implements `elfuse-oci unpack [--store] [--rootfs DIR] `. +func cmdUnpack(args []string) error { + cf, rootfs, ref, err := parseUnpackArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + digest, err := s.digestFor(ref) + if err != nil { + return err + } + if rootfs == "" { + rootfs, err = defaultRootfsForDigest(cf.store, digest) + if err != nil { + return err + } + } + fmt.Printf("Unpacking %s -> %s\n", ref, rootfs) + if err := unpackImage(s, ref, rootfs); err != nil { + return err + } + fmt.Printf("Unpacked %s\n", ref) + return nil +} + +func parseUnpackArgs(args []string) (commonFlags, string, string, error) { + var cf commonFlags + var rootfs string + fs := newCommandFlagSet("unpack", &cf) + fs.StringVar(&rootfs, "rootfs", "", "") + if err := fs.Parse(args); err != nil { + return cf, "", "", err + } + ref, err := oneArg("unpack", fs.Args(), "") + return cf, rootfs, ref, err +} + // cmdInspect implements `elfuse-oci inspect [--store] [--json] `. func cmdInspect(args []string) error { cf, asJSON, ref, err := parseInspectArgs(args) diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index 8bf083ee..182f852d 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -60,6 +60,22 @@ func TestParsePullArgs(t *testing.T) { } } +func TestParseUnpackArgs(t *testing.T) { + cf, rootfs, ref, err := parseUnpackArgs([]string{"--rootfs=/tmp/rootfs", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if cf.platform != defaultPlatform { + t.Errorf("platform = %+v, want default %+v", cf.platform, defaultPlatform) + } + if rootfs != "/tmp/rootfs" { + t.Errorf("rootfs = %q, want /tmp/rootfs", rootfs) + } + if ref != "alpine:3" { + t.Errorf("ref = %q, want alpine:3", ref) + } +} + func TestParseInspectArgs(t *testing.T) { _, asJSON, ref, err := parseInspectArgs([]string{"--json", "alpine:3"}) if err != nil { @@ -80,7 +96,7 @@ func TestParseCommandFlagErrors(t *testing.T) { }{ {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, - {"missing ref", func() error { _, _, err := parsePullArgs(nil); return err }()}, + {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }()}, } for _, tc := range cases { if tc.err == nil { diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index d9e77c3a..9308cc22 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -3,13 +3,14 @@ // elfuse-oci is the OCI image CLI for elfuse. // -// It owns the OCI image pipeline (pull, store, and inspect for now) -// using go-containerregistry. elfuse itself stays a pure Linux +// It owns the OCI image pipeline (pull, store, inspect, and unpack for +// now) using go-containerregistry. elfuse itself stays a pure Linux // syscall-to-Darwin runtime with no OCI awareness. // // Usage: // // elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] +// elfuse-oci unpack [--store DIR] [--rootfs DIR] // elfuse-oci inspect [--store DIR] [--json] // // is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., @@ -36,6 +37,7 @@ func usage() { commands: pull Pull an image reference into the local OCI store + unpack Unpack a stored image's layers into a rootfs directory inspect Print a stored image's manifest + config help Show this help version Print the elfuse-oci version @@ -74,6 +76,8 @@ func dispatch(cmd string, rest []string) error { return nil case "pull": return cmdPull(rest) + case "unpack": + return cmdUnpack(rest) case "inspect": return cmdInspect(rest) default: diff --git a/cmd/elfuse-oci/store_test.go b/cmd/elfuse-oci/store_test.go index 243af730..9b9073c2 100644 --- a/cmd/elfuse-oci/store_test.go +++ b/cmd/elfuse-oci/store_test.go @@ -193,3 +193,16 @@ func TestLoadPinsCorruptNullAndPinError(t *testing.T) { }) } } + +func TestCacheKeyForDigestRejectsInvalidAndUnsupported(t *testing.T) { + if _, err := cacheKeyForDigest("not-a-digest"); err == nil { + t.Fatal("cacheKeyForDigest accepted malformed digest") + } + if _, err := defaultRootfsForDigest(t.TempDir(), "not-a-digest"); err == nil { + t.Fatal("defaultRootfsForDigest accepted malformed digest") + } + unsupported := "sha512:" + strings.Repeat("1", 128) + if _, err := cacheKeyForDigest(unsupported); err == nil { + t.Fatalf("cacheKeyForDigest(%q) succeeded, want rejection", unsupported) + } +} diff --git a/cmd/elfuse-oci/unpack.go b/cmd/elfuse-oci/unpack.go new file mode 100644 index 00000000..a8cccd63 --- /dev/null +++ b/cmd/elfuse-oci/unpack.go @@ -0,0 +1,433 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// unpackImage extracts every layer of the stored image (base first) into a +// plain directory rootfs. Each layer is a tar stream (crane decompresses gzip +// and zstd transparently via layer.Uncompressed). +// +// Layer application implements the OCI whiteout conventions: +// - `.wh.` in a directory removes `` (from this and lower layers). +// - `.wh..wh..opq` in a directory clears that directory's existing contents +// before the layer's own additions are applied. +// +// Containment uses os.OpenRoot (Go 1.24+): every write is resolved relative +// to the rootfs and may not escape via ".." or a symlink. os.Root forbids +// absolute symlinks, so absolute symlink targets are rewritten to their +// equivalent relative form, which is behavior-preserving because under +// elfuse's --sysroot both forms resolve to the same guest path. +// +// Ownership is not applied here: elfuse runs as the host user and overrides +// identity at runtime via --user, so the rootfs carries only mode bits. +func unpackImage(s *store, ref, dest string) error { + img, err := s.image(ref) + if err != nil { + return err + } + if _, statErr := os.Lstat(dest); statErr == nil { + // An explicit pre-existing --rootfs directory: merge in place and + // never remove it, failed or not; it is not ours to delete. + return unpackInto(img, dest) + } else if !os.IsNotExist(statErr) { + return statErr + } + // The run paths treat the rootfs path's existence as "fully unpacked", so + // a partial tree must never be visible under dest, not even while this + // unpack is still running: a concurrent run probing dest with os.Stat + // would execute against the half-written tree. Unpack into a temp sibling + // (same volume, so the rename cannot degrade to a copy) and atomically + // rename into place on success. + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + tmp, err := os.MkdirTemp(filepath.Dir(dest), filepath.Base(dest)+".tmp-") + if err != nil { + return err + } + // MkdirTemp creates 0o700; the rootfs root must be traversable once + // published. + if err := os.Chmod(tmp, 0o755); err != nil { + os.RemoveAll(tmp) + return err + } + if err := unpackInto(img, tmp); err != nil { + os.RemoveAll(tmp) + return err + } + if err := os.Rename(tmp, dest); err != nil { + os.RemoveAll(tmp) + if _, statErr := os.Lstat(dest); statErr == nil { + // A concurrent unpack of the same image won the rename; its tree + // is complete, so use it. + return nil + } + return err + } + return nil +} + +func unpackInto(img v1.Image, dest string) error { + root, err := os.OpenRoot(dest) + if err != nil { + return fmt.Errorf("unpack: open rootfs %s: %w", dest, err) + } + defer root.Close() + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("unpack: list layers: %w", err) + } + for i, layer := range layers { + if err := applyLayer(root, layer); err != nil { + return fmt.Errorf("unpack: layer %d: %w", i, err) + } + } + return nil +} + +func applyLayer(root *os.Root, layer v1.Layer) error { + r, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("open layer: %w", err) + } + defer r.Close() + // Paths this layer has already created. Whiteouts hide lower-layer + // content only, but tar entry order within a layer is not guaranteed: an + // opaque marker may arrive after the directory's own same-layer children, + // which must survive the clear (Docker's unpacker keeps the same set). + applied := make(map[string]bool) + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("read tar entry: %w", err) + } + if err := applyEntry(root, hdr, tr, applied); err != nil { + return fmt.Errorf("entry %q: %w", hdr.Name, err) + } + } +} + +// whiteoutPrefix is the OCI/Docker whiteout marker prefix. +const whiteoutPrefix = ".wh." + +// opaqueMarker is the opaque-directory whiteout marker: a directory's +// existing children are hidden before the layer's own additions apply. +const opaqueMarker = whiteoutPrefix + ".wh..opq" + +// applyEntry applies one tar header to the rootfs. applied tracks the paths +// the current layer has created so far, so an opaque whiteout arriving after +// its directory's same-layer children does not delete them. +func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string]bool) error { + name := filepath.Clean(hdr.Name) + if strings.HasPrefix(name, "../") || name == ".." { + return fmt.Errorf("unsafe entry path %q", hdr.Name) + } + if name == "." || name == "/" { + // The layer root itself; nothing to create. + return nil + } + + base := filepath.Base(name) + if base == opaqueMarker { + return clearDirectory(root, filepath.Dir(name), applied) + } + if trimmed, ok := strings.CutPrefix(base, whiteoutPrefix); ok { + // A bare ".wh." would leave an empty target and Join(dir, "") is the + // directory itself; a malformed layer must fail, not delete its + // containing directory. + if trimmed == "" { + return fmt.Errorf("invalid whiteout entry %q", hdr.Name) + } + target := filepath.Join(filepath.Dir(name), trimmed) + return root.RemoveAll(target) + } + applied[name] = true + + // hdr.FileInfo().Mode() maps the tar header's unix mode bits to os.FileMode + // with the special bits (ModeSetuid/Setgid/Sticky) at os.FileMode's high + // positions, not the raw unix positions. os.Root.MkdirAll/OpenFile reject + // any non-permission bits, so split perm (0o777) from special bits and + // re-apply special bits via Chmod (which syscallMode maps to the syscall). + mode := hdr.FileInfo().Mode() + perm := mode.Perm() + special := mode & (os.ModeSetuid | os.ModeSetgid | os.ModeSticky) + switch hdr.Typeflag { + case tar.TypeDir: + return mkdirAll(root, name, perm, special) + case tar.TypeReg, tar.TypeRegA: + if err := ensureParent(root, name); err != nil { + return err + } + // Remove any prior entry (file, symlink, dir remnant) so we never + // write through a symlink planted by a lower layer. + _ = root.RemoveAll(name) + f, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + // Bound the copy to the header's declared size and require exactly + // that many bytes. The tar reader normally guarantees both; enforcing + // them here keeps the entry-size contract out of the caller's hands. + n, err := io.Copy(f, io.LimitReader(r, hdr.Size)) + if err != nil { + f.Close() + return err + } + if n != hdr.Size { + f.Close() + return fmt.Errorf("entry body: read %d bytes, want %d", n, hdr.Size) + } + if err := f.Close(); err != nil { + return err + } + return applyMode(root, name, perm, special) + case tar.TypeSymlink: + if err := ensureParent(root, name); err != nil { + return err + } + return makeSymlink(root, name, hdr.Linkname, mode) + case tar.TypeLink: + if err := ensureParent(root, name); err != nil { + return err + } + _ = root.RemoveAll(name) + return root.Link(filepath.Clean(hdr.Linkname), name) + case tar.TypeChar, tar.TypeBlock, tar.TypeFifo: + return fmt.Errorf("unsupported special file type %s", tarTypeName(hdr.Typeflag)) + default: + return fmt.Errorf("unsupported tar type %d", hdr.Typeflag) + } +} + +func tarTypeName(t byte) string { + switch t { + case tar.TypeChar: + return "char" + case tar.TypeBlock: + return "block" + case tar.TypeFifo: + return "fifo" + default: + return fmt.Sprintf("%d", t) + } +} + +// makeSymlink creates a symlink at name pointing to target. Absolute targets +// are rewritten to their equivalent relative form so os.Root accepts them +// (it rejects absolute symlinks); the relative form resolves to the same +// guest path under --sysroot, so this is behavior-preserving. +// +// Both name and target are guest paths. Clean an absolute target as a guest path +// first, then strip the leading "/" to make it rootfs-relative and compute the +// link-relative form with filepath.Rel (which needs both sides in the same +// form). +func makeSymlink(root *os.Root, name, target string, mode os.FileMode) error { + if err := ensureParent(root, name); err != nil { + return err + } + _ = root.RemoveAll(name) + if filepath.IsAbs(target) { + tgt := strings.TrimPrefix(filepath.Clean(target), string(filepath.Separator)) + if tgt == "" { + tgt = "." + } + rel, err := filepath.Rel(filepath.Dir(name), tgt) + if err != nil { + return fmt.Errorf("rewrite absolute symlink %q: %w", target, err) + } + target = rel + } + if err := root.Symlink(target, name); err != nil { + return err + } + // Symlink mode is not portable to set across platforms; ignore mode. + _ = mode + return nil +} + +// ensureParent creates name's missing parent directories with the 0o755 +// default. It never chmods: a parent that already exists may carry an exact +// mode from its own tar entry, which must not be reset to the default here. +// +// The walk Lstats every intermediate component instead of calling MkdirAll: +// a lower layer may have planted a symlink (or plain file) where this entry +// needs a directory, and MkdirAll would resolve through it, silently landing +// the entry in whatever the link points at; containment via os.Root still +// holds, but the file ends up in the wrong directory while the entry's own +// path stays unresolved. Replace any such component with a real directory, +// matching containerd's and Docker's unpackers. +func ensureParent(root *os.Root, name string) error { + dir := filepath.Dir(name) + if dir == "." { + return nil + } + cur := "" + for part := range strings.SplitSeq(dir, string(filepath.Separator)) { + cur = filepath.Join(cur, part) + fi, err := root.Lstat(cur) + if err == nil { + if fi.IsDir() { + continue + } + if err := root.RemoveAll(cur); err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + if err := root.Mkdir(cur, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + return nil +} + +// mkdirAll creates a directory entry and any missing parents, then finalizes +// the entry's own mode. os.Root.Mkdir rejects modes that carry +// setuid/setgid/sticky bits ("unsupported file mode"), so create with the +// permission bits only and finalize via applyMode. Parents go through +// ensureParent so lower-layer symlinks on the path are replaced, not +// traversed. +func mkdirAll(root *os.Root, name string, perm, special os.FileMode) error { + if err := ensureParent(root, name); err != nil { + return err + } + // A lower layer may have left a non-directory here, typically a symlink, + // which Mkdir would otherwise resolve through, silently handing this + // layer's children to whatever the link points at. Replace it with a real + // directory, mirroring the RemoveAll the regular-file path does. + if fi, err := root.Lstat(name); err == nil && !fi.IsDir() { + if err := root.RemoveAll(name); err != nil { + return err + } + } + if err := root.Mkdir(name, perm); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + return applyMode(root, name, perm, special) +} + +// applyMode finalizes a just-created entry's mode to exactly perm|special. +// The chmod is unconditional: creation modes passed to os.Root.OpenFile and +// MkdirAll are masked by the process umask, so a restrictive host umask +// (e.g. 0077) would otherwise silently corrupt layer permissions. It also +// re-applies setuid/setgid/sticky, which os.Root creation methods reject at +// create time; Chmod -> syscallMode maps os.FileMode's high special-bit flags +// to the corresponding syscall bits. +// +// When the host cannot set the special bits, degrade to the plain permission +// bits rather than aborting the whole unpack. An unprivileged chmod that sets +// setuid/setgid is rejected with EPERM on macOS when the unpacked file's group +// is one the invoking user is not a member of: a new file inherits its parent +// directory's group (BSD semantics), e.g. wheel under /tmp, not the tar's +// root/shadow owner. The rootfs is owned by the invoking user, so these bits +// could not be honored at runtime on such a host regardless. Debian-family +// images (their shadow suite: chage, passwd, ...) would otherwise fail to +// unpack entirely. +func applyMode(root *os.Root, name string, perm, special os.FileMode) error { + err := root.Chmod(name, perm|special) + if shouldDropSpecial(err, special) { + fmt.Fprintf(os.Stderr, + "elfuse-oci: unpack: dropped %s on %q (unprivileged host)\n", + specialBitNames(special), name) + return root.Chmod(name, perm) + } + return err +} + +// specialBitNames returns a "/"-joined list of the special mode bits present in +// mode (setuid, setgid, sticky), so the drop diagnostic names the bit actually +// lost instead of assuming setuid/setgid. +func specialBitNames(mode os.FileMode) string { + var names []string + if mode&os.ModeSetuid != 0 { + names = append(names, "setuid") + } + if mode&os.ModeSetgid != 0 { + names = append(names, "setgid") + } + if mode&os.ModeSticky != 0 { + names = append(names, "sticky") + } + return strings.Join(names, "/") +} + +// shouldDropSpecial reports whether a failed mode-finalizing chmod should be +// retried without the special bits. Only a permission error qualifies, and +// only when special bits were actually requested, so a genuine chmod failure +// (a permission error with no special bits, or any non-permission error) still +// surfaces to the caller unchanged. +func shouldDropSpecial(err error, special os.FileMode) bool { + return err != nil && special != 0 && errors.Is(err, os.ErrPermission) +} + +// clearDirectory removes the existing children of dir (opaque whiteout), +// keeping entries the current layer itself created: opaque markers hide +// lower-layer content, and a marker ordered after its directory's same-layer +// additions must not wipe them. +func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { + fi, err := root.Lstat(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + // The marker's directory may be a symlink (or other non-dir) planted by a + // lower layer. Reading through it would clear the link target's contents + // (files the image author never whited out). Replace it with a real + // empty directory instead: the opaque marker hides all lower content + // under this name anyway, mirroring the non-dir replacement mkdirAll and + // the regular-file path perform. + if !fi.IsDir() { + if applied[dir] { + // This layer created the non-dir itself; there is no lower + // content beneath it for the marker to hide. + return nil + } + if err := root.RemoveAll(dir); err != nil { + return err + } + return root.Mkdir(dir, 0o755) + } + d, err := root.Open(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + entries, err := d.ReadDir(-1) + d.Close() + if err != nil { + return err + } + for _, e := range entries { + child := filepath.Join(dir, e.Name()) + if applied[child] { + continue + } + if err := root.RemoveAll(child); err != nil { + return err + } + } + return nil +} diff --git a/cmd/elfuse-oci/unpack_image_test.go b/cmd/elfuse-oci/unpack_image_test.go new file mode 100644 index 00000000..a6cfdde7 --- /dev/null +++ b/cmd/elfuse-oci/unpack_image_test.go @@ -0,0 +1,597 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type tarEntry struct { + header tar.Header + body string +} + +func testTarLayer(t *testing.T, entries ...tarEntry) v1.Layer { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, e := range entries { + h := e.header + if h.Typeflag == tar.TypeReg || h.Typeflag == tar.TypeRegA { + h.Size = int64(len(e.body)) + } + if err := tw.WriteHeader(&h); err != nil { + t.Fatal(err) + } + if h.Size > 0 { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + opener := func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(buf.Bytes())), nil + } + layer, err := tarball.LayerFromOpener(opener) + if err != nil { + t.Fatal(err) + } + return layer +} + +func testImageWithLayers(t *testing.T, layers ...v1.Layer) v1.Image { + t.Helper() + img, err := mutate.AppendLayers(empty.Image, layers...) + if err != nil { + t.Fatal(err) + } + diffIDs := make([]v1.Hash, 0, len(layers)) + for _, layer := range layers { + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + diffIDs = append(diffIDs, diffID) + } + img, err = mutate.ConfigFile(img, &v1.ConfigFile{ + Architecture: "arm64", + OS: "linux", + Config: v1.Config{Cmd: []string{"/bin/sh"}}, + RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, + }) + if err != nil { + t.Fatal(err) + } + return img +} + +func TestUnpackImageAppliesLayersInOrder(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("etc", 0o755)}, + tarEntry{header: regHeader("etc/keep", 0o644, 0), body: "lower"}, + tarEntry{header: regHeader("etc/gone", 0o644, 0), body: "gone"}, + tarEntry{header: dirHeader("opt", 0o755)}, + tarEntry{header: regHeader("opt/lower", 0o644, 0), body: "hidden"}, + tarEntry{header: dirHeader("bin", 0o755)}, + tarEntry{header: regHeader("bin/busybox", 0o755, 0), body: "busy"}, + tarEntry{header: symHeader("bin/sh", "/bin/busybox")}, + ) + upper := testTarLayer(t, + tarEntry{header: regHeader("etc/keep", 0o600, 0), body: "upper"}, + tarEntry{header: tar.Header{Name: "etc/.wh.gone", Typeflag: tar.TypeReg, Mode: 0o644}}, + tarEntry{header: tar.Header{Name: "opt/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + tarEntry{header: regHeader("opt/new", 0o644, 0), body: "new"}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:layered", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:layered", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if b, err := os.ReadFile(filepath.Join(dest, "etc", "keep")); err != nil || string(b) != "upper" { + t.Fatalf("etc/keep = %q, err=%v; want upper", b, err) + } + if fi, err := os.Stat(filepath.Join(dest, "etc", "keep")); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("etc/keep mode = %v, err=%v; want 0600", fi, err) + } + if _, err := os.Stat(filepath.Join(dest, "etc", "gone")); !os.IsNotExist(err) { + t.Fatalf("whiteout target etc/gone = %v, want IsNotExist", err) + } + if _, err := os.Stat(filepath.Join(dest, "opt", "lower")); !os.IsNotExist(err) { + t.Fatalf("opaque-hidden opt/lower = %v, want IsNotExist", err) + } + if b, err := os.ReadFile(filepath.Join(dest, "opt", "new")); err != nil || string(b) != "new" { + t.Fatalf("opt/new = %q, err=%v; want new", b, err) + } + target, err := os.Readlink(filepath.Join(dest, "bin", "sh")) + if err != nil { + t.Fatal(err) + } + if target != "busybox" { + t.Fatalf("bin/sh target = %q, want busybox", target) + } +} + +// TestUnpackImageCleansUpPartialRootfs pins that a failed unpack never leaves +// anything at dest: the run paths infer "unpacked" from the path's existence, +// so a partial tree must not survive (or even be transiently visible under +// dest) to be executed by a later or concurrent run. The temp staging +// directory must not leak either. A pre-existing dest (explicit --rootfs) must +// be preserved. +func TestUnpackImageCleansUpPartialRootfs(t *testing.T) { + good := testTarLayer(t, tarEntry{header: regHeader("ok", 0o644, 0), body: "x"}) + bad := testTarLayer(t, + tarEntry{header: tar.Header{Name: "dev/fifo", Typeflag: tar.TypeFifo, Mode: 0o644}}) + + s := openTestStore(t) + if _, err := s.addImage("local:partial", testImageWithLayers(t, good, bad)); err != nil { + t.Fatal(err) + } + + parent := t.TempDir() + dest := filepath.Join(parent, "rootfs") + if err := unpackImage(s, "local:partial", dest); err == nil { + t.Fatal("unpackImage succeeded, want failure on fifo entry") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("partial rootfs still present after failed unpack: err=%v", err) + } + entries, err := os.ReadDir(parent) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("failed unpack left litter next to dest: %v", entries) + } + + pre := t.TempDir() + sentinel := filepath.Join(pre, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + if err := unpackImage(s, "local:partial", pre); err == nil { + t.Fatal("unpackImage succeeded, want failure on fifo entry") + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("pre-existing rootfs dir was deleted on failed unpack: %v", err) + } +} + +// TestUnpackImagePreexistingDestUnpacksInPlace pins the explicit --rootfs +// contract: a destination that already exists is merged into in place rather +// than staged-and-renamed, so files the caller already put there survive a +// successful unpack. +func TestUnpackImagePreexistingDestUnpacksInPlace(t *testing.T) { + layer := testTarLayer(t, tarEntry{header: regHeader("ok", 0o644, 0), body: "x"}) + s := openTestStore(t) + if _, err := s.addImage("local:merge", testImageWithLayers(t, layer)); err != nil { + t.Fatal(err) + } + + dest := t.TempDir() + sentinel := filepath.Join(dest, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + if err := unpackImage(s, "local:merge", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + if b, err := os.ReadFile(sentinel); err != nil || string(b) != "keep" { + t.Fatalf("sentinel = %q, err=%v; want preserved by in-place unpack", b, err) + } + if b, err := os.ReadFile(filepath.Join(dest, "ok")); err != nil || string(b) != "x" { + t.Fatalf("ok = %q, err=%v; want unpacked", b, err) + } +} + +func TestApplyLayerErrors(t *testing.T) { + root, _ := newRoot(t) + if err := applyLayer(root, fakeLayer{uncompressedErr: errors.New("open failed")}); err == nil || + !strings.Contains(err.Error(), "open layer") { + t.Fatalf("applyLayer open err = %v, want open layer error", err) + } + if err := applyLayer(root, fakeLayer{uncompressed: "not a tar archive"}); err == nil || + !strings.Contains(err.Error(), "read tar entry") { + t.Fatalf("applyLayer corrupt tar err = %v, want read tar entry error", err) + } +} + +func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { + root, dir := newRoot(t) + for _, name := range []string{".", "/"} { + h := regHeader(name, 0o644, 0) + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("applyEntry root no-op %q: %v", name, err) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("root no-op entries = %v, want empty root", entries) + } + + hardlink := linkHeader("escape-link", "../outside") + if err := applyEntry(root, &hardlink, strings.NewReader(""), map[string]bool{}); err == nil { + t.Fatal("applyEntry accepted hardlink target escaping root") + } + + outside := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(outside, []byte("outside"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(dir, "replace-me")); err != nil { + t.Fatal(err) + } + file := regHeader("replace-me", 0o644, int64(len("inside"))) + if err := applyEntry(root, &file, strings.NewReader("inside"), map[string]bool{}); err != nil { + t.Fatalf("applyEntry replacing symlink: %v", err) + } + if b, err := os.ReadFile(outside); err != nil || string(b) != "outside" { + t.Fatalf("outside target = %q, err=%v; want unchanged", b, err) + } + if li, err := os.Lstat(filepath.Join(dir, "replace-me")); err != nil || li.Mode()&os.ModeSymlink != 0 { + t.Fatalf("replace-me mode = %v, err=%v; want regular file", li, err) + } + if b, err := os.ReadFile(filepath.Join(dir, "replace-me")); err != nil || string(b) != "inside" { + t.Fatalf("replace-me content = %q, err=%v; want inside", b, err) + } +} + +func TestApplyEntryAdditionalErrorBranches(t *testing.T) { + t.Run("unsupported tar type", func(t *testing.T) { + root, _ := newRoot(t) + h := tar.Header{Name: "weird", Typeflag: 'x'} + err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "unsupported tar type") || !strings.Contains(err.Error(), tarTypeName('x')) { + t.Fatalf("unsupported type err = %v, want tar type error", err) + } + }) + + t.Run("absolute symlink to root", func(t *testing.T) { + root, dir := newRoot(t) + h := symHeader("usr/root-link", "/") + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("applyEntry symlink to root: %v", err) + } + target, err := os.Readlink(filepath.Join(dir, "usr", "root-link")) + if err != nil { + t.Fatal(err) + } + if target != ".." { + t.Fatalf("root symlink target = %q, want ..", target) + } + }) + + t.Run("parent path is regular file", func(t *testing.T) { + root, dir := newRoot(t) + parent := regHeader("parent", 0o644, 0) + if err := applyEntry(root, &parent, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatal(err) + } + // A non-directory on the parent path is replaced with a real + // directory (containerd/Docker behavior), not an error: layers may + // legitimately turn a lower layer's file into a directory. + child := regHeader("parent/child", 0o644, 0) + if err := applyEntry(root, &child, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("applyEntry child under regular-file parent: %v, want file replaced by directory", err) + } + fi, err := os.Lstat(filepath.Join(dir, "parent")) + if err != nil || !fi.IsDir() { + t.Fatalf("parent = %v, err=%v; want a real directory", fi, err) + } + if _, err := os.Stat(filepath.Join(dir, "parent", "child")); err != nil { + t.Fatalf("parent/child: %v, want created", err) + } + }) + + t.Run("opaque missing directory is no-op", func(t *testing.T) { + root, _ := newRoot(t) + h := tar.Header{Name: "missing/.wh..wh..opq", Typeflag: tar.TypeReg} + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("opaque missing dir: %v", err) + } + }) + + t.Run("opaque marker under lower-layer regular file replaces it", func(t *testing.T) { + root, dir := newRoot(t) + file := regHeader("notdir", 0o644, 0) + if err := applyEntry(root, &file, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatal(err) + } + // The opaque marker hides all lower content under the name, so a + // lower layer's non-directory there is replaced with an empty real + // directory rather than cleared through or rejected. + h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} + if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + t.Fatalf("opaque marker under regular file: %v, want file replaced by empty directory", err) + } + fi, err := os.Lstat(filepath.Join(dir, "notdir")) + if err != nil || !fi.IsDir() { + t.Fatalf("notdir = %v, err=%v; want a real directory", fi, err) + } + }) + + t.Run("opaque marker under same-layer regular file is a no-op", func(t *testing.T) { + root, dir := newRoot(t) + file := regHeader("notdir", 0o644, 0) + applied := map[string]bool{} + if err := applyEntry(root, &file, strings.NewReader(""), applied); err != nil { + t.Fatal(err) + } + h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} + if err := applyEntry(root, &h, strings.NewReader(""), applied); err != nil { + t.Fatalf("opaque marker under same-layer file: %v, want no-op", err) + } + fi, err := os.Lstat(filepath.Join(dir, "notdir")) + if err != nil || !fi.Mode().IsRegular() { + t.Fatalf("notdir = %v, err=%v; want the same-layer file kept", fi, err) + } + }) +} + +type fakeLayer struct { + uncompressed string + uncompressedErr error +} + +func (f fakeLayer) Digest() (v1.Hash, error) { + return v1.Hash{Algorithm: "sha256", Hex: strings.Repeat("1", 64)}, nil +} + +func (f fakeLayer) DiffID() (v1.Hash, error) { + return v1.Hash{Algorithm: "sha256", Hex: strings.Repeat("2", 64)}, nil +} + +func (f fakeLayer) Compressed() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("")), nil +} + +func (f fakeLayer) Uncompressed() (io.ReadCloser, error) { + if f.uncompressedErr != nil { + return nil, f.uncompressedErr + } + return io.NopCloser(strings.NewReader(f.uncompressed)), nil +} + +func (f fakeLayer) Size() (int64, error) { + return int64(len(f.uncompressed)), nil +} + +func (f fakeLayer) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + +// TestUnpackOpaqueAfterSameLayerChildren pins the whiteout scoping rule: +// opaque markers hide lower-layer content only. Tar entry order within a +// layer is not guaranteed, so a marker ordered after its directory's own +// same-layer additions must clear the lower content yet keep the additions. +func TestUnpackOpaqueAfterSameLayerChildren(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("opt", 0o755)}, + tarEntry{header: regHeader("opt/lower", 0o644, 0), body: "hidden"}, + ) + upper := testTarLayer(t, + tarEntry{header: regHeader("opt/new", 0o644, 0), body: "new"}, + tarEntry{header: tar.Header{Name: "opt/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:opq-late", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:opq-late", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if _, err := os.Stat(filepath.Join(dest, "opt", "lower")); !os.IsNotExist(err) { + t.Fatalf("opaque-hidden opt/lower = %v, want IsNotExist", err) + } + if b, err := os.ReadFile(filepath.Join(dest, "opt", "new")); err != nil || string(b) != "new" { + t.Fatalf("opt/new = %q, err=%v; want same-layer addition to survive the late marker", b, err) + } +} + +// TestUnpackRejectsBareWhiteout pins that a malformed ".wh." entry with no +// target suffix fails extraction instead of resolving to Join(dir, "") and +// deleting the containing directory. +func TestUnpackRejectsBareWhiteout(t *testing.T) { + dest := t.TempDir() + if err := os.MkdirAll(filepath.Join(dest, "opt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "opt", "keep"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(dest) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + hdr := tar.Header{Name: "opt/.wh.", Typeflag: tar.TypeReg, Mode: 0o644} + if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + t.Fatal("bare .wh. entry applied, want invalid-whiteout error") + } + if _, err := os.Stat(filepath.Join(dest, "opt", "keep")); err != nil { + t.Fatalf("opt/keep after rejected whiteout: %v, want untouched", err) + } +} + +// TestUnpackDirReplacesLowerSymlink pins that a directory entry replaces a +// lower layer's symlink at the same path. Without the replacement, MkdirAll +// resolves through the link and the layer's children land in the link target, +// materializing a different filesystem tree. +func TestUnpackDirReplacesLowerSymlink(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("real", 0o755)}, + tarEntry{header: symHeader("dir", "/real")}, + ) + upper := testTarLayer(t, + tarEntry{header: dirHeader("dir", 0o755)}, + tarEntry{header: regHeader("dir/f", 0o644, 0), body: "payload"}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:dir-over-link", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:dir-over-link", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + fi, err := os.Lstat(filepath.Join(dest, "dir")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("dir mode = %v, want a real directory replacing the lower symlink", fi.Mode()) + } + if b, err := os.ReadFile(filepath.Join(dest, "dir", "f")); err != nil || string(b) != "payload" { + t.Fatalf("dir/f = %q, err=%v; want payload", b, err) + } + if _, err := os.Stat(filepath.Join(dest, "real", "f")); !os.IsNotExist(err) { + t.Fatalf("real/f = %v, want IsNotExist (children must not leak through the lower symlink)", err) + } +} + +// TestUnpackParentSymlinkReplaced pins that an entry whose parent path +// component is a lower layer's symlink lands under a real directory at that +// name, not inside the link's target: MkdirAll-style parent creation would +// resolve through the link and write real/sub/f while leaving link/sub/f +// unresolved. +func TestUnpackParentSymlinkReplaced(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("real", 0o755)}, + tarEntry{header: symHeader("link", "/real")}, + ) + upper := testTarLayer(t, + tarEntry{header: regHeader("link/sub/f", 0o644, 0), body: "payload"}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:parent-link", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:parent-link", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + fi, err := os.Lstat(filepath.Join(dest, "link")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("link mode = %v, want a real directory replacing the lower symlink", fi.Mode()) + } + if b, err := os.ReadFile(filepath.Join(dest, "link", "sub", "f")); err != nil || string(b) != "payload" { + t.Fatalf("link/sub/f = %q, err=%v; want payload", b, err) + } + if _, err := os.Stat(filepath.Join(dest, "real", "sub")); !os.IsNotExist(err) { + t.Fatalf("real/sub = %v, want IsNotExist (entry must not land through the lower symlink)", err) + } +} + +// TestUnpackDirEntryParentSymlinkReplaced is the directory-entry variant of +// TestUnpackParentSymlinkReplaced: a dir entry beneath a lower-layer symlink +// parent must materialize under a real directory, not inside the link target. +func TestUnpackDirEntryParentSymlinkReplaced(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("real", 0o755)}, + tarEntry{header: symHeader("link", "/real")}, + ) + upper := testTarLayer(t, + tarEntry{header: dirHeader("link/sub", 0o750)}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:parent-link-dir", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:parent-link-dir", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + fi, err := os.Lstat(filepath.Join(dest, "link")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("link mode = %v, want a real directory replacing the lower symlink", fi.Mode()) + } + sub, err := os.Lstat(filepath.Join(dest, "link", "sub")) + if err != nil || !sub.IsDir() || sub.Mode().Perm() != 0o750 { + t.Fatalf("link/sub = %v, err=%v; want a 0750 directory", sub, err) + } + if _, err := os.Stat(filepath.Join(dest, "real", "sub")); !os.IsNotExist(err) { + t.Fatalf("real/sub = %v, want IsNotExist (dir must not land through the lower symlink)", err) + } +} + +// TestUnpackOpaqueThroughSymlinkKeepsTarget pins that an opaque whiteout whose +// directory is a lower layer's symlink does not clear the link target's +// contents: the marker hides lower content under its own name, so the link is +// replaced with an empty real directory and the target's files survive. +func TestUnpackOpaqueThroughSymlinkKeepsTarget(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("target", 0o755)}, + tarEntry{header: regHeader("target/keep", 0o644, 0), body: "keep"}, + tarEntry{header: symHeader("d", "/target")}, + ) + upper := testTarLayer(t, + tarEntry{header: tar.Header{Name: "d/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:opaque-link", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackImage(s, "local:opaque-link", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if b, err := os.ReadFile(filepath.Join(dest, "target", "keep")); err != nil || string(b) != "keep" { + t.Fatalf("target/keep = %q, err=%v; want untouched by opaque-through-symlink", b, err) + } + fi, err := os.Lstat(filepath.Join(dest, "d")) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Fatalf("d mode = %v, want a real empty directory replacing the symlink", fi.Mode()) + } + entries, err := os.ReadDir(filepath.Join(dest, "d")) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("d entries = %v, want empty", entries) + } +} diff --git a/cmd/elfuse-oci/unpack_test.go b/cmd/elfuse-oci/unpack_test.go new file mode 100644 index 00000000..5830eaee --- /dev/null +++ b/cmd/elfuse-oci/unpack_test.go @@ -0,0 +1,417 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +func newRoot(t *testing.T) (*os.Root, string) { + t.Helper() + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatalf("OpenRoot: %v", err) + } + t.Cleanup(func() { root.Close() }) + return root, dir +} + +func applyEntries(t *testing.T, root *os.Root, entries []tar.Header) { + t.Helper() + for _, h := range entries { + var content []byte + if (h.Typeflag == tar.TypeReg || h.Typeflag == tar.TypeRegA) && h.Size > 0 { + content = []byte(strings.Repeat("x", int(h.Size))) + } + hdr := h + if err := applyEntry(root, &hdr, bytes.NewReader(content), map[string]bool{}); err != nil { + t.Fatalf("applyEntry %q: %v", h.Name, err) + } + } +} + +func applyEntryWithContent(t *testing.T, root *os.Root, h tar.Header, content string) { + t.Helper() + hdr := h + hdr.Size = int64(len(content)) + if err := applyEntry(root, &hdr, strings.NewReader(content), map[string]bool{}); err != nil { + t.Fatalf("applyEntry %q: %v", h.Name, err) + } +} + +func regHeader(name string, mode int64, size int64) tar.Header { + return tar.Header{Name: name, Mode: mode, Typeflag: tar.TypeReg, Size: size} +} +func dirHeader(name string, mode int64) tar.Header { + return tar.Header{Name: name, Mode: mode, Typeflag: tar.TypeDir} +} +func symHeader(name, target string) tar.Header { + return tar.Header{Name: name, Typeflag: tar.TypeSymlink, Linkname: target} +} +func linkHeader(name, target string) tar.Header { + return tar.Header{Name: name, Typeflag: tar.TypeLink, Linkname: target} +} + +func TestUnpackRegularFilePerm(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("bin/prog", 0o755, 0), "") + applyEntryWithContent(t, root, regHeader("etc/secret", 0o600, 0), "") + + fi, err := os.Stat(filepath.Join(dir, "bin", "prog")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o755 { + t.Errorf("perm: got %o, want 755", fi.Mode().Perm()) + } + fi, _ = os.Stat(filepath.Join(dir, "etc", "secret")) + if fi.Mode().Perm() != 0o600 { + t.Errorf("perm: got %o, want 600", fi.Mode().Perm()) + } +} + +func TestUnpackOldStyleRegularFile(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, tar.Header{ + Name: "old-style", + Mode: 0o644, + Typeflag: tar.TypeRegA, + }, "hello") + + got, err := os.ReadFile(filepath.Join(dir, "old-style")) + if err != nil { + t.Fatal(err) + } + if string(got) != "hello" { + t.Errorf("old-style file content = %q, want hello", got) + } +} + +func TestUnpackStickyAndSetuid(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("tmp", 0o1777), + regHeader("bin/su", 0o4755, 0), + }) + + fi, _ := os.Stat(filepath.Join(dir, "tmp")) + if fi.Mode()&os.ModeSticky == 0 { + t.Errorf("tmp missing sticky bit: %o", fi.Mode()) + } + fi, _ = os.Stat(filepath.Join(dir, "bin", "su")) + if fi.Mode()&os.ModeSetuid == 0 { + t.Errorf("su missing setuid bit: %o", fi.Mode()) + } + if fi.Mode().Perm() != 0o755 { + t.Errorf("su perm: got %o, want 755", fi.Mode().Perm()) + } +} + +// TestShouldDropSpecial pins the degrade decision applyMode makes when a +// mode-finalizing chmod fails: retry without the special bits only for a +// permission error that actually carried special bits, so a genuine chmod +// failure still surfaces. This is the portable stand-in for the live EPERM, +// which cannot be forced deterministically (t.TempDir() sits under a +// staff-group path where an unprivileged setgid chmod succeeds). +func TestShouldDropSpecial(t *testing.T) { + for _, tc := range []struct { + name string + err error + special os.FileMode + want bool + }{ + {"eperm with setgid degrades", syscall.EPERM, os.ModeSetgid, true}, + {"wrapped eperm with setuid degrades", + &os.PathError{Op: "chmodat", Path: "usr/bin/su", Err: syscall.EPERM}, + os.ModeSetuid, true}, + {"eperm without special bits surfaces", syscall.EPERM, 0, false}, + {"non-permission error surfaces", syscall.EINVAL, os.ModeSetgid, false}, + {"success is not a degrade", nil, os.ModeSetgid, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := shouldDropSpecial(tc.err, tc.special); got != tc.want { + t.Errorf("shouldDropSpecial(%v, %o) = %v, want %v", + tc.err, tc.special, got, tc.want) + } + }) + } +} + +// TestUnpackSetgidForeignGroupDoesNotAbort pins that a setgid entry never +// aborts the unpack, even on a host that rejects the special-bit chmod (macOS, +// when the file's inherited group is one the invoking user is not in; see +// applyMode). The permission bits must always land; the setgid bit may or may +// not survive depending on the host group, so it is deliberately not asserted. +func TestUnpackSetgidForeignGroupDoesNotAbort(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("usr", 0o755), + dirHeader("usr/bin", 0o755), + // chage in Debian: setgid group shadow, mode 02755. + regHeader("usr/bin/chage", 0o2755, 0), + }) + fi, err := os.Stat(filepath.Join(dir, "usr", "bin", "chage")) + if err != nil { + t.Fatalf("stat chage: %v", err) + } + if fi.Mode().Perm() != 0o755 { + t.Errorf("chage perm: got %o, want 755", fi.Mode().Perm()) + } +} + +// TestUnpackModesSurviveUmask pins that layer permissions are finalized with +// an explicit chmod: creation modes are masked by the process umask, so an +// image mode like 0755 or 0644 must survive a restrictive host umask even +// when no setuid/setgid/sticky bit is present. It also pins that a parent +// directory's exact mode from its own tar entry is not reset to the 0755 +// default by the ensure-parent pass of a later child entry. +func TestUnpackModesSurviveUmask(t *testing.T) { + old := syscall.Umask(0o077) + defer syscall.Umask(old) + + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("opt", 0o755), + regHeader("opt/tool", 0o755, 0), + regHeader("opt/data", 0o644, 0), + dirHeader("secret", 0o700), + regHeader("secret/key", 0o600, 0), + }) + + for _, c := range []struct { + name string + want os.FileMode + }{ + {"opt", 0o755}, + {"opt/tool", 0o755}, + {"opt/data", 0o644}, + {"secret", 0o700}, + {"secret/key", 0o600}, + } { + fi, err := os.Stat(filepath.Join(dir, c.name)) + if err != nil { + t.Fatalf("stat %s: %v", c.name, err) + } + if fi.Mode().Perm() != c.want { + t.Errorf("%s perm: got %o, want %o", c.name, fi.Mode().Perm(), c.want) + } + } +} + +func TestUnpackAbsoluteSymlinkRewritten(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("bin", 0o755), + regHeader("bin/busybox", 0o755, 0), + // /bin/sh -> /bin/busybox (absolute). Should be rewritten to "busybox" + // (relative), resolving to /bin/busybox under the sysroot. + symHeader("bin/sh", "/bin/busybox"), + // /lib/ld -> /lib/ld-musl.so.1 + dirHeader("lib", 0o755), + regHeader("lib/ld-musl.so.1", 0o644, 0), + symHeader("lib/ld", "/lib/ld-musl.so.1"), + }) + + got, err := os.Readlink(filepath.Join(dir, "bin", "sh")) + if err != nil { + t.Fatal(err) + } + if filepath.IsAbs(got) { + t.Errorf("absolute symlink not rewritten: %q", got) + } + if got != "busybox" { + t.Errorf("rewritten target: got %q, want busybox", got) + } + // Resolving the rewritten link must reach the real file. + target := filepath.Join(dir, "bin", got) + if _, err := os.Stat(target); err != nil { + t.Errorf("rewritten link does not resolve: %v", err) + } + got2, _ := os.Readlink(filepath.Join(dir, "lib", "ld")) + if filepath.IsAbs(got2) { + t.Errorf("deep absolute symlink not rewritten: %q", got2) + } +} + +func TestUnpackAbsoluteSymlinkCleansRootTraversal(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("escape", 0o644, 0), "") + applyEntries(t, root, []tar.Header{ + dirHeader("usr", 0o755), + dirHeader("usr/bin", 0o755), + symHeader("usr/bin/link", "/../escape"), + }) + + link := filepath.Join(dir, "usr", "bin", "link") + got, err := os.Readlink(link) + if err != nil { + t.Fatal(err) + } + resolved := filepath.Clean(filepath.Join(filepath.Dir(link), got)) + want := filepath.Join(dir, "escape") + if resolved != want { + t.Errorf("rewritten target resolves to %s, want %s", resolved, want) + } +} + +func TestUnpackRelativeSymlinkPreserved(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("lib", 0o755), + regHeader("lib/real.so", 0o644, 0), + symHeader("lib/link.so", "real.so"), + }) + got, _ := os.Readlink(filepath.Join(dir, "lib", "link.so")) + if got != "real.so" { + t.Errorf("relative symlink changed: got %q, want real.so", got) + } +} + +func TestUnpackWhiteoutRemovesFile(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("etc", 0o755), + regHeader("etc/keep", 0o644, 0), + regHeader("etc/gone", 0o644, 0), + {Name: "etc/.wh.gone", Typeflag: tar.TypeReg}, + }) + if _, err := os.Stat(filepath.Join(dir, "etc", "gone")); !os.IsNotExist(err) { + t.Errorf("whiteout did not remove etc/gone: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "etc", "keep")); err != nil { + t.Errorf("whiteout removed etc/keep: %v", err) + } +} + +func TestUnpackOpaqueClearsDirectory(t *testing.T) { + root, dir := newRoot(t) + applyEntries(t, root, []tar.Header{ + dirHeader("opt", 0o755), + regHeader("opt/lower-a", 0o644, 0), + regHeader("opt/lower-b", 0o644, 0), + // Opaque marker clears opt, then this layer re-adds only lower-a. + {Name: "opt/.wh..wh..opq", Typeflag: tar.TypeReg}, + regHeader("opt/lower-a", 0o644, 0), + }) + if _, err := os.Stat(filepath.Join(dir, "opt", "lower-b")); !os.IsNotExist(err) { + t.Errorf("opaque did not clear opt/lower-b: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "opt", "lower-a")); err != nil { + t.Errorf("opaque removed re-added opt/lower-a: %v", err) + } +} + +func TestUnpackHardlink(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("etc/passwd", 0o644, 5), "hello") + applyEntries(t, root, []tar.Header{linkHeader("etc/passwd-link", "etc/passwd")}) + // Both must refer to the same inode (hardlink), same content. + orig, err := os.Stat(filepath.Join(dir, "etc", "passwd")) + if err != nil { + t.Fatalf("hardlink source missing: %v", err) + } + link, err := os.Stat(filepath.Join(dir, "etc", "passwd-link")) + if err != nil { + t.Fatalf("hardlink target missing: %v", err) + } + if !os.SameFile(orig, link) { + t.Fatal("passwd and passwd-link are distinct inodes, want a hardlink") + } + if b, err := os.ReadFile(filepath.Join(dir, "etc", "passwd-link")); err != nil || string(b) != "hello" { + t.Fatalf("hardlink content = %q, err=%v; want hello", b, err) + } +} + +func TestUnpackSpecialFilesRejected(t *testing.T) { + cases := []struct { + name string + typeflag byte + want string + }{ + {"dev/ttyS0", tar.TypeChar, "char"}, + {"dev/sda", tar.TypeBlock, "block"}, + {"run/pipe", tar.TypeFifo, "fifo"}, + } + for _, tc := range cases { + t.Run(tc.want, func(t *testing.T) { + root, dir := newRoot(t) + hdr := tar.Header{Name: tc.name, Mode: 0o644, Typeflag: tc.typeflag} + err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "unsupported special file type "+tc.want) { + t.Fatalf("applyEntry special %s err = %v, want unsupported error", tc.want, err) + } + if _, err := os.Lstat(filepath.Join(dir, tc.name)); !os.IsNotExist(err) { + t.Fatalf("special entry %s on disk: %v, want IsNotExist", tc.name, err) + } + }) + } +} + +func TestUnpackPathEscapeRejected(t *testing.T) { + root, _ := newRoot(t) + hdr := regHeader("../escape", 0o644, 0) + if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + t.Fatalf("applyEntry accepted ../escape path") + } +} + +func TestUnpackWritesFileContent(t *testing.T) { + root, dir := newRoot(t) + applyEntryWithContent(t, root, regHeader("msg.txt", 0o644, 5), "hello") + b, err := os.ReadFile(filepath.Join(dir, "msg.txt")) + if err != nil { + t.Fatal(err) + } + if string(b) != "hello" { + t.Errorf("content: got %q, want hello", b) + } +} + +// Ensure applyEntry reads exactly the header's Size bytes from the reader +// (no over-read, no short read). The reader carries trailing bytes beyond +// Size so an over-read would be visible in both the count and the content, +// and a too-small reader must fail rather than write a truncated file. +func TestUnpackReadsExactSize(t *testing.T) { + root, dir := newRoot(t) + content := "abc123" + r := &countingReader{b: []byte(content + "TRAILING-JUNK")} + hdr := regHeader("f", 0o644, int64(len(content))) + if err := applyEntry(root, &hdr, r, map[string]bool{}); err != nil { + t.Fatalf("applyEntry: %v", err) + } + if r.n != len(content) { + t.Errorf("bytes read: got %d, want %d", r.n, len(content)) + } + if b, _ := os.ReadFile(filepath.Join(dir, "f")); string(b) != content { + t.Errorf("content mismatch: got %q", b) + } + + short := &countingReader{b: []byte("abc")} + hdr = regHeader("g", 0o644, int64(len(content))) + if err := applyEntry(root, &hdr, short, map[string]bool{}); err == nil { + t.Fatal("applyEntry accepted a short body, want size-mismatch error") + } +} + +type countingReader struct { + b []byte + n int +} + +func (c *countingReader) Read(p []byte) (int, error) { + if c.n >= len(c.b) { + return 0, io.EOF + } + n := copy(p, c.b[c.n:]) + c.n += n + return n, nil +} From 71ad3d5abe00538161a948889110fa2bc2b15410 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:24:06 +0200 Subject: [PATCH 05/10] Add elfuse-oci run command run is the last pipeline stage: resolve the image config into a concrete runspec, materialize the rootfs, and exec the existing `elfuse --sysroot ...` positional launch path, reusing elfuse's HVF bring-up, shebang, and dynamic-linker plumbing rather than reinventing guest launch. elfuse is located as a sibling binary ($ELFUSE_BIN overrides for tests) and replaced via exec so the shell reaps the same pid and terminal signals reach the guest directly. The runspec resolves Entrypoint/Cmd/Env/User/WorkingDir with the usual precedence (--entrypoint drops image Cmd; --env and --clear-env follow env(1) semantics; --workdir must be guest-absolute). A PATH is guaranteed: when neither the image config nor --env supplies one, Docker's conventional default is appended, so a guest whose image omits PATH still has a search path after the --clear-env launch. A symbolic --user or image User is resolved against the image's own /etc/passwd and /etc/group through os.Root-bounded, no-follow opens, so a crafted rootfs cannot redirect resolution to host account files. run auto-pulls only when the ref is genuinely absent (errNotPulled) and surfaces store corruption instead of masking it behind a network pull; an explicit --platform must match the pinned image so a ref pulled for another architecture is not silently launched. Before exec, host-truth /etc/{resolv.conf,hosts,hostname} are injected into the rootfs through the same os.Root bounds. Tests cover runspec resolution and precedence, symbolic user lookup, runtime-file injection, and the exec argv shape via the execElfuseForRun seam and a subprocess exec probe. --- cmd/elfuse-oci/commands.go | 115 +++++++- cmd/elfuse-oci/common_test.go | 37 +++ cmd/elfuse-oci/etc.go | 118 ++++++++ cmd/elfuse-oci/etc_test.go | 263 ++++++++++++++++++ cmd/elfuse-oci/main.go | 23 +- cmd/elfuse-oci/run.go | 64 +++++ cmd/elfuse-oci/run_test.go | 97 +++++++ cmd/elfuse-oci/runspec.go | 297 ++++++++++++++++++++ cmd/elfuse-oci/runspec_test.go | 407 ++++++++++++++++++++++++++++ cmd/elfuse-oci/test_helpers_test.go | 15 + 10 files changed, 1432 insertions(+), 4 deletions(-) create mode 100644 cmd/elfuse-oci/etc.go create mode 100644 cmd/elfuse-oci/etc_test.go create mode 100644 cmd/elfuse-oci/run.go create mode 100644 cmd/elfuse-oci/run_test.go create mode 100644 cmd/elfuse-oci/runspec.go create mode 100644 cmd/elfuse-oci/runspec_test.go diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index 125b7471..c4ee195d 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -4,12 +4,14 @@ package main import ( + "errors" "fmt" "os" ) -// The subcommands share common-flag parsing (common.go) and the OCI -// image-layout store (store.go). pull/unpack/inspect are pure store ops. +// The four subcommands share common-flag parsing (common.go) and the OCI +// image-layout store (store.go). pull/unpack/inspect are pure store ops; run +// additionally resolves the runspec and execs elfuse (run.go). // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { @@ -98,3 +100,112 @@ func parseInspectArgs(args []string) (commonFlags, bool, string, error) { ref, err := oneArg("inspect", fs.Args(), "") return cf, asJSON, ref, err } + +// cmdRun runs an image: `elfuse-oci run [flags] [args...]`. +// +// Flags are parsed only up to the first positional (the reference); everything +// after the reference is the guest argv tail and is passed verbatim (no flag +// parsing), matching Docker's `run IMAGE ARGS` convention. +func cmdRun(args []string) error { + cf, rf, ref, tail, err := parseRunArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + img, err := s.image(ref) + if err != nil { + // Auto-pull only when the ref is simply absent, so `run` is + // self-sufficient on first use. Any other failure (corrupt refs.json, + // unreadable layout) must surface rather than mask itself behind a + // fresh network pull. + if !errors.Is(err, errNotPulled) { + return err + } + if err := pullImage(cf, s, ref); err != nil { + return err + } + img, err = s.image(ref) + if err != nil { + return err + } + } + cfg, err := img.ConfigFile() + if err != nil { + return err + } + // An explicit --platform must match the pinned image: the store pins one + // digest per ref, so a ref pulled for another platform would otherwise + // launch silently under the wrong architecture. + if cf.platformSet { + got := Platform{OS: cfg.OS, Arch: cfg.Architecture, Variant: cfg.Variant} + want := cf.platform + if got.OS != want.OS || got.Arch != want.Arch || + (want.Variant != "" && got.Variant != want.Variant) { + return fmt.Errorf( + "run: %s is pinned for %s, not %s; `pull --platform %s %s` (after rmi) to switch", + ref, got, want, want, ref) + } + } + digest, err := img.Digest() + if err != nil { + return err + } + if rf.rootfs == "" { + rf.rootfs, err = defaultRootfsForDigest(cf.store, digest.String()) + if err != nil { + return err + } + } + + // Ensure the rootfs is unpacked before computing the spec, because + // resolveUser reads /etc/passwd and /etc/group. Re-unpack only if + // absent; a stale rootfs is the user's concern (run `unpack` to refresh). + if _, err := os.Stat(rf.rootfs); err != nil { + if !os.IsNotExist(err) { + return err + } + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rf.rootfs) + if err := unpackImage(s, ref, rf.rootfs); err != nil { + return err + } + } + + spec, err := computeRunSpec(cfg, rf, rf.rootfs, tail) + if err != nil { + return err + } + // Inject host-truth /etc/{resolv.conf,hosts,hostname} into the rootfs so + // the guest's resolver/hostname work. On the plain path this mutates the + // unpacked rootfs directory (acceptable: --plain-rootfs is the v1/debug + // path; re-runs overwrite the same small files). + if err := injectRuntimeFiles(rf.rootfs); err != nil { + return err + } + + return execElfuseForRun(rf.rootfs, spec) +} + +func parseRunArgs(args []string) (commonFlags, runFlags, string, []string, error) { + var cf commonFlags + var rf runFlags + var env repeatedStringFlag + fs := newCommandFlagSet("run", &cf) + fs.StringVar(&rf.entrypoint, "entrypoint", "", "") + fs.Var(&env, "env", "") + fs.BoolVar(&rf.clearEnv, "clear-env", false, "") + fs.StringVar(&rf.user, "user", "", "") + fs.StringVar(&rf.workdir, "workdir", "", "") + fs.StringVar(&rf.rootfs, "rootfs", "", "") + if err := fs.Parse(args); err != nil { + return cf, rf, "", nil, err + } + rf.env = []string(env) + rest := fs.Args() + if len(rest) == 0 { + return cf, rf, "", nil, fmt.Errorf("run: expected [args...]") + } + return cf, rf, rest[0], rest[1:], nil +} diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index 182f852d..73dcee99 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -89,6 +89,42 @@ func TestParseInspectArgs(t *testing.T) { } } +func TestParseRunArgs(t *testing.T) { + cf, rf, ref, tail, err := parseRunArgs([]string{ + "--store", "/s", + "--entrypoint", "/bin/sh", + "--env", "A=1", + "--env=B=2", + "--clear-env", + "--user", "1000:1000", + "--workdir", "/work", + "--rootfs", "/tmp/rootfs", + "alpine:3", + "-c", "echo hi", + }) + if err != nil { + t.Fatal(err) + } + if cf.store != "/s" { + t.Errorf("store = %q, want /s", cf.store) + } + if ref != "alpine:3" { + t.Errorf("ref = %q, want alpine:3", ref) + } + if !reflect.DeepEqual(tail, []string{"-c", "echo hi"}) { + t.Errorf("tail = %v, want [-c echo hi]", tail) + } + if rf.entrypoint != "/bin/sh" || rf.user != "1000:1000" || rf.workdir != "/work" || rf.rootfs != "/tmp/rootfs" { + t.Errorf("run flags = %+v", rf) + } + if !rf.clearEnv { + t.Error("clearEnv = false, want true") + } + if !reflect.DeepEqual(rf.env, []string{"A=1", "B=2"}) { + t.Errorf("env = %v, want [A=1 B=2]", rf.env) + } +} + func TestParseCommandFlagErrors(t *testing.T) { cases := []struct { name string @@ -97,6 +133,7 @@ func TestParseCommandFlagErrors(t *testing.T) { {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }()}, + {"run missing ref", func() error { _, _, _, _, err := parseRunArgs([]string{"--env", "A=1"}); return err }()}, } for _, tc := range cases { if tc.err == nil { diff --git a/cmd/elfuse-oci/etc.go b/cmd/elfuse-oci/etc.go new file mode 100644 index 00000000..4d6e9ec7 --- /dev/null +++ b/cmd/elfuse-oci/etc.go @@ -0,0 +1,118 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "io/fs" + "os" + "time" +) + +var ( + hostnameForRuntime = os.Hostname + readHostResolvConfig = func() ([]byte, error) { return os.ReadFile("/etc/resolv.conf") } +) + +// injectRuntimeFiles writes host-truth /etc/{resolv.conf,hosts,hostname} into +// sysroot before elfuse launches the guest. Runtimes that consume OCI images +// synthesize these per-run rather than handing the guest the image's (often +// stub or empty) copies: the guest's resolver reads /etc/resolv.conf to find its +// nameserver, and because --sysroot redirects guest absolute paths into the +// rootfs, the guest would otherwise read the image's file, not the host's. +// elfuse does not do network namespacing (the guest uses the host network +// directly), so the host's resolver config is the correct one to hand it. +// +// Overwrite is intentional: these files are runtime-controlled, not image +// content. The caller passes the final sysroot elfuse will receive as +// --sysroot (the per-run COW clone on the case-sensitive path, or the plain +// rootfs directory on the --plain-rootfs path), so writes are isolated to +// this run except when the caller opted into mutating the base tree +// (--no-clone / --plain-rootfs). +func injectRuntimeFiles(sysroot string) error { + // All access goes through os.Root so image-controlled symlinks (a + // symlinked /etc directory or a symlinked target file such as + // etc/resolv.conf -> /etc/resolv.conf) cannot redirect the writes + // outside the rootfs. + root, err := os.OpenRoot(sysroot) + if err != nil { + return err + } + defer root.Close() + + // Guard against a stray non-directory at /etc (e.g. a malformed image): + // replace a symlink rather than chasing it, and reject any other + // non-directory up front; letting it slide would only surface later as + // an opaque "not a directory" from the first runtime-file write. + if li, err := root.Lstat("etc"); err == nil { + switch { + case li.Mode()&os.ModeSymlink != 0: + if err := root.Remove("etc"); err != nil { + return err + } + case !li.IsDir(): + return fmt.Errorf("rootfs /etc is a %s, want a directory", li.Mode().Type()) + } + } else if !os.IsNotExist(err) { + return err + } + if err := root.Mkdir("etc", 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + + host, err := hostnameForRuntime() + if err != nil || host == "" { + host = "localhost" + } + + if err := writeRuntimeFile(root, "etc/hostname", []byte(host+"\n")); err != nil { + return err + } + + // Minimal hosts map: localhost + the guest's own hostname, mirroring what + // image runtimes conventionally write. + hosts := "127.0.0.1\tlocalhost " + host + "\n::1\tlocalhost ip6-localhost\n" + if err := writeRuntimeFile(root, "etc/hosts", []byte(hosts)); err != nil { + return err + } + + // resolv.conf: copy the host's verbatim (host-truth) so the guest's DNS + // lookups hit the same nameservers the host uses. Fall back to a minimal + // default if the host file is absent or empty. + resolv, err := readHostResolvConfig() + if err != nil || len(resolv) == 0 { + resolv = []byte("nameserver 8.8.8.8\n") + } + return writeRuntimeFile(root, "etc/resolv.conf", resolv) +} + +// writeRuntimeFile replaces the rootfs-relative name with content. The +// content is written to a unique temp file beside name and renamed into +// place: rename replaces the existing directory entry without following it, +// so a symlink shipped by the image at that name is unlinked rather than +// chased, and a concurrent writer (two --no-clone / --plain-rootfs runs of +// the same digest share the base tree) never observes a missing or +// half-written file the way a remove-then-create sequence would expose. +func writeRuntimeFile(root *os.Root, name string, content []byte) error { + tmp := fmt.Sprintf("%s.tmp.%d.%d", name, os.Getpid(), time.Now().UnixNano()) + f, err := root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + if _, err := f.Write(content); err != nil { + f.Close() + _ = root.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + _ = root.Remove(tmp) + return err + } + if err := root.Rename(tmp, name); err != nil { + _ = root.Remove(tmp) + return err + } + return nil +} diff --git a/cmd/elfuse-oci/etc_test.go b/cmd/elfuse-oci/etc_test.go new file mode 100644 index 00000000..8df3d4e6 --- /dev/null +++ b/cmd/elfuse-oci/etc_test.go @@ -0,0 +1,263 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestInjectRuntimeFiles asserts the three runtime files are written with the +// expected shape, and that a second call overwrites (not appends). +func TestInjectRuntimeFiles(t *testing.T) { + root := t.TempDir() + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + + host, err := os.ReadFile(filepath.Join(root, "etc", "hostname")) + if err != nil { + t.Fatalf("hostname: %v", err) + } + hostname := strings.TrimSpace(string(host)) + if hostname == "" { + t.Error("hostname is empty") + } + + hosts, err := os.ReadFile(filepath.Join(root, "etc", "hosts")) + if err != nil { + t.Fatalf("hosts: %v", err) + } + hs := string(hosts) + if !strings.Contains(hs, "127.0.0.1\tlocalhost") { + t.Errorf("hosts missing 127.0.0.1 localhost: %q", hs) + } + if !strings.Contains(hs, "::1\tlocalhost") { + t.Errorf("hosts missing ::1 localhost: %q", hs) + } + if !strings.Contains(hs, hostname) { + t.Errorf("hosts missing hostname %q: %q", hostname, hs) + } + + resolv, err := os.ReadFile(filepath.Join(root, "etc", "resolv.conf")) + if err != nil { + t.Fatalf("resolv.conf: %v", err) + } + // Substring only: the host's nameserver varies across macOS/Linux CI, and + // the fallback is "nameserver 8.8.8.8"; either way a nameserver line is + // present. + if !strings.Contains(string(resolv), "nameserver") { + t.Errorf("resolv.conf missing nameserver: %q", resolv) + } + + // Second call overwrites in place, never appends: hostname stays the same. + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + host2, _ := os.ReadFile(filepath.Join(root, "etc", "hostname")) + if strings.TrimSpace(string(host2)) != hostname { + t.Errorf("hostname changed on re-inject: got %q want %q", host2, host) + } + + // Exactly the three runtime files: the temp-and-rename writes must not + // leave *.tmp.* staging litter behind. + entries, err := os.ReadDir(filepath.Join(root, "etc")) + if err != nil { + t.Fatal(err) + } + if len(entries) != 3 { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("etc entries = %v, want exactly hostname, hosts, resolv.conf", names) + } +} + +// TestWriteRuntimeFileLeavesNoTempOnFailure pins that a failed write does not +// leave a staging temp file behind. +func TestWriteRuntimeFileLeavesNoTempOnFailure(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("directory write permissions do not bind as root") + } + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + + if err := writeRuntimeFile(root, "resolv.conf", []byte("nameserver 8.8.8.8\n")); err == nil { + t.Fatal("writeRuntimeFile into read-only dir succeeded, want error") + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("failed write left litter: %v", entries) + } +} + +// TestInjectRuntimeFilesReplacesSymlinkEtc pins the symlink guard: a stray +// /etc symlink (e.g. from a malformed image) is replaced with a real directory +// so the writes cannot escape the rootfs. +func TestInjectRuntimeFilesReplacesSymlinkEtc(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "elsewhere") + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatal(err) + } + etcLink := filepath.Join(root, "etc") + if err := os.Symlink(target, etcLink); err != nil { + t.Fatal(err) + } + + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + + li, err := os.Lstat(etcLink) + if err != nil { + t.Fatalf("Lstat etc: %v", err) + } + if li.Mode()&os.ModeSymlink != 0 { + t.Fatalf("etc is still a symlink: mode %o", li.Mode()) + } + if !li.IsDir() { + t.Fatalf("etc is not a directory: mode %o", li.Mode()) + } + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + if _, err := os.Stat(filepath.Join(etcLink, name)); err != nil { + t.Errorf("etc/%s missing after symlink replacement: %v", name, err) + } + } + // The symlink target directory must not have received the files. + if _, err := os.Stat(filepath.Join(target, "hostname")); err == nil { + t.Error("hostname leaked into the symlink target directory") + } +} + +// TestInjectRuntimeFilesReplacesSymlinkTargets asserts that a symlink shipped +// by the image AT a runtime file's own name (etc/resolv.conf -> host path) is +// replaced with a regular file rather than followed: the write must not land +// in the symlink's target outside the rootfs. +func TestInjectRuntimeFilesReplacesSymlinkTargets(t *testing.T) { + outside := t.TempDir() + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + + const sentinel = "host-owned\n" + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + hostFile := filepath.Join(outside, name) + if err := os.WriteFile(hostFile, []byte(sentinel), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(hostFile, filepath.Join(root, "etc", name)); err != nil { + t.Fatal(err) + } + } + + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + li, err := os.Lstat(filepath.Join(root, "etc", name)) + if err != nil { + t.Fatalf("Lstat etc/%s: %v", name, err) + } + if li.Mode()&os.ModeSymlink != 0 { + t.Errorf("etc/%s is still a symlink after inject", name) + } + got, err := os.ReadFile(filepath.Join(outside, name)) + if err != nil { + t.Fatalf("read outside %s: %v", name, err) + } + if string(got) != sentinel { + t.Errorf("outside %s was overwritten through the symlink: %q", name, got) + } + } +} + +func TestInjectRuntimeFilesFallbacks(t *testing.T) { + oldHostname := hostnameForRuntime + oldReadResolv := readHostResolvConfig + hostnameForRuntime = func() (string, error) { return "", errors.New("hostname unavailable") } + readHostResolvConfig = func() ([]byte, error) { return nil, errors.New("resolv unavailable") } + t.Cleanup(func() { + hostnameForRuntime = oldHostname + readHostResolvConfig = oldReadResolv + }) + + root := t.TempDir() + if err := injectRuntimeFiles(root); err != nil { + t.Fatal(err) + } + hostname, err := os.ReadFile(filepath.Join(root, "etc", "hostname")) + if err != nil { + t.Fatal(err) + } + if string(hostname) != "localhost\n" { + t.Fatalf("fallback hostname = %q, want localhost", hostname) + } + hosts, err := os.ReadFile(filepath.Join(root, "etc", "hosts")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(hosts), "localhost") { + t.Fatalf("fallback hosts = %q, want localhost mapping", hosts) + } + resolv, err := os.ReadFile(filepath.Join(root, "etc", "resolv.conf")) + if err != nil { + t.Fatal(err) + } + if string(resolv) != "nameserver 8.8.8.8\n" { + t.Fatalf("fallback resolv.conf = %q, want Google DNS fallback", resolv) + } +} + +func TestInjectRuntimeFilesFilesystemErrors(t *testing.T) { + rootFile := filepath.Join(t.TempDir(), "sysroot-file") + if err := os.WriteFile(rootFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + if err := injectRuntimeFiles(rootFile); err == nil { + t.Fatal("injectRuntimeFiles with file sysroot succeeded, want error") + } + + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "etc"), []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + if err := injectRuntimeFiles(root); err == nil { + t.Fatal("injectRuntimeFiles with regular-file etc succeeded, want error") + } +} + +// TestInjectRuntimeFilesRejectsRegularFileEtc pins the up-front check: an +// image shipping /etc as a regular file must fail with a clear error, not a +// confusing "not a directory" from the first runtime-file write. +func TestInjectRuntimeFilesRejectsRegularFileEtc(t *testing.T) { + sysroot := t.TempDir() + if err := os.WriteFile(filepath.Join(sysroot, "etc"), []byte("not a dir"), 0o644); err != nil { + t.Fatal(err) + } + err := injectRuntimeFiles(sysroot) + if err == nil || !strings.Contains(err.Error(), "want a directory") { + t.Fatalf("injectRuntimeFiles err = %v, want explicit non-directory /etc error", err) + } + if b, rerr := os.ReadFile(filepath.Join(sysroot, "etc")); rerr != nil || string(b) != "not a dir" { + t.Fatalf("etc file after rejection = %q, err=%v; want untouched", b, rerr) + } +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index 9308cc22..e6d10c5f 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -3,8 +3,11 @@ // elfuse-oci is the OCI image CLI for elfuse. // -// It owns the OCI image pipeline (pull, store, inspect, and unpack for -// now) using go-containerregistry. elfuse itself stays a pure Linux +// It owns the OCI image pipeline (pull, store, inspect, unpack, and run +// orchestration) using go-containerregistry. For `run` it execs the existing +// `elfuse --sysroot ` positional launch path, +// reusing elfuse's HVF bring-up / shebang / dynamic-linker plumbing rather +// than reinventing guest launch. elfuse itself stays a pure Linux // syscall-to-Darwin runtime with no OCI awareness. // // Usage: @@ -12,6 +15,9 @@ // elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] // elfuse-oci unpack [--store DIR] [--rootfs DIR] // elfuse-oci inspect [--store DIR] [--json] +// elfuse-oci run [--store DIR] [--entrypoint E] [--env K=V]... +// [--user UID[:GID]] [--workdir DIR] [--platform ...] +// [args...] // // is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., // localhost:5000/foo:tag, or name@sha256:...). The default store is @@ -39,6 +45,7 @@ commands: pull Pull an image reference into the local OCI store unpack Unpack a stored image's layers into a rootfs directory inspect Print a stored image's manifest + config + run Pull + unpack + exec the image's entrypoint under elfuse help Show this help version Print the elfuse-oci version @@ -46,6 +53,16 @@ common flags: --store DIR OCI store directory (default $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci) --platform os/arch[/variant] Target platform (default linux/arm64) + +run flags: + --entrypoint PATH Override the image Entrypoint (drops image Cmd) + --env KEY=VAL Set a guest env var (repeatable; bare KEY inherits + from the host environ) + --clear-env Start the guest env empty (only --env apply) + --user UID[:GID] Run as UID (and GID; defaults to UID). Symbolic names + are resolved against the image /etc/passwd and + /etc/group before exec. + --workdir DIR Guest-absolute initial working directory `) } @@ -80,6 +97,8 @@ func dispatch(cmd string, rest []string) error { return cmdUnpack(rest) case "inspect": return cmdInspect(rest) + case "run": + return cmdRun(rest) default: usage() return fmt.Errorf("unknown command: %s", cmd) diff --git a/cmd/elfuse-oci/run.go b/cmd/elfuse-oci/run.go new file mode 100644 index 00000000..653f5f00 --- /dev/null +++ b/cmd/elfuse-oci/run.go @@ -0,0 +1,64 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +var execElfuseForRun = execElfuse + +// elfuseBin locates the elfuse binary to exec for `run`. Precedence: +// - $ELFUSE_BIN (an override hook for tests and wrapper scripts); +// - the sibling of this executable (build/elfuse-oci -> build/elfuse). +func elfuseBin() (string, error) { + if p := os.Getenv("ELFUSE_BIN"); p != "" { + return p, nil + } + exe, err := os.Executable() + if err != nil { + return "", fmt.Errorf("locate elfuse: %w", err) + } + return filepath.Join(filepath.Dir(exe), "elfuse"), nil +} + +// execElfuse replaces this process with `elfuse --sysroot --user U:G +// --workdir D --clear-env --env K=V ... `. +// +// --clear-env plus every final env var as an explicit --env makes the guest +// see exactly the runspec env (image Env merged with --env overrides, per the +// precedence matrix) rather than the host environ. +// +// syscall.Exec replaces elfuse-oci in place: the invoking shell reaps +// the same pid, and signals such as Ctrl-C go straight to elfuse rather than +// through a Go middleman. +func execElfuse(rootfs string, spec *runSpec) error { + bin, err := elfuseBin() + if err != nil { + return err + } + if _, err := os.Stat(bin); err != nil { + return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) + } + + argv := []string{ + "elfuse", + "--sysroot", rootfs, + "--user", fmt.Sprintf("%d:%d", spec.UID, spec.GID), + "--workdir", spec.Workdir, + "--clear-env", + } + for _, e := range spec.Env { + argv = append(argv, "--env", e) + } + argv = append(argv, spec.Args...) + + if err := syscall.Exec(bin, argv, os.Environ()); err != nil { + return fmt.Errorf("exec %s: %w", bin, err) + } + return nil // unreachable +} diff --git a/cmd/elfuse-oci/run_test.go b/cmd/elfuse-oci/run_test.go new file mode 100644 index 00000000..f254182b --- /dev/null +++ b/cmd/elfuse-oci/run_test.go @@ -0,0 +1,97 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// writeElfuseStub writes a #!/bin/sh stub script that stands in for the +// elfuse binary, and points elfuseBin() at it via $ELFUSE_BIN. spawnElfuseWait +// exec.Command's whatever $ELFUSE_BIN names, so no real elfuse (and no HVF) is +// needed. t.Setenv restores the env on cleanup. +func writeElfuseStub(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "elfuse-stub.sh") + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("ELFUSE_BIN", p) + return p +} + +func TestElfuseBinEnvAndFallback(t *testing.T) { + want := filepath.Join(t.TempDir(), "elfuse-custom") + t.Setenv("ELFUSE_BIN", want) + got, err := elfuseBin() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("elfuseBin with env = %q, want %q", got, want) + } + + t.Setenv("ELFUSE_BIN", "") + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + want = filepath.Join(filepath.Dir(exe), "elfuse") + got, err = elfuseBin() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("elfuseBin fallback = %q, want %q", got, want) + } +} + +func TestExecElfuseMissingBinary(t *testing.T) { + t.Setenv("ELFUSE_BIN", filepath.Join(t.TempDir(), "missing-elfuse")) + err := execElfuse(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}) + if err == nil || !strings.Contains(err.Error(), "elfuse binary not found") { + t.Fatalf("execElfuse missing binary err = %v, want not found", err) + } +} + +func TestExecElfuseSuccessSubprocess(t *testing.T) { + dir := t.TempDir() + outPath := filepath.Join(dir, "argv.txt") + stub := filepath.Join(dir, "elfuse-stub.sh") + body := "printf '%s\\n' \"$@\" > \"$ELFUSE_EXEC_ARGV_OUT\"\nexit 17\n" + if err := os.WriteFile(stub, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(dir, "rootfs") + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + cmd := exec.Command(os.Args[0], "-test.run=^$") + cmd.Env = append(os.Environ(), + "ELFUSE_EXEC_ELFUSE_TEST=1", + "ELFUSE_BIN="+stub, + "ELFUSE_EXEC_ROOTFS="+rootfs, + "ELFUSE_EXEC_ARGV_OUT="+outPath, + ) + err := cmd.Run() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 17 { + t.Fatalf("execElfuse subprocess err = %T %v, want stub exit 17", err, err) + } + b, err := os.ReadFile(outPath) + if err != nil { + t.Fatal(err) + } + out := string(b) + for _, want := range []string{"--sysroot", rootfs, "--user", "1:2", "--workdir", "/", "--clear-env", "--env", "A=1", "/bin/echo", "hi"} { + if !strings.Contains(out, want) { + t.Fatalf("exec argv missing %q in:\n%s", want, out) + } + } +} diff --git a/cmd/elfuse-oci/runspec.go b/cmd/elfuse-oci/runspec.go new file mode 100644 index 00000000..307de249 --- /dev/null +++ b/cmd/elfuse-oci/runspec.go @@ -0,0 +1,297 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// runSpec is the fully-resolved launch specification handed to elfuse. +type runSpec struct { + // Args is the final command vector: resolved Entrypoint followed by the + // resolved Cmd (image Cmd, the CLI tail, or nothing per the precedence + // matrix below). + Args []string + // Env is the final environment (image Env, overridden/appended by --env; + // --clear-env starts from empty). Bare KEY entries are expanded against + // the host environ here so elfuse receives only KEY=VAL. + Env []string + // Workdir is the guest-absolute initial working directory. + Workdir string + // UID/GID are the resolved numeric identity. + UID uint32 + GID uint32 +} + +// defaultGuestPath is Docker's conventional default PATH. computeRunSpec +// appends it when neither the image config nor --env supplies a PATH: run +// launches elfuse with --clear-env, so a guest whose image config omits PATH +// would otherwise start with no search path at all. +const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +// runFlags are the run-specific flags parsed between the common flags and the +// image reference. Everything after the reference is the guest argv tail and +// is not flag-parsed. +type runFlags struct { + entrypoint string + env []string + clearEnv bool + user string + workdir string + rootfs string +} + +// computeRunSpec applies the Entrypoint/Cmd/Env/WorkingDir/User precedence. +// +// Command (Docker/OCI semantics, matching the branch's runspec.c): +// - --entrypoint overrides the image Entrypoint AND discards the image Cmd. +// The CLI tail then becomes the new Cmd; with no tail the command is just +// the --entrypoint. +// - Without --entrypoint, a non-empty CLI tail replaces the image Cmd while +// the image Entrypoint is kept. +// - With neither --entrypoint nor a tail, the command is image Entrypoint + +// image Cmd. +// +// Env: +// - --clear-env starts from empty; otherwise the base is the image Env. +// - --env KEY=VAL overrides any existing KEY and appends if new. +// - --env KEY (bare) inherits KEY from the host environ (resolved here). +// - a PATH is guaranteed: when the merged result carries none, Docker's +// conventional default is appended (defaultGuestPath). +// +// WorkingDir: --workdir, else image WorkingDir, else "/". +// User: --user, else image User, resolved to numeric uid:gid against the +// rootfs /etc/passwd and /etc/group. +func computeRunSpec(cfg *v1.ConfigFile, rf runFlags, rootfs string, tail []string) (*runSpec, error) { + args := resolveArgs(cfg.Config.Entrypoint, cfg.Config.Cmd, rf.entrypoint, tail) + if len(args) == 0 { + return nil, fmt.Errorf("no command: image has no Entrypoint/Cmd and none given") + } + + env := resolveEnv(cfg.Config.Env, rf.env, rf.clearEnv) + if !slices.ContainsFunc(env, func(kv string) bool { + return strings.HasPrefix(kv, "PATH=") + }) { + env = append(env, "PATH="+defaultGuestPath) + } + + workdir := rf.workdir + if workdir == "" { + workdir = cfg.Config.WorkingDir + } + if workdir == "" { + workdir = "/" + } + if !filepath.IsAbs(workdir) { + return nil, fmt.Errorf("workdir %q is not guest-absolute", workdir) + } + + user := rf.user + if user == "" { + user = cfg.Config.User + } + uid, gid, err := resolveUser(rootfs, user) + if err != nil { + return nil, err + } + + return &runSpec{ + Args: args, + Env: env, + Workdir: workdir, + UID: uid, + GID: gid, + }, nil +} + +// resolveArgs implements the Entrypoint/Cmd precedence described above. +func resolveArgs(imgEntry, imgCmd []string, cliEntry string, tail []string) []string { + if cliEntry != "" { + // --entrypoint clobbers image Entrypoint and image Cmd. The CLI tail, + // if any, is the new Cmd. + return append([]string{cliEntry}, tail...) + } + if len(tail) > 0 { + // CLI args replace image Cmd, keep image Entrypoint. + return slices.Concat(imgEntry, tail) + } + // No --entrypoint, no tail: image Entrypoint + image Cmd. + return slices.Concat(imgEntry, imgCmd) +} + +// resolveEnv builds the final environment list. +func resolveEnv(imgEnv []string, overrides []string, clearEnv bool) []string { + var out []string + seen := map[string]int{} + set := func(k, v string) { + if idx, ok := seen[k]; ok { + out[idx] = k + "=" + v + return + } + seen[k] = len(out) + out = append(out, k+"="+v) + } + if !clearEnv { + for _, kv := range imgEnv { + if k, v, ok := strings.Cut(kv, "="); ok { + set(k, v) + } + } + } + for _, e := range overrides { + if k, v, ok := strings.Cut(e, "="); ok { + set(k, v) + continue + } + // Bare KEY: inherit from the host environ, or skip if unset. + if v, ok := os.LookupEnv(e); ok { + set(e, v) + } + } + return out +} + +// resolveUser resolves a user spec ("uid", "uid:gid", "name", "name:group") +// to numeric uid:gid against the rootfs /etc/passwd and /etc/group. A bare +// numeric uid defaults gid to uid (matching elfuse's --user convention). +// "root" resolves through /etc/passwd like any other name (its gid can +// differ from 0), but falls back to 0:0 when the rootfs has no usable +// passwd entry for it: root's uid is 0 by definition, so FROM scratch-style +// images with USER root keep working. An explicit ":group" part is still +// resolved normally. +func resolveUser(rootfs, spec string) (uint32, uint32, error) { + if spec == "" { + return 0, 0, nil + } + userPart, groupPart, _ := strings.Cut(spec, ":") + + uid, puidGid, err := resolveUserPart(rootfs, userPart) + if err != nil { + if userPart != "root" { + return 0, 0, err + } + // No readable /etc/passwd or no root entry: uid 0 by definition, + // gid 0 as the only sane default. + uid, puidGid = 0, 0 + } + var gid uint32 + switch { + case groupPart == "": + gid = puidGid // passwd gid, or == uid for bare numeric + case isAllDigits(groupPart): + g, err := strconv.ParseUint(groupPart, 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("invalid gid %q: %w", groupPart, err) + } + gid = uint32(g) + default: + g, err := lookupGroup(rootfs, groupPart) + if err != nil { + return 0, 0, err + } + gid = g + } + return uid, gid, nil +} + +// resolveUserPart resolves the user component to (uid, defaultGid). For a +// numeric uid the default gid is the uid itself; for a name it is the gid +// field of the matching /etc/passwd entry. +func resolveUserPart(rootfs, part string) (uint32, uint32, error) { + if isAllDigits(part) { + u, err := strconv.ParseUint(part, 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("invalid uid %q: %w", part, err) + } + uid := uint32(u) + return uid, uid, nil + } + return lookupPasswd(rootfs, part) +} + +func isAllDigits(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if c < '0' || c > '9' { + return false + } + } + return true +} + +// openInRootfs opens a rootfs-relative path via os.Root so an +// image-controlled symlink (e.g. etc/passwd -> /etc/passwd) cannot redirect +// the read to host files outside the rootfs. The returned file stays valid +// after the root handle is closed. +func openInRootfs(rootfs, name string) (*os.File, error) { + root, err := os.OpenRoot(rootfs) + if err != nil { + return nil, err + } + defer root.Close() + return root.Open(name) +} + +// findColonEntry scans the colon-separated database / (e.g. +// etc/passwd) for the line whose first field is name and has at least +// minFields fields, returning the fields. Errors name the file guest-absolute +// so callers only add their own context prefix. +func findColonEntry(rootfs, file, name string, minFields int) ([]string, error) { + f, err := openInRootfs(rootfs, file) + if err != nil { + return nil, fmt.Errorf("open /%s: %w", file, err) + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Split(sc.Text(), ":") + if len(fields) >= minFields && fields[0] == name { + return fields, nil + } + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("scan /%s: %w", file, err) + } + return nil, fmt.Errorf("not found in /%s", file) +} + +// lookupPasswd finds name in /etc/passwd, returning (uid, gid). +func lookupPasswd(rootfs, name string) (uint32, uint32, error) { + fields, err := findColonEntry(rootfs, "etc/passwd", name, 4) + if err != nil { + return 0, 0, fmt.Errorf("resolve user %q: %w", name, err) + } + uid, err := strconv.ParseUint(fields[2], 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("resolve user %q: bad uid in /etc/passwd: %w", name, err) + } + gid, err := strconv.ParseUint(fields[3], 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("resolve user %q: bad gid in /etc/passwd: %w", name, err) + } + return uint32(uid), uint32(gid), nil +} + +// lookupGroup finds name in /etc/group, returning gid. +func lookupGroup(rootfs, name string) (uint32, error) { + fields, err := findColonEntry(rootfs, "etc/group", name, 3) + if err != nil { + return 0, fmt.Errorf("resolve group %q: %w", name, err) + } + gid, err := strconv.ParseUint(fields[2], 10, 32) + if err != nil { + return 0, fmt.Errorf("resolve group %q: bad gid in /etc/group: %w", name, err) + } + return uint32(gid), nil +} diff --git a/cmd/elfuse-oci/runspec_test.go b/cmd/elfuse-oci/runspec_test.go new file mode 100644 index 00000000..9c7d3b65 --- /dev/null +++ b/cmd/elfuse-oci/runspec_test.go @@ -0,0 +1,407 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" +) + +func TestResolveArgs(t *testing.T) { + cases := []struct { + name string + imgEntry, imgCmd []string + cliEntry string + tail []string + want []string + }{ + {"image entry+cmd, no overrides", []string{"/ep"}, []string{"-c"}, "", nil, []string{"/ep", "-c"}}, + {"tail replaces cmd, keeps entry", []string{"/ep"}, []string{"-c"}, "", []string{"-x"}, []string{"/ep", "-x"}}, + {"--entrypoint clobbers entry+cmd, no tail", []string{"/ep"}, []string{"-c"}, "/new", nil, []string{"/new"}}, + {"--entrypoint + tail", []string{"/ep"}, []string{"-c"}, "/new", []string{"-x"}, []string{"/new", "-x"}}, + {"no entrypoint, image cmd", nil, []string{"/bin/sh"}, "", nil, []string{"/bin/sh"}}, + {"no entrypoint, tail replaces cmd", nil, []string{"/bin/sh"}, "", []string{"/bin/echo", "hi"}, []string{"/bin/echo", "hi"}}, + {"nothing at all", nil, nil, "", nil, nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := resolveArgs(c.imgEntry, c.imgCmd, c.cliEntry, c.tail) + if !reflect.DeepEqual(got, c.want) { + t.Errorf("resolveArgs: got %v, want %v", got, c.want) + } + }) + } +} + +func TestResolveEnv(t *testing.T) { + t.Setenv("ELFUSE_TEST_HOST", "from-host") + cases := []struct { + name string + imgEnv []string + overrides []string + clearEnv bool + want []string + }{ + {"image env only", []string{"A=1", "B=2"}, nil, false, []string{"A=1", "B=2"}}, + {"override existing", []string{"A=1"}, []string{"A=9"}, false, []string{"A=9"}}, + {"append new", []string{"A=1"}, []string{"B=2"}, false, []string{"A=1", "B=2"}}, + {"clear-env drops image env", []string{"A=1"}, []string{"B=2"}, true, []string{"B=2"}}, + {"bare KEY inherits host", []string{"A=1"}, []string{"ELFUSE_TEST_HOST"}, false, []string{"A=1", "ELFUSE_TEST_HOST=from-host"}}, + {"bare KEY unset on host is skipped", []string{"A=1"}, []string{"DEFINITELY_UNSET_XYZ"}, false, []string{"A=1"}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := resolveEnv(c.imgEnv, c.overrides, c.clearEnv) + if !reflect.DeepEqual(got, c.want) { + t.Errorf("resolveEnv: got %v, want %v", got, c.want) + } + }) + } +} + +func TestResolveUser(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte( + "root:x:0:0:root:/root:/bin/sh\nbin:x:1:1:bin:/bin:/sbin/nologin\nnobody:x:65534:65534:nobody:/:/sbin/nologin\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "group"), []byte( + "root:x:0:\nbin:x:1:\nstaff:x:20:\n"), 0o644); err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + spec string + wantUID uint32 + wantGID uint32 + wantErr bool + }{ + {"empty is root", "", 0, 0, false}, + {"root name", "root", 0, 0, false}, + {"bare numeric uid defaults gid=uid", "1000", 1000, 1000, false}, + {"numeric uid:gid", "1000:20", 1000, 20, false}, + {"name from passwd", "bin", 1, 1, false}, + {"name:group", "bin:staff", 1, 20, false}, + {"name:numeric gid", "bin:99", 1, 99, false}, + {"unknown user errors", "ghost", 0, 0, true}, + {"unknown group errors", "bin:ghost", 0, 0, true}, + {"root:group resolves the group part", "root:staff", 0, 20, false}, + {"root with unknown group errors", "root:ghost", 0, 0, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + uid, gid, err := resolveUser(root, c.spec) + if (err != nil) != c.wantErr { + t.Fatalf("resolveUser(%q): err=%v, wantErr=%v", c.spec, err, c.wantErr) + } + if c.wantErr { + return + } + if uid != c.wantUID || gid != c.wantGID { + t.Errorf("resolveUser(%q): uid=%d gid=%d, want %d:%d", c.spec, uid, gid, c.wantUID, c.wantGID) + } + }) + } +} + +// TestResolveUserRootGidFromPasswd pins that "root" resolves through +// /etc/passwd like any other name: a root entry with a non-zero gid wins +// over the 0:0 fallback. +func TestResolveUserRootGidFromPasswd(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte( + "root:x:0:50:root:/root:/bin/sh\n"), 0o644); err != nil { + t.Fatal(err) + } + uid, gid, err := resolveUser(root, "root") + if err != nil { + t.Fatalf("resolveUser(root): %v", err) + } + if uid != 0 || gid != 50 { + t.Errorf("resolveUser(root): uid=%d gid=%d, want 0:50", uid, gid) + } +} + +// TestResolveUserRootWithoutPasswd pins the FROM scratch fallback: with no +// readable /etc/passwd (or one lacking a root entry), "root" must resolve +// to 0:0 instead of erroring. +func TestResolveUserRootWithoutPasswd(t *testing.T) { + cases := []struct { + name string + passwd string // written to etc/passwd when non-empty + }{ + {"no passwd", ""}, + {"no root entry", "bin:x:1:1:bin:/bin:/sbin/nologin\n"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + root := t.TempDir() + if c.passwd != "" { + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte(c.passwd), 0o644); err != nil { + t.Fatal(err) + } + } + uid, gid, err := resolveUser(root, "root") + if err != nil { + t.Fatalf("resolveUser(root): %v", err) + } + if uid != 0 || gid != 0 { + t.Errorf("resolveUser(root): uid=%d gid=%d, want 0:0", uid, gid) + } + }) + } +} + +func TestLookupPasswdScannerError(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + longLine := strings.Repeat("x", bufio.MaxScanTokenSize+1) + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte(longLine), 0o644); err != nil { + t.Fatal(err) + } + _, _, err := lookupPasswd(root, "root") + if err == nil || !strings.Contains(err.Error(), "scan /etc/passwd") { + t.Fatalf("lookupPasswd err = %v, want scan /etc/passwd error", err) + } +} + +// TestLookupPasswdRejectsSymlinkEscape pins the rootfs-bounded open: an image +// whose etc/passwd is a symlink to a file outside the rootfs must not have +// user resolution read that host file. +func TestLookupPasswdRejectsSymlinkEscape(t *testing.T) { + outside := t.TempDir() + hostPasswd := filepath.Join(outside, "passwd") + if err := os.WriteFile(hostPasswd, []byte("evil:x:0:0::/:/bin/sh\n"), 0o644); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(hostPasswd, filepath.Join(root, "etc", "passwd")); err != nil { + t.Fatal(err) + } + if _, _, err := lookupPasswd(root, "evil"); err == nil { + t.Fatal("lookupPasswd resolved a user through a symlink escaping the rootfs") + } + + if err := os.Symlink(hostPasswd, filepath.Join(root, "etc", "group")); err != nil { + t.Fatal(err) + } + if _, err := lookupGroup(root, "evil"); err == nil { + t.Fatal("lookupGroup resolved a group through a symlink escaping the rootfs") + } +} + +func TestLookupGroupScannerError(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + longLine := strings.Repeat("x", bufio.MaxScanTokenSize+1) + if err := os.WriteFile(filepath.Join(root, "etc", "group"), []byte(longLine), 0o644); err != nil { + t.Fatal(err) + } + _, err := lookupGroup(root, "root") + if err == nil || !strings.Contains(err.Error(), "scan /etc/group") { + t.Fatalf("lookupGroup err = %v, want scan /etc/group error", err) + } +} + +// TestComputeRunSpecNoCommand exercises the empty-command error branch: no +// image Entrypoint/Cmd and no --entrypoint/tail yields an error. computeRunSpec +// takes a *v1.ConfigFile directly, so no real image is needed; with User empty, +// resolveUser returns 0:0 without touching /etc/passwd. +func TestComputeRunSpecNoCommand(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{}} // no Entrypoint, no Cmd + if _, err := computeRunSpec(cfg, runFlags{}, t.TempDir(), nil); err == nil || + !strings.Contains(err.Error(), "no command") { + t.Fatalf("err=%v, want an error containing %q", err, "no command") + } +} + +// TestComputeRunSpecWorkdirNotAbsolute covers the non-absolute workdir error +// branch. The command check (runspec.go:73) runs before the workdir check +// (:89), so the config must carry a valid Cmd to reach it. A subtest covers +// the image-config WorkingDir path too. +func TestComputeRunSpecWorkdirNotAbsolute(t *testing.T) { + t.Run("flag workdir", func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/hello"}}} + rf := runFlags{workdir: "relative/path"} + _, err := computeRunSpec(cfg, rf, t.TempDir(), nil) + if err == nil || !strings.Contains(err.Error(), "not guest-absolute") { + t.Fatalf("err=%v, want an error containing %q", err, "not guest-absolute") + } + }) + t.Run("image workdir", func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/hello"}, WorkingDir: "rel"}} + _, err := computeRunSpec(cfg, runFlags{}, t.TempDir(), nil) + if err == nil || !strings.Contains(err.Error(), "not guest-absolute") { + t.Fatalf("err=%v, want an error containing %q", err, "not guest-absolute") + } + }) +} + +func writeUserFiles(t *testing.T, root, passwd, group string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "etc", "passwd"), []byte(passwd), 0o644); err != nil { + t.Fatal(err) + } + if group != "" { + if err := os.WriteFile(filepath.Join(root, "etc", "group"), []byte(group), 0o644); err != nil { + t.Fatal(err) + } + } +} + +func TestComputeRunSpecSuccessFullPrecedence(t *testing.T) { + root := t.TempDir() + writeUserFiles(t, root, + "root:x:0:0:root:/root:/bin/sh\nbin:x:1:1:bin:/bin:/sbin/nologin\n", + "root:x:0:\nstaff:x:20:\n", + ) + cfg := &v1.ConfigFile{Config: v1.Config{ + Entrypoint: []string{"/entry"}, + Cmd: []string{"image-cmd"}, + Env: []string{"A=1", "B=2"}, + WorkingDir: "/image-workdir", + User: "root", + }} + rf := runFlags{ + env: []string{"B=9", "C=3"}, + workdir: "/flag-workdir", + user: "bin:staff", + } + spec, err := computeRunSpec(cfg, rf, root, []string{"tail-cmd", "arg"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(spec.Args, []string{"/entry", "tail-cmd", "arg"}) { + t.Fatalf("Args = %v, want entrypoint plus CLI tail", spec.Args) + } + if !reflect.DeepEqual(spec.Env, []string{"A=1", "B=9", "C=3", "PATH=" + defaultGuestPath}) { + t.Fatalf("Env = %v, want ordered override plus default PATH", spec.Env) + } + if spec.Workdir != "/flag-workdir" { + t.Fatalf("Workdir = %q, want flag workdir", spec.Workdir) + } + if spec.UID != 1 || spec.GID != 20 { + t.Fatalf("UID:GID = %d:%d, want 1:20", spec.UID, spec.GID) + } +} + +// TestComputeRunSpecDefaultPath pins the PATH guarantee: the guest always +// receives a PATH, the image's own PATH is never rewritten, and an --env +// override wins over both. +func TestComputeRunSpecDefaultPath(t *testing.T) { + cases := []struct { + name string + imgEnv []string + env []string + clearEnv bool + want []string + }{ + {"no PATH anywhere gets the default", []string{"A=1"}, nil, false, + []string{"A=1", "PATH=" + defaultGuestPath}}, + {"image PATH is preserved", []string{"PATH=/opt/bin"}, nil, false, + []string{"PATH=/opt/bin"}}, + {"--env PATH wins", []string{"PATH=/opt/bin"}, []string{"PATH=/bin"}, false, + []string{"PATH=/bin"}}, + {"--clear-env still yields a PATH", []string{"PATH=/opt/bin"}, nil, true, + []string{"PATH=" + defaultGuestPath}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg := &v1.ConfigFile{Config: v1.Config{ + Cmd: []string{"/bin/true"}, + Env: c.imgEnv, + }} + rf := runFlags{env: c.env, clearEnv: c.clearEnv} + spec, err := computeRunSpec(cfg, rf, t.TempDir(), nil) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(spec.Env, c.want) { + t.Errorf("Env = %v, want %v", spec.Env, c.want) + } + }) + } +} + +func TestResolveArgsDoesNotMutateInputs(t *testing.T) { + entry := []string{"/entry"} + cmd := []string{"image-cmd"} + tail := []string{"tail"} + got := resolveArgs(entry, cmd, "", tail) + got[0] = "/changed" + if !reflect.DeepEqual(entry, []string{"/entry"}) { + t.Fatalf("entry mutated to %v", entry) + } + if !reflect.DeepEqual(cmd, []string{"image-cmd"}) { + t.Fatalf("cmd mutated to %v", cmd) + } + if !reflect.DeepEqual(tail, []string{"tail"}) { + t.Fatalf("tail mutated to %v", tail) + } +} + +func TestResolveEnvDuplicateOrdering(t *testing.T) { + got := resolveEnv([]string{"A=1", "B=2"}, []string{"A=3", "C=4", "B=5"}, false) + want := []string{"A=3", "B=5", "C=4"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolveEnv = %v, want %v", got, want) + } +} + +func TestResolveUserErrorBranches(t *testing.T) { + cases := []struct { + name string + passwd string // when empty, /etc/passwd is not written + group string // when empty, /etc/group is not written + user string + wantErr string + }{ + {"missing passwd", "", "", "bin", "open /etc/passwd"}, + {"bad passwd uid", "bin:x:not-a-uid:1:bin:/bin:/bin/sh\n", "", "bin", "bad uid"}, + {"bad passwd gid", "bin:x:1:not-a-gid:bin:/bin:/bin/sh\n", "", "bin", "bad gid"}, + {"missing group", "bin:x:1:1:bin:/bin:/bin/sh\n", "", "bin:staff", "open /etc/group"}, + {"bad group gid", "bin:x:1:1:bin:/bin:/bin/sh\n", "staff:x:not-a-gid:\n", "bin:staff", "bad gid"}, + {"numeric uid overflow", "", "", strings.Repeat("9", 20), "invalid uid"}, + {"numeric gid overflow", "", "", "1:" + strings.Repeat("9", 20), "invalid gid"}, + {"empty user part", "bin:x:1:1:bin:/bin:/bin/sh\n", "staff:x:20:\n", ":staff", `resolve user ""`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + if tc.passwd != "" { + writeUserFiles(t, root, tc.passwd, tc.group) + } + _, _, err := resolveUser(root, tc.user) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("resolveUser(%q) err = %v, want %q", tc.user, err, tc.wantErr) + } + }) + } +} diff --git a/cmd/elfuse-oci/test_helpers_test.go b/cmd/elfuse-oci/test_helpers_test.go index fcb0c7af..e07c21d0 100644 --- a/cmd/elfuse-oci/test_helpers_test.go +++ b/cmd/elfuse-oci/test_helpers_test.go @@ -6,6 +6,7 @@ package main import ( "archive/tar" "bytes" + "fmt" "io" "os" "os/exec" @@ -30,6 +31,20 @@ func TestMain(m *testing.M) { main() return } + if os.Getenv("ELFUSE_EXEC_ELFUSE_TEST") == "1" { + spec := &runSpec{ + Args: []string{"/bin/echo", "hi"}, + Env: []string{"A=1"}, + Workdir: "/", + UID: 1, + GID: 2, + } + if err := execElfuse(os.Getenv("ELFUSE_EXEC_ROOTFS"), spec); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(98) + } + os.Exit(99) + } os.Exit(m.Run()) } From 7309bdeb581e44f43dbcbe333653af4553375628 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:24:37 +0200 Subject: [PATCH 06/10] Add macOS sparsebundle rootfs with COW clones A plain directory rootfs on the default case-insensitive APFS volume folds Linux filenames that differ only by case. Default runs now use a case-sensitive APFS sparsebundle per pinned manifest digest, with a per-run clonefile COW rootfs so guest writes never mutate the warm base tree and repeated runs skip the unpack. Liveness and lifecycle transitions are decided by per-digest advisory flocks in the bundle directory (bundlelock.go), not by pids or directory scans: every live run holds run.lock shared from before the volume is attached until guest exit, so a killed run cannot leak liveness, and attach.lock serializes provisioning (always acquired before run.lock, making the exclusive-to-shared downgrade in provision race-free). Provisioning reuses a mount a live run still holds and only force-detaches one proven stale by an exclusive run.lock probe, so a second run of the same digest can never rip the rootfs out from under a live guest. --keep clones carry an .elfuse-keep marker so sweeps preserve them. The plain-rootfs path remains available with --plain-rootfs, and the non-Darwin stub keeps elfuse-oci buildable for pull/inspect/unpack tests on Linux. Darwin tests drive runCaseSensitive through seams for provisioning, clonefile, spawn, cleanup, and exit, and cover sparsebundle provisioning, mount/detach, attach-failure teardown, and the lock protocol with a fake hdiutil; the mount-probe and force-detach hooks are function variables so tests need no real disk images. The spawn path intercepts SIGHUP alongside INT/TERM/QUIT and installs the handler before the child starts, so a hangup or an early signal still flows through the forward/reap/teardown path. elfuse is invoked with a "--" separator before the guest command, so an image Entrypoint beginning with "-" cannot steer the host launcher. hdiutil attach failures keep stderr in the error message (stdout stays clean for the plist parse), and the parsed mount path is XML-entity-decoded so a store path containing "&" or quotes still round-trips. --- cmd/elfuse-oci/bundlelock.go | 131 +++++++++ cmd/elfuse-oci/bundlelock_test.go | 122 ++++++++ cmd/elfuse-oci/commands.go | 41 ++- cmd/elfuse-oci/common_test.go | 7 + cmd/elfuse-oci/csrun.go | 168 ++++++++++++ cmd/elfuse-oci/csrun_darwin_test.go | 364 ++++++++++++++++++++++++ cmd/elfuse-oci/csrun_other.go | 22 ++ cmd/elfuse-oci/main.go | 9 + cmd/elfuse-oci/run.go | 109 ++++++-- cmd/elfuse-oci/run_test.go | 110 ++++++++ cmd/elfuse-oci/runspec.go | 6 + cmd/elfuse-oci/sparsebundle.go | 308 +++++++++++++++++++++ cmd/elfuse-oci/sparsebundle_test.go | 412 ++++++++++++++++++++++++++++ go.mod | 6 +- 14 files changed, 1783 insertions(+), 32 deletions(-) create mode 100644 cmd/elfuse-oci/bundlelock.go create mode 100644 cmd/elfuse-oci/bundlelock_test.go create mode 100644 cmd/elfuse-oci/csrun.go create mode 100644 cmd/elfuse-oci/csrun_darwin_test.go create mode 100644 cmd/elfuse-oci/csrun_other.go create mode 100644 cmd/elfuse-oci/sparsebundle.go create mode 100644 cmd/elfuse-oci/sparsebundle_test.go diff --git a/cmd/elfuse-oci/bundlelock.go b/cmd/elfuse-oci/bundlelock.go new file mode 100644 index 00000000..d25e4432 --- /dev/null +++ b/cmd/elfuse-oci/bundlelock.go @@ -0,0 +1,131 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "syscall" +) + +// Per-digest bundle locks. +// +// A case-sensitive sparsebundle bundle (/cs///) is shared +// mutable state: concurrent `run`s of one digest share its attached volume, +// while prune --cache and rmi --force want to detach and remove it. Liveness +// is decided by advisory flocks, not by pids or directory scans: a held +// lock proves a live holder regardless of pid reuse, and a free lock proves +// the holder is gone regardless of what the directory contains. +// +// Two lock files live in the bundle directory, deliberately OUTSIDE the +// mounted volume: hdiutil detach -force revokes descriptors inside the +// volume, which would silently drop a lock held there, and the locks must be +// probeable while the volume is not attached at all. +// +// - attach.lock: exclusive, serializes bundle lifecycle transitions. A run +// holds it (blocking) across provisioning (stale-mount recovery, +// hdiutil create/attach) and the last-one-out detach; a sweep holds it +// (non-blocking) for its whole reap-detach-remove sequence. +// - run.lock: every live run holds it shared from before the volume is +// attached until the guest exits (the process exiting releases it, so a +// killed run cannot leak liveness). Anyone holding it exclusively has +// proven there are zero live runs: sweeps take it non-blocking (busy => +// skip the bundle), and provision probes it to tell a stale leftover +// mount from one that is live. +// +// Lock ordering: attach.lock is always acquired before run.lock is taken +// exclusively. That makes the EX->SH downgrade in provision safe (flock +// downgrades by release-and-reacquire, but no EX taker can slip in without +// attach.lock, which the downgrader holds) and rules out lock-order cycles +// with the store-level .lock, which prune/rmi already hold around the sweep +// while runs never take bundle locks under the store lock. + +// errCacheBusy reports that a bundle lock is held by a live run (or an +// in-flight provision), so the caller must not detach or remove the bundle. +var errCacheBusy = errors.New("in use by a live run") + +func attachLockPath(bundle string) string { return filepath.Join(bundle, "attach.lock") } +func runLockPath(bundle string) string { return filepath.Join(bundle, "run.lock") } + +// flockFile is an open file holding (or having held) an advisory flock. +type flockFile struct { + f *os.File +} + +// acquireFlock opens path (creating it if absent) and takes the flock mode +// `how` (syscall.LOCK_SH or LOCK_EX, optionally |LOCK_NB). A non-blocking +// request that loses returns errCacheBusy (wrapped with the path). +// +// A sweeper removes the whole bundle directory, lock files included, +// while holding both locks. A racing acquirer may then have opened the path +// just before the unlink and be holding a lock on an orphaned inode no later +// process can observe. Guard against that: after locking, verify the path +// still resolves to the locked inode; otherwise retry against the recreated +// file. The retry count is a defense bound, not a correctness knob; one +// retry per concurrent unlink is the worst case. +func acquireFlock(path string, how int) (*flockFile, error) { + for range 16 { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return nil, err + } + if err := flockRetryIntr(int(f.Fd()), how); err != nil { + f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return nil, fmt.Errorf("%s: %w", path, errCacheBusy) + } + return nil, fmt.Errorf("lock %s: %w", path, err) + } + var pathSt, fdSt syscall.Stat_t + if err := syscall.Stat(path, &pathSt); err != nil { + f.Close() + if errors.Is(err, syscall.ENOENT) { + continue // unlinked under us; retry on the recreated file + } + return nil, fmt.Errorf("lock %s: %w", path, err) + } + if err := syscall.Fstat(int(f.Fd()), &fdSt); err != nil { + f.Close() + return nil, fmt.Errorf("lock %s: %w", path, err) + } + if pathSt.Dev == fdSt.Dev && pathSt.Ino == fdSt.Ino { + return &flockFile{f: f}, nil + } + f.Close() // path now names a different file; lock that one instead + } + return nil, fmt.Errorf("lock %s: persistent unlink race", path) +} + +// flockRetryIntr issues flock, retrying on EINTR (a blocking acquisition may +// be interrupted by the signal forwarding the run wrapper installs). +func flockRetryIntr(fd, how int) error { + for { + err := syscall.Flock(fd, how) + if !errors.Is(err, syscall.EINTR) { + return err + } + } +} + +// Downgrade converts a held exclusive lock to shared. flock implements this +// as release-then-reacquire, so it is race-free only while the caller holds +// attach.lock: every exclusive taker of run.lock acquires attach.lock first, +// so none can slip into the gap. +func (l *flockFile) Downgrade() error { + return flockRetryIntr(int(l.f.Fd()), syscall.LOCK_SH) +} + +// Close releases the lock and closes the file. Safe on nil and after a prior +// Close. +func (l *flockFile) Close() error { + if l == nil || l.f == nil { + return nil + } + _ = syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) + err := l.f.Close() + l.f = nil + return err +} diff --git a/cmd/elfuse-oci/bundlelock_test.go b/cmd/elfuse-oci/bundlelock_test.go new file mode 100644 index 00000000..21e9fe18 --- /dev/null +++ b/cmd/elfuse-oci/bundlelock_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestAcquireFlockSharedCoexists(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer a.Close() + b, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB) + if err != nil { + t.Fatalf("second shared lock: %v, want success", err) + } + defer b.Close() +} + +func TestAcquireFlockExclusiveBlockedIsCacheBusy(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer a.Close() + _, err = acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB) + if !errors.Is(err, errCacheBusy) { + t.Fatalf("exclusive over shared err = %v, want errCacheBusy", err) + } + + // Releasing the shared lock frees the exclusive probe. + if err := a.Close(); err != nil { + t.Fatal(err) + } + b, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + t.Fatalf("exclusive after release: %v, want success", err) + } + defer b.Close() +} + +func TestFlockDowngradeAdmitsSharedBlocksExclusive(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_EX) + if err != nil { + t.Fatal(err) + } + defer a.Close() + if _, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("shared over exclusive err = %v, want errCacheBusy", err) + } + if err := a.Downgrade(); err != nil { + t.Fatalf("Downgrade: %v", err) + } + b, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB) + if err != nil { + t.Fatalf("shared after downgrade: %v, want success", err) + } + defer b.Close() + if _, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("exclusive after downgrade err = %v, want errCacheBusy", err) + } +} + +// TestAcquireFlockUnlinkRace pins the verify-retry: when a sweeper unlinks +// the lock file while another process still holds a lock on the orphaned +// inode, a fresh acquire must land on the recreated file, not block on or +// share fate with the orphan. +func TestAcquireFlockUnlinkRace(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + orphan, err := acquireFlock(path, syscall.LOCK_EX) + if err != nil { + t.Fatal(err) + } + defer orphan.Close() + // Simulate the sweeper's RemoveAll of the bundle: the path is gone while + // the orphan's lock is still held on the old inode. + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + fresh, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + t.Fatalf("acquire after unlink: %v, want success on recreated file", err) + } + defer fresh.Close() +} + +func TestFlockCloseIdempotentAndNilSafe(t *testing.T) { + path := filepath.Join(t.TempDir(), "run.lock") + a, err := acquireFlock(path, syscall.LOCK_EX) + if err != nil { + t.Fatal(err) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + if err := a.Close(); err != nil { + t.Fatalf("second Close: %v, want nil", err) + } + var nilLock *flockFile + if err := nilLock.Close(); err != nil { + t.Fatalf("nil Close: %v, want nil", err) + } +} + +func TestBundleLockPaths(t *testing.T) { + if got := attachLockPath("/store/cs/sha256/ab"); got != "/store/cs/sha256/ab/attach.lock" { + t.Fatalf("attachLockPath = %q", got) + } + if got := runLockPath("/store/cs/sha256/ab"); got != "/store/cs/sha256/ab/run.lock" { + t.Fatalf("runLockPath = %q", got) + } +} diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index c4ee195d..54539a4f 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -11,7 +11,7 @@ import ( // The four subcommands share common-flag parsing (common.go) and the OCI // image-layout store (store.go). pull/unpack/inspect are pure store ops; run -// additionally resolves the runspec and execs elfuse (run.go). +// additionally resolves the runspec and execs elfuse (run.go, csrun.go). // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { @@ -68,7 +68,7 @@ func parseUnpackArgs(args []string) (commonFlags, string, string, error) { var cf commonFlags var rootfs string fs := newCommandFlagSet("unpack", &cf) - fs.StringVar(&rootfs, "rootfs", "", "") + fs.StringVar(&rootfs, "rootfs", "", "unpack into DIR (default: the store's digest-keyed rootfs cache)") if err := fs.Parse(args); err != nil { return cf, "", "", err } @@ -93,7 +93,7 @@ func parseInspectArgs(args []string) (commonFlags, bool, string, error) { var cf commonFlags var asJSON bool fs := newCommandFlagSet("inspect", &cf) - fs.BoolVar(&asJSON, "json", false, "") + fs.BoolVar(&asJSON, "json", false, "print the raw image config JSON") if err := fs.Parse(args); err != nil { return cf, false, "", err } @@ -153,13 +153,26 @@ func cmdRun(args []string) error { if err != nil { return err } + digestStr := digest.String() + + // Choose the rootfs path. Default: a case-sensitive APFS sparsebundle so + // the guest's case-sensitive filenames don't collide on the host's + // case-insensitive volume, with a per-run COW clone for isolation and + // warm-run speed. --plain-rootfs (or an explicit --rootfs) opts out to the + // plain-directory path (syscall.Exec, no mount lifecycle). + useCS := rf.rootfs == "" && !rf.plainRootfs + + if useCS { + return runCaseSensitive(cf, s, ref, digestStr, cfg, rf, tail) + } + + // Plain-directory path. if rf.rootfs == "" { - rf.rootfs, err = defaultRootfsForDigest(cf.store, digest.String()) + rf.rootfs, err = defaultRootfsForDigest(cf.store, digestStr) if err != nil { return err } } - // Ensure the rootfs is unpacked before computing the spec, because // resolveUser reads /etc/passwd and /etc/group. Re-unpack only if // absent; a stale rootfs is the user's concern (run `unpack` to refresh). @@ -172,7 +185,6 @@ func cmdRun(args []string) error { return err } } - spec, err := computeRunSpec(cfg, rf, rf.rootfs, tail) if err != nil { return err @@ -184,7 +196,6 @@ func cmdRun(args []string) error { if err := injectRuntimeFiles(rf.rootfs); err != nil { return err } - return execElfuseForRun(rf.rootfs, spec) } @@ -193,12 +204,16 @@ func parseRunArgs(args []string) (commonFlags, runFlags, string, []string, error var rf runFlags var env repeatedStringFlag fs := newCommandFlagSet("run", &cf) - fs.StringVar(&rf.entrypoint, "entrypoint", "", "") - fs.Var(&env, "env", "") - fs.BoolVar(&rf.clearEnv, "clear-env", false, "") - fs.StringVar(&rf.user, "user", "", "") - fs.StringVar(&rf.workdir, "workdir", "", "") - fs.StringVar(&rf.rootfs, "rootfs", "", "") + fs.StringVar(&rf.entrypoint, "entrypoint", "", "override the image Entrypoint (drops the image Cmd)") + fs.Var(&env, "env", "set a guest env var KEY=VAL (repeatable; bare KEY inherits from the host)") + fs.BoolVar(&rf.clearEnv, "clear-env", false, "start the guest env empty (only --env applies)") + fs.StringVar(&rf.user, "user", "", "run as UID[:GID] or name[:group] resolved via the image /etc/passwd,group") + fs.StringVar(&rf.workdir, "workdir", "", "guest-absolute initial working directory") + fs.StringVar(&rf.rootfs, "rootfs", "", "use an explicit rootfs directory (plain dir, no sparsebundle)") + fs.BoolVar(&rf.plainRootfs, "plain-rootfs", false, "use a plain directory rootfs instead of the macOS sparsebundle") + fs.StringVar(&rf.sparseSize, "sparse-size", "", "sparsebundle virtual size (default 16g; macOS only)") + fs.BoolVar(&rf.noClone, "no-clone", false, "run against the base tree without a per-run COW clone (macOS only)") + fs.BoolVar(&rf.keepRootfs, "keep", false, "keep the per-run COW clone and mount for inspection (macOS only)") if err := fs.Parse(args); err != nil { return cf, rf, "", nil, err } diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index 73dcee99..a65d120d 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -99,6 +99,10 @@ func TestParseRunArgs(t *testing.T) { "--user", "1000:1000", "--workdir", "/work", "--rootfs", "/tmp/rootfs", + "--plain-rootfs", + "--sparse-size", "32g", + "--no-clone", + "--keep", "alpine:3", "-c", "echo hi", }) @@ -117,6 +121,9 @@ func TestParseRunArgs(t *testing.T) { if rf.entrypoint != "/bin/sh" || rf.user != "1000:1000" || rf.workdir != "/work" || rf.rootfs != "/tmp/rootfs" { t.Errorf("run flags = %+v", rf) } + if !rf.plainRootfs || rf.sparseSize != "32g" || !rf.noClone || !rf.keepRootfs { + t.Errorf("sparse run flags = %+v", rf) + } if !rf.clearEnv { t.Error("clearEnv = false, want true") } diff --git a/cmd/elfuse-oci/csrun.go b/cmd/elfuse-oci/csrun.go new file mode 100644 index 00000000..d1237eb2 --- /dev/null +++ b/cmd/elfuse-oci/csrun.go @@ -0,0 +1,168 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/google/go-containerregistry/pkg/v1" + "golang.org/x/sys/unix" +) + +var ( + ensureCaseSensitiveRootfsForRun = ensureCaseSensitiveRootfs + clonefileForRun = unix.Clonefile + spawnElfuseWaitForRun = spawnElfuseWait + cleanupCloneAndMountForRun = cleanupCloneAndMount + osExitForRun = os.Exit + runNowUnixNano = func() int64 { return time.Now().UnixNano() } +) + +// csBundleDirForDigest is /cs//: it holds the case-sensitive +// sparsebundle image and the attach mount point for one pinned manifest digest. +func csBundleDirForDigest(store, digest string) (string, error) { + key, err := cacheKeyForDigest(digest) + if err != nil { + return "", err + } + return filepath.Join(store, csCacheDirName, key), nil +} + +// ensureCaseSensitiveRootfs provisions (creating if absent) and attaches a +// case-sensitive APFS sparsebundle for ref, unpacking the image's layers into +// /rootfs when that base tree is absent. It returns the attached mount +// (the caller must Close it to detach) and the rootfs path to use as --sysroot. +// +// The unpacked base tree persists in the sparsebundle image file across +// attach/detach cycles, so warm re-runs skip the (slow) unpack and pay only the +// attach. +func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { + bundle, err := csBundleDirForDigest(cf.store, digest) + if err != nil { + return nil, "", err + } + mountPath := filepath.Join(bundle, "mnt") + m, err := provisionCaseSensitive(bundle, mountPath, size) + if err != nil { + return nil, "", err + } + rootfs := m.rootfsDir() + if _, err := os.Stat(rootfs); err != nil { + if !os.IsNotExist(err) { + return nil, "", errors.Join(err, closeMount(m)) + } + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rootfs) + if err := unpackImage(s, ref, rootfs); err != nil { + return nil, "", errors.Join(err, closeMount(m)) + } + } + return m, rootfs, nil +} + +// runCaseSensitive is the default `run` path: attach the digest-keyed +// case-sensitive sparsebundle, make a per-run COW clone of the warm base tree, +// exec elfuse against the clone, then tear the clone down and detach. On the +// happy path it does not return: it os.Exits with elfuse's status. It returns +// an error on setup failure, and on a post-run cleanup failure after a guest +// exit of zero; when the guest exits nonzero, the cleanup error is only +// printed and the guest status still wins (os.Exit), so a guest failure code +// is never masked by teardown. +// +// The clone lives in the same APFS volume as the base tree (clonefile is +// intra-volume only), so it is instant and free until the guest writes (COW). +// It isolates each run's mutations from the warm base, so re-runs stay clean. +func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { + m, baseRootfs, err := ensureCaseSensitiveRootfsForRun(cf, s, ref, digest, rf.sparseSize) + if err != nil { + return err + } + // NOTE: we cannot defer m.Close() because os.Exit below skips defers, + // which would leak the attached sparsebundle. Close explicitly on every + // exit path. + + // Per-run COW clone. --no-clone runs against the base tree directly (mutations + // then persist into the warm tree; useful for debugging or when clonefile is + // unavailable). + sysroot := baseRootfs + var cloneDir string + if !rf.noClone { + cloneDir = filepath.Join(m.mountPath, fmt.Sprintf("run-%d-%d", os.Getpid(), runNowUnixNano())) + if err := os.RemoveAll(cloneDir); err != nil { + err = fmt.Errorf("remove stale COW clone %s: %w", cloneDir, err) + return errors.Join(err, closeMount(m)) + } + if err := clonefileForRun(baseRootfs, cloneDir, unix.CLONE_NOFOLLOW); err != nil { + err = fmt.Errorf("COW clone %s -> %s: %w", baseRootfs, cloneDir, err) + return errors.Join(err, closeMount(m)) + } + sysroot = cloneDir + } + + spec, err := computeRunSpec(cfg, rf, sysroot, tail) + if err != nil { + return errors.Join(err, cleanupCloneAndMountForRun(cloneDir, rf.keepRootfs, m)) + } + // Inject host-truth /etc/{resolv.conf,hosts,hostname} into the sysroot + // before launch. On the clone path sysroot is the ephemeral COW clone, so + // the warm base tree stays clean; under --no-clone sysroot is the base tree + // and these small files are overwritten in place (the user opted into + // mutating the base). + if err := injectRuntimeFiles(sysroot); err != nil { + return errors.Join(err, cleanupCloneAndMountForRun(cloneDir, rf.keepRootfs, m)) + } + + code, err := spawnElfuseWaitForRun(sysroot, spec) + var cleanupErr error + if rf.keepRootfs { + // --keep leaves the clone and the mount in place for inspection. The + // clone lives in the sparsebundle volume, so the mount must stay + // attached for it to be reachable on the host; a later run reattaches + // (detaching this stale mount first) and the kept clone reappears. + if cloneDir != "" { + fmt.Fprintf(os.Stderr, "kept clone: %s\n", cloneDir) + } + fmt.Fprintf(os.Stderr, "mount stays attached: %s\n", m.mountPath) + } else { + cleanupErr = cleanupCloneAndMountForRun(cloneDir, false, m) + } + if err != nil { + return errors.Join(err, cleanupErr) + } + if cleanupErr != nil { + if code != 0 { + fmt.Fprintf(os.Stderr, "elfuse-oci: cleanup after exit %d: %v\n", code, cleanupErr) + osExitForRun(code) + return nil // unreachable + } + return cleanupErr + } + osExitForRun(code) + return nil // unreachable +} + +func cleanupCloneAndMount(cloneDir string, keep bool, m *csMount) error { + return errors.Join(removeClone(cloneDir, keep), closeMount(m)) +} + +func closeMount(m *csMount) error { + if err := m.Close(); err != nil { + return fmt.Errorf("detach %s: %w", m.mountPath, err) + } + return nil +} + +// removeClone deletes the ephemeral COW clone unless --keep was requested or +// there is none (the --no-clone path). +func removeClone(cloneDir string, keep bool) error { + if cloneDir == "" || keep { + return nil + } + return os.RemoveAll(cloneDir) +} diff --git a/cmd/elfuse-oci/csrun_darwin_test.go b/cmd/elfuse-oci/csrun_darwin_test.go new file mode 100644 index 00000000..3516c767 --- /dev/null +++ b/cmd/elfuse-oci/csrun_darwin_test.go @@ -0,0 +1,364 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "golang.org/x/sys/unix" +) + +type runExitCode int + +func withCSRunSeams(t *testing.T) { + t.Helper() + oldEnsure := ensureCaseSensitiveRootfsForRun + oldClone := clonefileForRun + oldSpawn := spawnElfuseWaitForRun + oldCleanup := cleanupCloneAndMountForRun + oldExit := osExitForRun + oldNow := runNowUnixNano + t.Cleanup(func() { + ensureCaseSensitiveRootfsForRun = oldEnsure + clonefileForRun = oldClone + spawnElfuseWaitForRun = oldSpawn + cleanupCloneAndMountForRun = oldCleanup + osExitForRun = oldExit + runNowUnixNano = oldNow + }) +} + +func TestRunCaseSensitiveCloneSpawnCleanupAndExit(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 123 } + expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-123", os.Getpid())) + var clonedSrc, clonedDst string + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { + if cf.store != "store" || ref != "local:a" || digest != "sha256:"+strings.Repeat("7", 64) || size != "64m" { + t.Fatalf("ensure args = store=%q ref=%q digest=%q size=%q", cf.store, ref, digest, size) + } + return m, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + clonedSrc, clonedDst = src, dst + if flags != unix.CLONE_NOFOLLOW { + t.Fatalf("clone flags = %d, want CLONE_NOFOLLOW", flags) + } + return os.MkdirAll(dst, 0o755) + } + var spawnRootfs string + var spawnSpec *runSpec + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec) (int, error) { + spawnRootfs = rootfs + spawnSpec = spec + return 7, nil + } + var cleanupClone string + var cleanupKeep bool + cleanupCloneAndMountForRun = func(cloneDir string, keep bool, got *csMount) error { + cleanupClone, cleanupKeep = cloneDir, keep + if got != m { + t.Fatalf("cleanup mount = %+v, want fake mount", got) + } + return nil + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 7 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 7", r, r) + } + if clonedSrc != base || clonedDst != expectedClone { + t.Fatalf("clone = %q -> %q, want %q -> %q", clonedSrc, clonedDst, base, expectedClone) + } + if spawnRootfs != expectedClone { + t.Fatalf("spawn rootfs = %q, want clone %q", spawnRootfs, expectedClone) + } + if spawnSpec == nil || !reflect.DeepEqual(spawnSpec.Args, []string{"/cmd"}) { + t.Fatalf("spawn spec = %+v, want /cmd", spawnSpec) + } + if cleanupClone != expectedClone || cleanupKeep { + t.Fatalf("cleanup clone=%q keep=%v, want clone and keep=false", cleanupClone, cleanupKeep) + } + }() + + cfg := &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}} + err := runCaseSensitive( + commonFlags{store: "store"}, + &store{}, + "local:a", + "sha256:"+strings.Repeat("7", 64), + cfg, + runFlags{sparseSize: "64m"}, + nil, + ) + t.Fatalf("runCaseSensitive returned %v, want osExitForRun panic", err) +} + +func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + return &csMount{mountPath: mount}, base, nil + } + clonefileForRun = func(string, string, int) error { + t.Fatal("clonefile called with --no-clone") + return nil + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec) (int, error) { + if rootfs != base { + t.Fatalf("spawn rootfs = %q, want base rootfs", rootfs) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(string, bool, *csMount) error { + t.Fatal("cleanup called with --keep") + return nil + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) + } + }() + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("8", 64), + &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, + runFlags{noClone: true, keepRootfs: true}, + nil) + t.Fatalf("runCaseSensitive returned %v, want exit panic", err) +} + +func TestRunCaseSensitiveSpecErrorCleansCloneAndMount(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 456 } + expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-456", os.Getpid())) + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + return m, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + return os.MkdirAll(dst, 0o755) + } + spawnElfuseWaitForRun = func(string, *runSpec) (int, error) { + t.Fatal("spawn called after spec error") + return 0, nil + } + var cleanupClone string + cleanupCloneAndMountForRun = func(cloneDir string, keep bool, got *csMount) error { + cleanupClone = cloneDir + if keep { + t.Fatal("cleanup keep = true, want false") + } + if got != m { + t.Fatalf("cleanup mount = %+v, want fake mount", got) + } + return nil + } + osExitForRun = func(code int) { t.Fatalf("exit called after spec error with code %d", code) } + + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("9", 64), + &v1.ConfigFile{Config: v1.Config{}}, + runFlags{}, + nil) + if err == nil || !strings.Contains(err.Error(), "no command") { + t.Fatalf("runCaseSensitive spec err = %v, want no command", err) + } + if cleanupClone != expectedClone { + t.Fatalf("cleanup clone = %q, want %q", cleanupClone, expectedClone) + } +} + +func TestEnsureCaseSensitiveRootfsProvisionsUnpacksAndSkipsExisting(t *testing.T) { + t.Run("unpacks missing rootfs", func(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("HDIUTIL_MOUNT", actualMount) + s := openTestStore(t) + digest, err := s.addImage("local:tiny", tinyImage(t)) + if err != nil { + t.Fatal(err) + } + + m, rootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + if err != nil { + t.Fatalf("ensureCaseSensitiveRootfs: %v", err) + } + t.Cleanup(func() { _ = m.Close() }) + if rootfs != filepath.Join(actualMount, "rootfs") { + t.Fatalf("rootfs = %q, want actual mount rootfs", rootfs) + } + if b, err := os.ReadFile(filepath.Join(rootfs, "hello")); err != nil || string(b) != "world" { + t.Fatalf("rootfs hello = %q, err=%v; want world", b, err) + } + }) + + t.Run("keeps existing rootfs", func(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + actualMount := filepath.Join(t.TempDir(), "actual-mount") + rootfs := filepath.Join(actualMount, "rootfs") + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootfs, "marker"), []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("HDIUTIL_MOUNT", actualMount) + s := openTestStore(t) + digest, err := s.addImage("local:tiny", tinyImage(t)) + if err != nil { + t.Fatal(err) + } + + m, gotRootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + if err != nil { + t.Fatalf("ensureCaseSensitiveRootfs existing: %v", err) + } + t.Cleanup(func() { _ = m.Close() }) + if gotRootfs != rootfs { + t.Fatalf("rootfs = %q, want %q", gotRootfs, rootfs) + } + if b, err := os.ReadFile(filepath.Join(rootfs, "marker")); err != nil || string(b) != "keep" { + t.Fatalf("marker = %q, err=%v; want keep", b, err) + } + if _, err := os.Stat(filepath.Join(rootfs, "hello")); !os.IsNotExist(err) { + t.Fatalf("existing rootfs was unpacked over: %v", err) + } + }) +} + +func TestEnsureCaseSensitiveRootfsClosesMountOnUnpackError(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + detachLog := filepath.Join(t.TempDir(), "detach.log") + t.Setenv("HDIUTIL_MOUNT", actualMount) + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + s := openTestStore(t) + + _, _, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:missing", "sha256:"+strings.Repeat("a", 64), "32m") + if err == nil || !strings.Contains(err.Error(), "not pulled") { + t.Fatalf("ensureCaseSensitiveRootfs err = %v, want unpack not-pulled error", err) + } + b, readErr := os.ReadFile(detachLog) + if readErr != nil { + t.Fatal(readErr) + } + if !strings.Contains(string(b), actualMount) { + t.Fatalf("detach log = %q, want actual mount %s", b, actualMount) + } +} + +func TestCleanupCloneAndMountAndCloseMount(t *testing.T) { + oldDetach := detachForce + var detached string + detachForce = func(path string) error { + detached = path + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + clone := filepath.Join(t.TempDir(), "clone") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: "/tmp/cleanup-mount", owned: true} + if err := cleanupCloneAndMount(clone, false, m); err != nil { + t.Fatalf("cleanupCloneAndMount: %v", err) + } + if _, err := os.Stat(clone); !os.IsNotExist(err) { + t.Fatalf("clone after cleanup: %v, want IsNotExist", err) + } + if detached != "/tmp/cleanup-mount" || m.owned { + t.Fatalf("detached=%q owned=%v, want detached mount and owned=false", detached, m.owned) + } + + detachForce = func(path string) error { return fmt.Errorf("detach boom") } + err := closeMount(&csMount{mountPath: "/tmp/bad-mount", owned: true}) + if err == nil || !strings.Contains(err.Error(), "detach /tmp/bad-mount") || !strings.Contains(err.Error(), "detach boom") { + t.Fatalf("closeMount err = %v, want wrapped detach error", err) + } +} + +func TestCmdRunDefaultCaseSensitivePath(t *testing.T) { + withCSRunSeams(t) + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + runNowUnixNano = func() int64 { return 789 } + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, _ *store, ref, digest, size string) (*csMount, string, error) { + if cf.store != s.root || ref != "local:a" || size != "" { + t.Fatalf("ensure from cmdRun got store=%q ref=%q size=%q", cf.store, ref, size) + } + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("ensure digest = %q, want sha256 digest", digest) + } + return &csMount{mountPath: mount}, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + return os.MkdirAll(dst, 0o755) + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec) (int, error) { + if !strings.Contains(rootfs, fmt.Sprintf("run-%d-789", os.Getpid())) { + t.Fatalf("spawn rootfs = %q, want generated clone", rootfs) + } + if !reflect.DeepEqual(spec.Args, []string{"/image-cmd"}) { + t.Fatalf("spec args = %v, want image cmd", spec.Args) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(string, bool, *csMount) error { return nil } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("cmdRun default sparse path panic = %T %v, want exit 0", r, r) + } + }() + err := cmdRun([]string{"--store", s.root, "local:a"}) + t.Fatalf("cmdRun returned %v, want exit panic", err) +} diff --git a/cmd/elfuse-oci/csrun_other.go b/cmd/elfuse-oci/csrun_other.go new file mode 100644 index 00000000..b16d9b68 --- /dev/null +++ b/cmd/elfuse-oci/csrun_other.go @@ -0,0 +1,22 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !darwin + +package main + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// runCaseSensitive is the macOS case-sensitive sparsebundle + COW clone path. +// On non-Darwin there is no APFS/hdiutil/clonefile, and `run` is unusable +// anyway without Hypervisor.framework, so the default (case-sensitive) path +// reports a clear error and directs the user at --plain-rootfs. This stub +// exists so elfuse-oci compiles on Linux, where pull/inspect/unpack and +// conformance/interop tests run. +func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { + return fmt.Errorf("case-sensitive sparsebundle rootfs requires macOS; pass --plain-rootfs for a plain directory") +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index e6d10c5f..15407a27 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -63,6 +63,15 @@ run flags: are resolved against the image /etc/passwd and /etc/group before exec. --workdir DIR Guest-absolute initial working directory + --rootfs DIR Use an explicit rootfs directory (implies a plain dir, + no sparsebundle/clone lifecycle) + --plain-rootfs Use a plain directory rootfs instead of the default + macOS case-sensitive sparsebundle + --sparse-size SIZE Sparsebundle virtual size (default 16g; macOS only) + --no-clone Run against the base tree, skipping the per-run COW + clone (mutations persist; macOS only) + --keep Keep the per-run COW clone and mount for inspection + (macOS only) `) } diff --git a/cmd/elfuse-oci/run.go b/cmd/elfuse-oci/run.go index 653f5f00..50aaeb3b 100644 --- a/cmd/elfuse-oci/run.go +++ b/cmd/elfuse-oci/run.go @@ -6,6 +6,8 @@ package main import ( "fmt" "os" + "os/exec" + "os/signal" "path/filepath" "syscall" ) @@ -26,25 +28,17 @@ func elfuseBin() (string, error) { return filepath.Join(filepath.Dir(exe), "elfuse"), nil } -// execElfuse replaces this process with `elfuse --sysroot --user U:G -// --workdir D --clear-env --env K=V ... `. +// elfuseArgv builds the argv for `elfuse --sysroot --user U:G +// --workdir D --clear-env --env K=V ... -- `. // -// --clear-env plus every final env var as an explicit --env makes the guest -// see exactly the runspec env (image Env merged with --env overrides, per the +// --clear-env plus every final env var as an explicit --env makes the guest see +// exactly the runspec env (image Env merged with --env overrides, per the // precedence matrix) rather than the host environ. // -// syscall.Exec replaces elfuse-oci in place: the invoking shell reaps -// the same pid, and signals such as Ctrl-C go straight to elfuse rather than -// through a Go middleman. -func execElfuse(rootfs string, spec *runSpec) error { - bin, err := elfuseBin() - if err != nil { - return err - } - if _, err := os.Stat(bin); err != nil { - return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) - } - +// "--" ends elfuse's own option parsing: spec.Args comes from untrusted image +// config, so an Entrypoint beginning with "-" must reach the guest as its +// argv, not steer the host launcher (e.g. an image config carrying "--gdb"). +func elfuseArgv(rootfs string, spec *runSpec) []string { argv := []string{ "elfuse", "--sysroot", rootfs, @@ -55,10 +49,91 @@ func execElfuse(rootfs string, spec *runSpec) error { for _, e := range spec.Env { argv = append(argv, "--env", e) } + argv = append(argv, "--") argv = append(argv, spec.Args...) + return argv +} - if err := syscall.Exec(bin, argv, os.Environ()); err != nil { +// execElfuse replaces this process with elfuse (syscall.Exec). Used for the +// plain-rootfs path, which owns no mount to tear down: elfuse-oci +// becomes elfuse in place, so the invoking shell reaps the same pid and +// terminal signals such as Ctrl-C go straight to elfuse rather than through a +// Go middleman. +func execElfuse(rootfs string, spec *runSpec) error { + bin, err := elfuseBin() + if err != nil { + return err + } + if _, err := os.Stat(bin); err != nil { + return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) + } + if err := syscall.Exec(bin, elfuseArgv(rootfs, spec), os.Environ()); err != nil { return fmt.Errorf("exec %s: %w", bin, err) } return nil // unreachable } + +// spawnElfuseWait runs elfuse as a child and waits for it, returning the exit +// status the way a shell would (exit code, or 128+signal for signal death). +// Unlike execElfuse, elfuse-oci stays alive to reap the child, letting the +// case-sensitive path tear down its mount and COW clone after elfuse exits. +// +// The child shares this process's process group, so terminal signals (Ctrl-C) +// reach it directly; we additionally forward any such signal we receive to the +// child so a signal targeted at elfuse-oci alone still propagates, and we +// survive to reap and report the child's status rather than dying first. +func spawnElfuseWait(rootfs string, spec *runSpec) (int, error) { + bin, err := elfuseBin() + if err != nil { + return 0, err + } + if _, err := os.Stat(bin); err != nil { + return 0, fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) + } + // exec.Command uses `bin` as argv[0], so drop the leading "elfuse" + // program-name that elfuseArgv includes for syscall.Exec's sake; otherwise + // elfuse would see "elfuse" as its first positional and try to boot a + // guest path named "elfuse". + cmd := exec.Command(bin, elfuseArgv(rootfs, spec)[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // Intercept before Start so no window exists where a signal takes the + // default action and kills this wrapper between launching the child and + // entering the forward/reap loop; the channel buffers until then. SIGHUP + // is included so a terminal hangup also flows through the forward/reap + // path and the caller's mount/clone teardown still runs. + sigCh := make(chan os.Signal, 4) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, + syscall.SIGHUP) + defer signal.Stop(sigCh) + + if err := cmd.Start(); err != nil { + return 0, fmt.Errorf("spawn %s: %w", bin, err) + } + + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + + for { + select { + case err := <-done: + state := cmd.ProcessState + if state == nil { + return 0, err + } + if ws, ok := state.Sys().(syscall.WaitStatus); ok { + if ws.Signaled() { + return 128 + int(ws.Signal()), nil + } + return ws.ExitStatus(), nil + } + return state.ExitCode(), nil + case sig := <-sigCh: + if cmd.Process != nil { + _ = cmd.Process.Signal(sig) + } + } + } +} diff --git a/cmd/elfuse-oci/run_test.go b/cmd/elfuse-oci/run_test.go index f254182b..b8b9a2a7 100644 --- a/cmd/elfuse-oci/run_test.go +++ b/cmd/elfuse-oci/run_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "strings" "testing" ) @@ -25,6 +26,72 @@ func writeElfuseStub(t *testing.T, body string) string { return p } +// TestSpawnElfuseWaitExitCode verifies the child's exit code is returned as-is. +func TestSpawnElfuseWaitExitCode(t *testing.T) { + writeElfuseStub(t, "exit 42") + spec := &runSpec{Args: []string{"/hello"}, Workdir: "/", UID: 0, GID: 0} + code, err := spawnElfuseWait(t.TempDir(), spec) + if err != nil { + t.Fatalf("spawnElfuseWait: %v", err) + } + if code != 42 { + t.Errorf("exit code: got %d, want 42", code) + } +} + +// TestSpawnElfuseWaitSignalDeath verifies signal death is reported the +// shell-style way: 128 + signal. The child kills itself with SIGTERM (15), so +// cmd.Wait() observes WaitStatus.Signaled() independently of elfuse-oci's own +// signal forwarding. +func TestSpawnElfuseWaitSignalDeath(t *testing.T) { + writeElfuseStub(t, "kill -TERM $$") + spec := &runSpec{Args: []string{"/hello"}, Workdir: "/", UID: 0, GID: 0} + code, err := spawnElfuseWait(t.TempDir(), spec) + if err != nil { + t.Fatalf("spawnElfuseWait: %v", err) + } + if code != 143 { // 128 + SIGTERM(15) + t.Errorf("signal death: got %d, want 143 (128+15)", code) + } +} + +// TestElfuseArgvShape verifies the argv handed to elfuse is exactly +// elfuseArgv(rootfs, spec) minus the leading "elfuse" program-name (exec.Command +// prepends the binary path as argv[0], so spawnElfuseWait drops elfuseArgv[0]). +// The stub records its own argv ($@, which excludes $0) one per line. +func TestElfuseArgvShape(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "argv.txt") + t.Setenv("ELFUSE_ARGV_OUT", outPath) + writeElfuseStub(t, `printf '%s\n' "$@" > "$ELFUSE_ARGV_OUT"`) + + rootfs := t.TempDir() + spec := &runSpec{ + Args: []string{"/bin/echo", "hi"}, + Env: []string{"A=1", "B=2"}, + Workdir: "/work", + UID: 1000, + GID: 1000, + } + wantArgv := elfuseArgv(rootfs, spec)[1:] + + code, err := spawnElfuseWait(rootfs, spec) + if err != nil { + t.Fatalf("spawnElfuseWait: %v", err) + } + if code != 0 { + t.Fatalf("stub exited %d, want 0", code) + } + + data, err := os.ReadFile(outPath) + if err != nil { + t.Fatalf("read argv out: %v", err) + } + gotLines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + if !reflect.DeepEqual(gotLines, wantArgv) { + t.Errorf("argv:\n got %v\nwant %v", gotLines, wantArgv) + } +} + func TestElfuseBinEnvAndFallback(t *testing.T) { want := filepath.Join(t.TempDir(), "elfuse-custom") t.Setenv("ELFUSE_BIN", want) @@ -95,3 +162,46 @@ func TestExecElfuseSuccessSubprocess(t *testing.T) { } } } + +func TestSpawnElfuseWaitMissingAndStartErrors(t *testing.T) { + t.Setenv("ELFUSE_BIN", filepath.Join(t.TempDir(), "missing-elfuse")) + if _, err := spawnElfuseWait(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}); err == nil || + !strings.Contains(err.Error(), "elfuse binary not found") { + t.Fatalf("spawn missing binary err = %v, want not found", err) + } + + nonExecutable := filepath.Join(t.TempDir(), "elfuse-not-executable") + if err := os.WriteFile(nonExecutable, []byte("#!/bin/sh\nexit 0\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("ELFUSE_BIN", nonExecutable) + if _, err := spawnElfuseWait(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}); err == nil || + !strings.Contains(err.Error(), "spawn") { + t.Fatalf("spawn start err = %v, want spawn error", err) + } +} + +// TestElfuseArgvSeparatesGuestArgs pins the "--" end-of-options marker: the +// guest command comes from untrusted image config, so an Entrypoint that +// begins with "-" must arrive as guest argv, not be parsed as an elfuse +// option by the host launcher. +func TestElfuseArgvSeparatesGuestArgs(t *testing.T) { + spec := &runSpec{ + Args: []string{"--gdb", "1234"}, + Workdir: "/", + } + argv := elfuseArgv("/rootfs", spec) + sep := -1 + for i, a := range argv { + if a == "--" { + sep = i + break + } + } + if sep < 0 { + t.Fatalf("argv %v carries no \"--\" separator before guest args", argv) + } + if !reflect.DeepEqual(argv[sep+1:], spec.Args) { + t.Fatalf("argv after -- = %v, want %v", argv[sep+1:], spec.Args) + } +} diff --git a/cmd/elfuse-oci/runspec.go b/cmd/elfuse-oci/runspec.go index 307de249..0b3b71a7 100644 --- a/cmd/elfuse-oci/runspec.go +++ b/cmd/elfuse-oci/runspec.go @@ -48,6 +48,12 @@ type runFlags struct { user string workdir string rootfs string + + // Case-sensitive sparsebundle and COW clone controls. + plainRootfs bool // --plain-rootfs: skip the sparsebundle, use a plain dir + sparseSize string // --sparse-size SIZE: sparsebundle virtual size (default 16g) + noClone bool // --no-clone: run against the base tree, no COW clone + keepRootfs bool // --keep: do not remove the COW clone on exit } // computeRunSpec applies the Entrypoint/Cmd/Env/WorkingDir/User precedence. diff --git a/cmd/elfuse-oci/sparsebundle.go b/cmd/elfuse-oci/sparsebundle.go new file mode 100644 index 00000000..056213ab --- /dev/null +++ b/cmd/elfuse-oci/sparsebundle.go @@ -0,0 +1,308 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "bytes" + "errors" + "fmt" + "html" + "os" + "os/exec" + "path/filepath" + "regexp" + "syscall" +) + +var isMountPointFn = isMountPoint + +// csMount is a case-sensitive APFS sparsebundle attached at a mount point. +// It mirrors the C sysroot_create_mount machinery in src/core/sysroot.c so +// the guest rootfs is case-sensitive (the host volume is not), fixing +// the case-collision limitation of a plain-directory rootfs. +type csMount struct { + mountPath string // where the volume is attached + owned bool // we attached (or share) it; tear down on Close + bundleDir string // bundle directory holding the lock files; "" = no locking (unit tests) + runLock *flockFile // shared liveness lock held for this run's lifetime +} + +// defaultSparseSize is the sparsebundle's virtual size. APFS sparsebundles are +// sparse, so this is a ceiling, not preallocation; 16g matches the C side and +// comfortably covers base images (the actual disk use is the unpacked size). +const defaultSparseSize = "16g" + +// provisionCaseSensitive creates (if absent) and attaches a case-sensitive +// APFS sparsebundle. The unpacked base tree lives at /rootfs and +// persists in the sparsebundle image file across attach/detach cycles, so warm +// re-runs skip the unpack. The caller must Close the returned mount (which +// detaches it when this is the last live run) when done. +// +// Locking (see bundlelock.go): the whole provision runs under an exclusive +// attach.lock, and the returned mount holds run.lock shared until Close, so +// this run is visible to prune/rmi sweeps from BEFORE the volume is +// attached: there is no window in which the mount exists but no liveness +// marker does. An attached leftover mount is detached only after winning the +// run.lock exclusive probe, which proves no live run is executing out of it; +// when the probe reports busy the mount belongs to live runs of the same +// digest and is shared instead of ripped out from under them. +func provisionCaseSensitive(bundleDir, mountPath, size string) (*csMount, error) { + if size == "" { + size = defaultSparseSize + } + if err := os.MkdirAll(bundleDir, 0o755); err != nil { + return nil, err + } + image := filepath.Join(bundleDir, "rootfs.sparsebundle") + + attachLock, err := acquireFlock(attachLockPath(bundleDir), syscall.LOCK_EX) + if err != nil { + return nil, err + } + defer attachLock.Close() + + // Probe run.lock. Winning it exclusively proves zero live runs: any + // attached mount is stale (crash, kill, --keep) and safe to detach; hold + // the lock and downgrade to shared once provisioned (safe under + // attach.lock, see Downgrade). Losing the probe proves live runs exist: + // take it shared (which cannot block, since exclusive takers must hold + // attach.lock) and never detach. + staleDetachOK := false + runLock, err := acquireFlock(runLockPath(bundleDir), syscall.LOCK_EX|syscall.LOCK_NB) + switch { + case err == nil: + staleDetachOK = true + case errors.Is(err, errCacheBusy): + runLock, err = acquireFlock(runLockPath(bundleDir), syscall.LOCK_SH) + if err != nil { + return nil, err + } + default: + return nil, err + } + fail := func(err error) (*csMount, error) { + runLock.Close() + return nil, err + } + + if _, err := os.Stat(image); os.IsNotExist(err) { + out, err := exec.Command("hdiutil", "create", + "-fs", "Case-sensitive APFS", + "-size", size, + "-type", "SPARSEBUNDLE", + "-volname", "elfuse_sysroot", + image).CombinedOutput() + if err != nil { + return fail(fmt.Errorf("hdiutil create %s: %w: %s", image, err, out)) + } + } else if err != nil { + return fail(err) + } + + // Reject a symlinked mount path before any mount-status probe or detach: + // isMountPoint/detachForce follow the link (os.Stat) and could force-detach + // an unrelated volume. clearDir has the same guard, but only runs after the + // detach below. + if li, err := os.Lstat(mountPath); err == nil && li.Mode()&os.ModeSymlink != 0 { + return fail(fmt.Errorf("mount path %s is a symlink; refusing to detach/clear", mountPath)) + } + + if isMountPointFn(mountPath) { + if !staleDetachOK { + // Live runs of this digest own the attach; share it. + return &csMount{mountPath: mountPath, owned: true, bundleDir: bundleDir, runLock: runLock}, nil + } + // A prior run left the volume attached (crash, kill, --keep) and the + // won run.lock probe proves nothing is executing out of it: detach so + // we own a clean attach. + if err := detachForce(mountPath); err != nil { + return fail(fmt.Errorf("detach stale %s: %w", mountPath, err)) + } + } + // Ensure the mount point is an empty directory so hdiutil will mount onto + // it. + if err := clearDir(mountPath); err != nil { + return fail(err) + } + + // Keep stdout (the plist) separate from stderr: the failure message must + // carry hdiutil's diagnostic, which Output() alone would discard, while + // CombinedOutput() would corrupt the plist parse. + attach := exec.Command("hdiutil", "attach", + "-mountpoint", mountPath, "-plist", image) + var attachStderr bytes.Buffer + attach.Stderr = &attachStderr + out, err := attach.Output() + if err != nil { + return fail(fmt.Errorf("hdiutil attach %s: %w: %s%s", image, err, out, + attachStderr.Bytes())) + } + actual, err := parseMountpoint(string(out)) + if err != nil { + err = fmt.Errorf("parse attach plist for %s: %w", image, err) + return fail(detachAfterAttachError(mountPath, err)) + } + + if err := writeSpotlightMarker(actual); err != nil { + err = fmt.Errorf("spotlight marker: %w", err) + return fail(detachAfterAttachError(actual, err)) + } + if staleDetachOK { + if err := runLock.Downgrade(); err != nil { + return fail(detachAfterAttachError(actual, err)) + } + } + return &csMount{mountPath: actual, owned: true, bundleDir: bundleDir, runLock: runLock}, nil +} + +// rootfsDir is the base unpacked tree inside the volume. +func (m *csMount) rootfsDir() string { return filepath.Join(m.mountPath, "rootfs") } + +// Close releases this run's liveness lock and detaches the volume when this +// was the last live run of the digest (last-one-out): with concurrent runs +// sharing one attach, an unconditional detach here would rip the rootfs out +// from under the survivors. A csMount without a bundleDir (unit tests, +// hand-built mounts) has no locks to consult and detaches unconditionally. +func (m *csMount) Close() error { + if !m.owned { + return nil + } + if m.bundleDir == "" { + if err := detachForce(m.mountPath); err != nil { + return err + } + m.owned = false + return nil + } + // Take attach.lock BEFORE releasing our shared run.lock: it fences out a + // concurrent sweep, which could otherwise win both locks between our + // release and our probe and remove the bundle we are about to detach. If + // the lifecycle lock is busy (another provision or a sweep), its holder + // owns the mount's fate; just drop our liveness and go. + attachLock, err := acquireFlock(attachLockPath(m.bundleDir), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + m.runLock.Close() + m.runLock = nil + m.owned = false + if errors.Is(err, errCacheBusy) { + return nil + } + return err + } + defer attachLock.Close() + if err := m.runLock.Close(); err != nil { + m.owned = false + return err + } + m.runLock = nil + m.owned = false + runLock, err := acquireFlock(runLockPath(m.bundleDir), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + if errors.Is(err, errCacheBusy) { + // Other live runs share the attach; leave the volume to them. + return nil + } + return err + } + defer runLock.Close() + return detachForce(m.mountPath) +} + +func detachAfterAttachError(mountPath string, cause error) error { + if err := detachForce(mountPath); err != nil { + return errors.Join(cause, fmt.Errorf("detach %s: %w", mountPath, err)) + } + return cause +} + +var detachForce = func(mountPath string) error { + out, err := exec.Command("hdiutil", "detach", "-force", mountPath).CombinedOutput() + if err != nil { + return fmt.Errorf("hdiutil detach %s: %w: %s", mountPath, err, out) + } + return nil +} + +// writeSpotlightMarker drops .metadata_never_index so Spotlight does not index +// the (potentially large) rootfs volume. +func writeSpotlightMarker(mountPath string) error { + marker := filepath.Join(mountPath, ".metadata_never_index") + f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + return f.Close() +} + +// isMountPoint reports whether path is currently a mount point by comparing its +// device id against its parent's. +func isMountPoint(path string) bool { + if fi, err := os.Stat(path); err != nil || !fi.IsDir() { + return false + } + dev, ok := devOf(path) + if !ok { + return false + } + parent, ok := devOf(filepath.Dir(path)) + if !ok { + return false + } + return dev != parent +} + +func devOf(path string) (int64, bool) { + var st syscall.Stat_t + if err := syscall.Stat(path, &st); err != nil { + return 0, false + } + return int64(st.Dev), true +} + +// clearDir removes all children of dir (creating it if absent) without removing +// dir itself, so hdiutil can mount onto it. A symlink at dir is rejected: +// ReadDir/RemoveAll would follow it and empty the link's target instead of the +// mount point, so a corrupt or tampered store must fail here rather than +// delete files elsewhere. +func clearDir(dir string) error { + if li, err := os.Lstat(dir); err == nil { + if li.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("mount point %s is a symlink; refusing to clear it", dir) + } + } else if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } + } + return nil +} + +var mountpointRe = regexp.MustCompile(`mount-point\s*([^<]+)`) + +// parseMountpoint extracts the mount-point from an hdiutil attach -plist output +// (mirroring the C parse_attach_mountpoint string scan). The scan reads raw +// XML text content, so entity references must be decoded: a store path +// containing "&" or "'" is otherwise returned in its escaped form and every +// later use (markers, rootfs, detach) targets a nonexistent path. +// html.UnescapeString covers the XML predefined entities plus numeric +// references. +func parseMountpoint(plist string) (string, error) { + m := mountpointRe.FindStringSubmatch(plist) + if m == nil { + return "", fmt.Errorf("mount-point key not found in plist") + } + return html.UnescapeString(m[1]), nil +} diff --git a/cmd/elfuse-oci/sparsebundle_test.go b/cmd/elfuse-oci/sparsebundle_test.go new file mode 100644 index 00000000..01b027c8 --- /dev/null +++ b/cmd/elfuse-oci/sparsebundle_test.go @@ -0,0 +1,412 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// withDarwinCacheSeams swaps the darwin mount-probe and force-detach seams +// for a test and restores them on cleanup. +func withDarwinCacheSeams(t *testing.T, isMount func(string) bool, detach func(string) error) { + t.Helper() + oldIsMount := isMountPointFn + oldDetach := detachForce + if isMount != nil { + isMountPointFn = isMount + } + if detach != nil { + detachForce = detach + } + t.Cleanup(func() { + isMountPointFn = oldIsMount + detachForce = oldDetach + }) +} + +// hdiutil attach -plist output is an array of system entity dictionaries. We +// only care about the mount-point. This fixture is a trimmed-down capture of the +// real shape (system-entities -> dict -> mount-point key/string). +const attachPlistFixture = ` + + + + + system-entities + + + content-hint + Apple_APFS + dev-entry + /dev/disk3s1 + mount-point + /Volumes/elfuse_sysroot + potentially-mountable + 1 + + + + +` + +func TestParseMountpoint(t *testing.T) { + got, err := parseMountpoint(attachPlistFixture) + if err != nil { + t.Fatal(err) + } + if got != "/Volumes/elfuse_sysroot" { + t.Errorf("got %q, want /Volumes/elfuse_sysroot", got) + } +} + +func TestParseMountpointMissing(t *testing.T) { + if _, err := parseMountpoint(""); err == nil { + t.Fatal("expected error when mount-point absent") + } +} + +func TestParseMountpointWhitespaceBetweenKeyAndString(t *testing.T) { + in := `mount-point + /Volumes/x` + got, err := parseMountpoint(in) + if err != nil { + t.Fatal(err) + } + if got != "/Volumes/x" { + t.Errorf("got %q, want /Volumes/x", got) + } +} + +func TestIsMountPointPlainDir(t *testing.T) { + dir := t.TempDir() + if isMountPoint(dir) { + t.Errorf("fresh temp dir reported as mount point") + } +} + +func TestRemoveCloneRemovesDir(t *testing.T) { + clone := filepath.Join(t.TempDir(), "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + if err := removeClone(clone, false); err != nil { + t.Fatalf("removeClone: %v", err) + } + if _, err := os.Stat(clone); !os.IsNotExist(err) { + t.Fatalf("clone after removeClone: %v, want IsNotExist", err) + } +} + +func TestRemoveCloneKeepLeavesDir(t *testing.T) { + clone := filepath.Join(t.TempDir(), "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + if err := removeClone(clone, true); err != nil { + t.Fatalf("removeClone keep: %v", err) + } + if _, err := os.Stat(clone); err != nil { + t.Fatalf("clone after keep: %v, want present", err) + } +} + +func TestCSMountCloseReportsDetachError(t *testing.T) { + oldDetach := detachForce + t.Cleanup(func() { detachForce = oldDetach }) + detachForce = func(path string) error { + return fmt.Errorf("detach failed for %s", path) + } + + m := &csMount{mountPath: "/tmp/elfuse-test-mount", owned: true} + err := m.Close() + if err == nil || !strings.Contains(err.Error(), "detach failed") { + t.Fatalf("Close err = %v, want detach failure", err) + } + if !m.owned { + t.Fatal("Close cleared ownership after failed detach") + } +} + +func TestCSMount(t *testing.T) { + t.Run("rootfsDir", func(t *testing.T) { + m := &csMount{mountPath: "/tmp/elfuse-mounted", owned: true} + if got := m.rootfsDir(); got != "/tmp/elfuse-mounted/rootfs" { + t.Fatalf("rootfsDir = %q, want /tmp/elfuse-mounted/rootfs", got) + } + }) + + t.Run("close detaches once", func(t *testing.T) { + oldDetach := detachForce + var detached []string + detachForce = func(path string) error { + detached = append(detached, path) + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + m := &csMount{mountPath: "/tmp/elfuse-mounted", owned: true} + if err := m.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if m.owned { + t.Fatal("Close left mount owned after successful detach") + } + if len(detached) != 1 || detached[0] != "/tmp/elfuse-mounted" { + t.Fatalf("detached = %v, want [/tmp/elfuse-mounted]", detached) + } + if err := m.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } + if len(detached) != 1 { + t.Fatalf("second Close detached again: %v", detached) + } + }) + + t.Run("detachAfterAttachError joins both errors", func(t *testing.T) { + oldDetach := detachForce + detachForce = func(path string) error { return errors.New("detach failed") } + t.Cleanup(func() { detachForce = oldDetach }) + + err := detachAfterAttachError("/tmp/mnt", errors.New("attach parse failed")) + if err == nil || !strings.Contains(err.Error(), "attach parse failed") || !strings.Contains(err.Error(), "detach failed") { + t.Fatalf("detachAfterAttachError = %v, want joined cause and detach error", err) + } + }) +} + +func TestSparsebundleFilesystemHelpers(t *testing.T) { + dir := t.TempDir() + if err := writeSpotlightMarker(dir); err != nil { + t.Fatalf("writeSpotlightMarker: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, ".metadata_never_index")); err != nil { + t.Fatalf("spotlight marker missing: %v", err) + } + + childFile := filepath.Join(dir, "file") + childDir := filepath.Join(dir, "subdir") + if err := os.WriteFile(childFile, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(childDir, 0o755); err != nil { + t.Fatal(err) + } + if err := clearDir(dir); err != nil { + t.Fatalf("clearDir existing: %v", err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("clearDir left entries %v, want empty dir", entries) + } + + missing := filepath.Join(t.TempDir(), "created") + if err := clearDir(missing); err != nil { + t.Fatalf("clearDir missing: %v", err) + } + if fi, err := os.Stat(missing); err != nil || !fi.IsDir() { + t.Fatalf("clearDir missing produced fi=%v err=%v, want dir", fi, err) + } + + // A symlinked mount dir must be rejected, not followed: clearing through + // it would empty the link's target directory outside the OCI cache. + target := t.TempDir() + if err := os.WriteFile(filepath.Join(target, "precious"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "mnt") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if err := clearDir(link); err == nil { + t.Fatal("clearDir followed a symlinked mount dir, want error") + } + if _, err := os.Stat(filepath.Join(target, "precious")); err != nil { + t.Fatalf("symlink target contents were removed: %v", err) + } + + if _, ok := devOf(dir); !ok { + t.Fatal("devOf temp dir returned ok=false") + } + if _, ok := devOf(filepath.Join(dir, "does-not-exist")); ok { + t.Fatal("devOf missing path returned ok=true") + } +} + +func installFakeHdiutil(t *testing.T) { + t.Helper() + dir := t.TempDir() + script := filepath.Join(dir, "hdiutil") + body := `#!/bin/sh +case "$1" in +create) + if [ "${HDIUTIL_FAIL_CREATE:-}" = "1" ]; then + echo "create failed" + exit 7 + fi + for last do :; done + mkdir -p "$last" + exit 0 + ;; +attach) + if [ "${HDIUTIL_FAIL_ATTACH:-}" = "1" ]; then + echo "attach diagnostic on stderr" >&2 + exit 5 + fi + if [ "${HDIUTIL_BAD_PLIST:-}" = "1" ]; then + printf '' + exit 0 + fi + printf 'mount-point%s' "$HDIUTIL_MOUNT" + exit 0 + ;; +detach) + if [ -n "${HDIUTIL_DETACH_LOG:-}" ]; then + echo "$3" >> "$HDIUTIL_DETACH_LOG" + fi + exit 0 + ;; +*) + echo "unexpected hdiutil command $1" + exit 9 + ;; +esac +` + if err := os.WriteFile(script, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestProvisionCaseSensitiveWithFakeHdiutilSuccess(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + bundle := filepath.Join(t.TempDir(), "bundle") + requestedMount := filepath.Join(t.TempDir(), "requested-mount") + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("HDIUTIL_MOUNT", actualMount) + + m, err := provisionCaseSensitive(bundle, requestedMount, "32m") + if err != nil { + t.Fatalf("provisionCaseSensitive: %v", err) + } + if _, err := os.Stat(filepath.Join(bundle, "rootfs.sparsebundle")); err != nil { + t.Fatalf("sparsebundle image not created: %v", err) + } + if m.mountPath != actualMount || !m.owned { + t.Fatalf("mount = %+v, want actual mount and owned", m) + } + if _, err := os.Stat(filepath.Join(actualMount, ".metadata_never_index")); err != nil { + t.Fatalf("spotlight marker missing: %v", err) + } + if err := m.Close(); err != nil { + t.Fatalf("Close fake mount: %v", err) + } +} + +func TestProvisionCaseSensitiveWithFakeHdiutilFailures(t *testing.T) { + cases := []struct { + name string + // setup configures the fake hdiutil for this failure mode and returns the + // requested mount path plus the mount path the detach log must record + // ("" to skip the detach-log check). + setup func(t *testing.T, detachLog string) (requestedMount, wantDetach string) + wantErr []string + }{ + { + name: "create failure", + setup: func(t *testing.T, detachLog string) (string, string) { + t.Setenv("HDIUTIL_FAIL_CREATE", "1") + t.Setenv("HDIUTIL_MOUNT", t.TempDir()) + return filepath.Join(t.TempDir(), "mnt"), "" + }, + wantErr: []string{"hdiutil create", "create failed"}, + }, + { + // hdiutil writes its diagnostics to stderr, which must reach the + // error message: with a bare Output() the operator only sees + // "exit status N". + name: "attach failure surfaces hdiutil stderr", + setup: func(t *testing.T, detachLog string) (string, string) { + t.Setenv("HDIUTIL_FAIL_ATTACH", "1") + t.Setenv("HDIUTIL_MOUNT", t.TempDir()) + return filepath.Join(t.TempDir(), "mnt"), "" + }, + wantErr: []string{"hdiutil attach", "attach diagnostic on stderr"}, + }, + { + name: "bad attach plist detaches requested mount", + setup: func(t *testing.T, detachLog string) (string, string) { + t.Setenv("HDIUTIL_BAD_PLIST", "1") + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + requestedMount := filepath.Join(t.TempDir(), "requested") + return requestedMount, requestedMount + }, + wantErr: []string{"parse attach plist"}, + }, + { + name: "marker failure detaches actual mount", + setup: func(t *testing.T, detachLog string) (string, string) { + actualMount := filepath.Join(t.TempDir(), "missing-parent", "actual") + t.Setenv("HDIUTIL_MOUNT", actualMount) + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + return filepath.Join(t.TempDir(), "requested"), actualMount + }, + wantErr: []string{"spotlight marker"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + installFakeHdiutil(t) + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + detachLog := filepath.Join(t.TempDir(), "detach.log") + requestedMount, wantDetach := tc.setup(t, detachLog) + _, err := provisionCaseSensitive(filepath.Join(t.TempDir(), "bundle"), requestedMount, "32m") + if err == nil { + t.Fatalf("provisionCaseSensitive succeeded, want error containing %v", tc.wantErr) + } + for _, want := range tc.wantErr { + if !strings.Contains(err.Error(), want) { + t.Fatalf("err = %v, want substring %q", err, want) + } + } + if wantDetach != "" { + b, readErr := os.ReadFile(detachLog) + if readErr != nil { + t.Fatal(readErr) + } + if !strings.Contains(string(b), wantDetach) { + t.Fatalf("detach log = %q, want mount %s", b, wantDetach) + } + } + }) + } +} + +// TestParseMountpointDecodesXMLEntities pins entity decoding: hdiutil's plist +// escapes XML-special characters in the mount path (a store path may carry +// "&" or "'"), and the raw escaped form would make every later use of the +// path (markers, rootfs, detach) target a nonexistent location. +func TestParseMountpointDecodesXMLEntities(t *testing.T) { + in := `mount-point/tmp/a & b's store/mnt` + got, err := parseMountpoint(in) + if err != nil { + t.Fatal(err) + } + if want := "/tmp/a & b's store/mnt"; got != want { + t.Fatalf("parseMountpoint = %q, want %q", got, want) + } +} diff --git a/go.mod b/go.mod index d004d0b6..69efb907 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module github.com/sysprog21/elfuse go 1.26.4 -require github.com/google/go-containerregistry v0.21.7 +require ( + github.com/google/go-containerregistry v0.21.7 + golang.org/x/sys v0.46.0 +) require ( github.com/docker/cli v29.5.3+incompatible // indirect @@ -12,6 +15,5 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect gotest.tools/v3 v3.5.2 // indirect ) From 8a7e6ebe2a90da3b25b01feb3ceb81e175dd9303 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:33:29 +0200 Subject: [PATCH 07/10] Add OCI lifecycle commands and reachability GC Add list/images, rmi, and prune on top of the OCI image-layout store. rmi and prune use reachability GC: shared manifests, configs, and layers stay on disk while any remaining ref reaches them. Cache cleanup handles plain rootfs caches and macOS sparsebundle caches through platform-specific seams, with safety rules for active workloads: a prune without --all never touches a still-pinned digest's cache, and a cache a live run still uses is never reclaimed. Liveness is the same flock discipline on both cache kinds: the bundle's run.lock, and for the plain digest-keyed rootfs a sibling .lock that a run holds shared from before the existence probe until guest exit. The plain run path execs elfuse in place, so the lock descriptor is made exec-survivable: the kernel releases it exactly when the elfuse process exits, SIGKILL included. prune takes each lock exclusively non-blocking and skips busy caches (dry runs never advertise them); rmi refuses a busy cache even with --force, before any pin or descriptor is touched. The lock is a sibling of the cache dir, not inside it, because the dir is the guest's / and its existence is unpackImage's atomic-rename publication signal. prune's GC sweep and list's snapshot run under the store lock, closing the windows where a concurrent pull's fresh blobs could be reclaimed before their descriptor lands or a listing could observe a half-removed ref. prune scopes its GC-plus-cache sweep with store.withLock and drops the lock before the reporting output; list and rmi keep the explicit acquire-and-defer because their entire body is the critical section, so a closure would only add indentation. list reports the full os/arch/variant platform. Tests cover reachability GC (shared blobs, stale temp blobs, digest prefixes), the prune and rmi busy semantics for both cache kinds (live holder, stale lock file, dry-run visibility), lock survival across the exec boundary (FD_CLOEXEC clearing plus a child-process flock-lifetime proxy), which runs hold the lock at exec time, and end-to-end command wrappers over pull/run/list/rmi/prune. The darwin cache sweep is driven through the isMountPointFn and detachForce seams with reapSweepableClones deciding the reap set, so the sweep is testable without real disk images. --- cmd/elfuse-oci/bundlelock_test.go | 68 ++ cmd/elfuse-oci/cache_darwin.go | 320 +++++ cmd/elfuse-oci/cache_darwin_test.go | 613 ++++++++++ cmd/elfuse-oci/cache_key.go | 7 +- cmd/elfuse-oci/cache_other.go | 54 + cmd/elfuse-oci/clone_sweep.go | 145 +++ cmd/elfuse-oci/commands.go | 116 +- cmd/elfuse-oci/commands_integration_test.go | 578 +++++++++ cmd/elfuse-oci/common_test.go | 47 +- cmd/elfuse-oci/csrun.go | 73 +- cmd/elfuse-oci/csrun_darwin_test.go | 171 ++- cmd/elfuse-oci/csrun_other.go | 2 +- cmd/elfuse-oci/gc.go | 415 +++++++ cmd/elfuse-oci/inspect.go | 16 +- cmd/elfuse-oci/inspect_test.go | 74 ++ cmd/elfuse-oci/lifecycle_darwin_test.go | 119 ++ cmd/elfuse-oci/lifecycle_test.go | 1217 +++++++++++++++++++ cmd/elfuse-oci/list.go | 146 +++ cmd/elfuse-oci/main.go | 29 +- cmd/elfuse-oci/main_command_test.go | 69 ++ cmd/elfuse-oci/prune.go | 225 ++++ cmd/elfuse-oci/rmi.go | 59 + cmd/elfuse-oci/rootfslock.go | 168 +++ cmd/elfuse-oci/run.go | 22 +- cmd/elfuse-oci/run_test.go | 2 +- cmd/elfuse-oci/sparsebundle_test.go | 151 ++- cmd/elfuse-oci/store.go | 191 ++- cmd/elfuse-oci/store_test.go | 303 +++++ cmd/elfuse-oci/test_helpers_test.go | 2 +- cmd/elfuse-oci/unpack.go | 80 +- cmd/elfuse-oci/unpack_image_test.go | 94 +- cmd/elfuse-oci/unpack_test.go | 12 +- 32 files changed, 5430 insertions(+), 158 deletions(-) create mode 100644 cmd/elfuse-oci/cache_darwin.go create mode 100644 cmd/elfuse-oci/cache_darwin_test.go create mode 100644 cmd/elfuse-oci/cache_other.go create mode 100644 cmd/elfuse-oci/clone_sweep.go create mode 100644 cmd/elfuse-oci/commands_integration_test.go create mode 100644 cmd/elfuse-oci/gc.go create mode 100644 cmd/elfuse-oci/lifecycle_darwin_test.go create mode 100644 cmd/elfuse-oci/lifecycle_test.go create mode 100644 cmd/elfuse-oci/list.go create mode 100644 cmd/elfuse-oci/prune.go create mode 100644 cmd/elfuse-oci/rmi.go create mode 100644 cmd/elfuse-oci/rootfslock.go diff --git a/cmd/elfuse-oci/bundlelock_test.go b/cmd/elfuse-oci/bundlelock_test.go index 21e9fe18..45351a5e 100644 --- a/cmd/elfuse-oci/bundlelock_test.go +++ b/cmd/elfuse-oci/bundlelock_test.go @@ -6,9 +6,12 @@ package main import ( "errors" "os" + "os/exec" "path/filepath" "syscall" "testing" + + "golang.org/x/sys/unix" ) func TestAcquireFlockSharedCoexists(t *testing.T) { @@ -120,3 +123,68 @@ func TestBundleLockPaths(t *testing.T) { t.Fatalf("runLockPath = %q", got) } } + +// TestPreserveAcrossExecClearsCloexec pins the exec-survival half of the +// plain-rootfs run lock: Go opens files close-on-exec, so without the +// FD_SETFD clear the flock would silently drop at syscall.Exec. +func TestPreserveAcrossExecClearsCloexec(t *testing.T) { + l, err := acquireFlock(filepath.Join(t.TempDir(), "x.lock"), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer l.Close() + flags, err := unix.FcntlInt(l.f.Fd(), unix.F_GETFD, 0) + if err != nil { + t.Fatal(err) + } + if flags&unix.FD_CLOEXEC == 0 { + t.Fatal("lock fd unexpectedly not close-on-exec before PreserveAcrossExec") + } + if err := l.PreserveAcrossExec(); err != nil { + t.Fatal(err) + } + flags, err = unix.FcntlInt(l.f.Fd(), unix.F_GETFD, 0) + if err != nil { + t.Fatal(err) + } + if flags&unix.FD_CLOEXEC != 0 { + t.Fatal("FD_CLOEXEC still set after PreserveAcrossExec") + } +} + +// TestRootfsLockSurvivesIntoChildProcess models the exec handoff with the +// closest testable analog: hand the lock's descriptor to a child via +// ExtraFiles (a dup shares the open file description, exactly like exec +// inheritance), drop the parent's fd WITHOUT unlocking, and require the +// flock to live precisely as long as the child: released by the kernel at +// kill, no unlock code run. An in-process syscall.Exec is untestable by +// construction; this pins the same kernel behavior the run path relies on. +func TestRootfsLockSurvivesIntoChildProcess(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cache") + hold, err := acquireRootfsRunLock(dir) + if err != nil { + t.Fatal(err) + } + child := exec.Command("/bin/sleep", "30") + child.ExtraFiles = []*os.File{hold.f} + if err := child.Start(); err != nil { + t.Fatal(err) + } + // Close the parent's fd only (no LOCK_UN), mirroring the process image + // being replaced. The child's dup keeps the description locked. + if err := hold.f.Close(); err != nil { + t.Fatal(err) + } + hold.f = nil + + if !rootfsCacheBusy(dir) { + t.Error("lock not held while child lives; exec inheritance would drop it") + } + if err := child.Process.Kill(); err != nil { + t.Fatal(err) + } + _ = child.Wait() + if rootfsCacheBusy(dir) { + t.Error("lock still held after child death; kernel should have released it") + } +} diff --git a/cmd/elfuse-oci/cache_darwin.go b/cmd/elfuse-oci/cache_darwin.go new file mode 100644 index 00000000..755fb7af --- /dev/null +++ b/cmd/elfuse-oci/cache_darwin.go @@ -0,0 +1,320 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "syscall" +) + +// On Darwin an unpacked cache can be either (or both) of: +// - a case-sensitive APFS sparsebundle bundle at cs/// holding the +// warm unpacked base tree (image file rootfs.sparsebundle + mount point mnt), +// - a plain rootfs// directory (the --plain-rootfs path). +// +// cacheExists / removeRefCaches / pruneCaches are the lifecycle seam the +// cross-platform gc.go/rmi.go/prune.go code calls; the darwin versions add the +// sparsebundle lifecycle (detach a still-mounted volume before removing its +// bundle directory) on top of the shared rootfs sweep. + +// cacheHasKeptData reports whether digest's cache holds run --keep retained +// output. A --keep run drops the `kept` sidecar beside the sparsebundle (outside +// the mounted volume, like the bundle flocks), so this is a cheap stat that does +// not need to attach a cold, detached bundle to inspect its clones. rmi refuses +// to reclaim such a cache without force. +func cacheHasKeptData(root, digest string) (bool, error) { + bundle, err := csBundleDirForDigest(root, digest) + if err != nil { + // An unparseable digest key has no bundle and so no kept data; a real + // rmi target is always a valid digest. + return false, nil + } + if _, err := os.Stat(keptSidecarPath(bundle)); err == nil { + return true, nil + } else if !os.IsNotExist(err) { + return false, err + } + return false, nil +} + +// cacheExists reports whether digest has any unpacked cache under the store: the +// case-sensitive sparsebundle bundle and/or the plain rootfs directory. +func cacheExists(root, digest string) bool { + bundle, err := csBundleDirForDigest(root, digest) + if err == nil { + if _, err := os.Stat(bundle); err == nil { + return true + } + } + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + return false + } + if _, err := os.Stat(rootfs); err == nil { + return true + } + return false +} + +// removeRefCaches deletes digest's unpacked caches: the case-sensitive +// sparsebundle and the plain rootfs, when a digest has both. A crash leftover +// mount is recovered (orphan clones reaped, stale volume detached) so the +// bundle can be removed, but a volume that still hosts a live run's clone, or a +// plain rootfs a live --plain-rootfs run holds, refuses: dropping the cache +// (via rmi) means "reclaim derived state", not "rip the rootfs out from under a +// running guest". +// +// Both cache locks are preflighted before either form is deleted, so a busy +// side leaves BOTH caches intact. Deleting the sparsebundle first and only then +// discovering the plain rootfs is busy (or vice versa) would strand a +// half-deleted cache under a still-live pin. The plain reference lock is taken +// first, matching a run's acquisition order (reference lock, then bundle +// locks); both acquisitions are non-blocking, so the order cannot deadlock. +func removeRefCaches(s *store, digest string) error { + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + return err + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return err + } + busyErr := fmt.Errorf("cache for %s is in use by a live run; stop it before removing the image", digest) + + // Preflight the plain rootfs lock. An ENOENT (no rootfs/ scaffolding) means + // no plain cache and no plain run can exist, so there is nothing to hold. + rootfsUnlock, rootfsBusy, err := lockRootfsCacheForRemoval(rootfs) + haveRootfsLock := err == nil + if err != nil && !os.IsNotExist(err) { + return err + } + if rootfsBusy { + return busyErr + } + if haveRootfsLock { + defer rootfsUnlock() + } + + if _, err := os.Stat(bundle); err == nil { + _, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + return err + } + if busy { + // The plain lock (if held) is released by the deferred unlock; no + // cache was deleted, so the refusal leaves both forms intact. + return busyErr + } + // Hold the bundle locks across the removal: a concurrent run's + // provision would otherwise race in between the sweep and the + // RemoveAll and lose its freshly attached volume. + err = os.RemoveAll(bundle) + unlock() + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + + // Plain rootfs, removed under the lock already held from the preflight. + if haveRootfsLock { + if err := os.RemoveAll(rootfs); err != nil { + return err + } + if err := os.Remove(rootfsCacheLockPath(rootfs)); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil +} + +// pruneCaches drops elfuse's unpacked caches. Without opts.all, only caches for +// refs no longer pinned (orphan caches) are dropped; with opts.all, every +// cache. The plain rootfs sweep is shared (pruneRootfsCaches); the darwin-only +// sparsebundle sweep walks cs/// plus legacy cs// directories, +// detaching a still-mounted volume before removing its bundle. The bytes +// reported for a sparsebundle are the on-disk allocation of its image file +// (dirSize of rootfs.sparsebundle), not the 16g virtual ceiling and not a live +// mount's contents. +func (s *store) pruneCaches(opts pruneOpts) (pruneReport, error) { + live, err := s.liveCacheKeys() + if err != nil { + return pruneReport{}, err + } + rep, err := pruneRootfsCaches(s, live, opts) + if err != nil { + return rep, err + } + + csBase := filepath.Join(s.root, csCacheDirName) + entries, err := os.ReadDir(csBase) + if err != nil { + if os.IsNotExist(err) { + return rep, nil + } + return rep, err + } + for _, e := range entries { + if !e.IsDir() { + continue + } + top := filepath.Join(csBase, e.Name()) + if e.Name() == "sha256" { + children, err := os.ReadDir(top) + if err != nil { + return rep, err + } + for _, child := range children { + if !child.IsDir() { + continue + } + key := filepath.Join("sha256", child.Name()) + bundle := filepath.Join(top, child.Name()) + var err error + rep, err = pruneCSBundle(rep, bundle, key, live, opts) + if err != nil { + return rep, err + } + } + continue + } + + // Legacy ref-named sparsebundle caches are no longer live under the + // digest-keyed scheme; prune --cache reclaims them as orphan caches. + var err error + rep, err = pruneCSBundle(rep, top, "", live, opts) + if err != nil { + return rep, err + } + } + return rep, nil +} + +func pruneCSBundle(rep pruneReport, bundle, key string, live map[string]bool, opts pruneOpts) (pruneReport, error) { + // A still-pinned digest is off-limits to a non---all prune BEFORE any + // sweep: its attached volume may belong to an active run, and + // sweepCSBundle cannot tell a crashed leftover mount from a live one by + // mount state alone. A crashed pinned bundle's stale mount is recovered + // by the next run's provision (which re-attaches cleanly) or by an + // explicit prune --cache --all. + if key != "" && !opts.all && live[key] { + return rep, nil + } + + // Crash recovery: if a killed run left this bundle's volume attached, + // reap orphan COW clones inside it and detach the stale mount. If a live + // run still owns a clone in the volume, leave the whole bundle alone; + // force-detaching would rip the rootfs out from under that guest + // (reachable with --all, or via a legacy/unpinned bundle). A dry-run + // makes the same decisions read-only: it reports the orphan clones a + // real prune would reap and skips a busy bundle a real prune would + // leave, but never detaches or removes anything. + if opts.dryRun { + // A busy bundle (a live run holds run.lock) is left alone, exactly as + // a real prune would; otherwise report the clones a real sweep would + // reap. Read-only: probe the locks and list, never detach or remove. + if csBundleBusy(bundle) { + return rep, nil + } + mnt := filepath.Join(bundle, "mnt") + if isMountPointFn(mnt) { + rep.CacheDirs = append(rep.CacheDirs, listSweepableClones(mnt)...) + } + image := filepath.Join(bundle, "rootfs.sparsebundle") + rep.Bytes += dirSize(image) + rep.CacheDirs = append(rep.CacheDirs, bundle) + return rep, nil + } + + reaped, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + return rep, err + } + if len(reaped) > 0 { + rep.CacheDirs = append(rep.CacheDirs, reaped...) + } + if busy { + return rep, nil + } + + image := filepath.Join(bundle, "rootfs.sparsebundle") + rep.Bytes += dirSize(image) + // sweepCSBundle already detached a stale mount if there was one, and holds + // the bundle locks so a concurrent provision cannot re-populate the bundle + // between here and the removal. + err = os.RemoveAll(bundle) + unlock() + if err != nil { + return rep, err + } + rep.CacheDirs = append(rep.CacheDirs, bundle) + return rep, nil +} + +// sweepCSBundle performs crash recovery for one sparsebundle bundle under the +// per-bundle advisory flocks. It acquires attach.lock and run.lock +// exclusively (non-blocking): if either is held, a live run (or an in-flight +// provision) owns the bundle, so it returns busy=true without touching +// anything. Holding run.lock exclusively proves no run is executing out of the +// volume, so every clone left inside is abandoned by construction: reap the +// sweepable ones (all but --keep-marked clones, plus unpack leftovers) and +// detach a still-attached stale mount. +// +// On success it returns the reaped clone directories and an unlock func that +// releases both locks. The caller must invoke unlock AFTER it finishes with +// the bundle (typically after os.RemoveAll), so a concurrent provision cannot +// re-attach and re-populate the bundle in the gap. On busy or error the +// returned unlock is a non-nil no-op, so callers may always defer it. +func sweepCSBundle(bundle string) (reaped []string, busy bool, unlock func(), err error) { + noop := func() {} + attachLock, err := acquireFlock(attachLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + if isCacheBusy(err) { + return nil, true, noop, nil + } + return nil, false, noop, err + } + runLock, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + attachLock.Close() + if isCacheBusy(err) { + return nil, true, noop, nil + } + return nil, false, noop, err + } + release := func() { + runLock.Close() + attachLock.Close() + } + + mnt := filepath.Join(bundle, "mnt") + // Reject a symlinked mount path before any mount-status probe, clone reap, + // or detach: isMountPoint (os.Stat) and detachForce follow the link, so a + // tampered store with mnt symlinked at an unrelated attached volume would + // otherwise get that volume's contents reaped and the volume force-detached. + // provisionCaseSensitive guards its own attach path the same way; the + // destructive sweep (prune --cache, rmi) needs the guard too. + if li, err := os.Lstat(mnt); err == nil && li.Mode()&os.ModeSymlink != 0 { + release() + return nil, false, noop, fmt.Errorf("mount path %s is a symlink; refusing to detach/reap", mnt) + } + if isMountPointFn(mnt) { + reaped = reapSweepableClones(mnt) + if err := detachForce(mnt); err != nil { + release() + return reaped, false, noop, fmt.Errorf("detach %s: %w", mnt, err) + } + } + return reaped, false, release, nil +} + +func isCacheBusy(err error) bool { + return errors.Is(err, errCacheBusy) +} diff --git a/cmd/elfuse-oci/cache_darwin_test.go b/cmd/elfuse-oci/cache_darwin_test.go new file mode 100644 index 00000000..2c753613 --- /dev/null +++ b/cmd/elfuse-oci/cache_darwin_test.go @@ -0,0 +1,613 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +func withDarwinCacheSeams(t *testing.T, isMount func(string) bool, detach func(string) error) { + t.Helper() + oldIsMount := isMountPointFn + oldDetach := detachForce + if isMount != nil { + isMountPointFn = isMount + } + if detach != nil { + detachForce = detach + } + t.Cleanup(func() { + isMountPointFn = oldIsMount + detachForce = oldDetach + }) +} + +// holdRunLock takes a shared run.lock on the bundle for the test's lifetime, +// simulating a live run so a sweep sees the bundle busy. +func holdRunLock(t *testing.T, bundle string) { + t.Helper() + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + l, err := acquireFlock(runLockPath(bundle), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) +} + +func writeSparseBundleMarker(t *testing.T, bundle string) { + t.Helper() + image := filepath.Join(bundle, "rootfs.sparsebundle") + if err := os.MkdirAll(image, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(image, "band"), []byte("data"), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestDarwinCacheExistsBundleAndPlainRootfs(t *testing.T) { + root := t.TempDir() + digest := "sha256:" + strings.Repeat("1", 64) + if cacheExists(root, digest) { + t.Fatal("cacheExists returned true for absent cache") + } + + bundle, err := csBundleDirForDigest(root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + if !cacheExists(root, digest) { + t.Fatal("cacheExists returned false for sparsebundle cache") + } + if err := os.RemoveAll(bundle); err != nil { + t.Fatal(err) + } + + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + if !cacheExists(root, digest) { + t.Fatal("cacheExists returned false for plain rootfs cache") + } + if cacheExists(root, "not-a-digest") { + t.Fatal("cacheExists returned true for invalid digest") + } +} + +func TestDarwinCacheHasKeptData(t *testing.T) { + root := t.TempDir() + digest := "sha256:" + strings.Repeat("6", 64) + + if kept, err := cacheHasKeptData(root, digest); err != nil || kept { + t.Fatalf("cacheHasKeptData absent = (%v, %v), want (false, nil)", kept, err) + } + + bundle, err := csBundleDirForDigest(root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + // A bundle from a normal (non-keep) run has no sidecar: not kept. + if kept, err := cacheHasKeptData(root, digest); err != nil || kept { + t.Fatalf("cacheHasKeptData no-sidecar = (%v, %v), want (false, nil)", kept, err) + } + + if err := os.WriteFile(keptSidecarPath(bundle), nil, 0o644); err != nil { + t.Fatal(err) + } + if kept, err := cacheHasKeptData(root, digest); err != nil || !kept { + t.Fatalf("cacheHasKeptData with sidecar = (%v, %v), want (true, nil)", kept, err) + } +} + +// TestDarwinRmiRefusesKeptCacheWithoutForce pins the one case rmi still refuses: +// a bundle holding run --keep retained output is not discarded without --force, +// and --force then drops the whole bundle and removes the image. +func TestDarwinRmiRefusesKeptCacheWithoutForce(t *testing.T) { + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + digest, err := s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keptSidecarPath(bundle), nil, 0o644); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err == nil || !strings.Contains(err.Error(), "keep") { + t.Fatalf("rmi kept cache without --force err = %v, want --keep refusal", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Fatalf("pin lost after refused rmi: %v", err) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("bundle removed after refused rmi: %v, want present", err) + } + + rep, err := s.rmi("local:a", true) + if err != nil { + t.Fatalf("rmi --force kept cache: %v", err) + } + if !rep.CacheDropped { + t.Error("rmi --force did not report dropping the cache") + } + if _, err := os.Stat(bundle); !os.IsNotExist(err) { + t.Fatalf("bundle after rmi --force: %v, want removed", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("pin present after rmi --force, want gone") + } +} + +func TestDarwinRemoveRefCachesDropsBundleAndRootfs(t *testing.T) { + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("2", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + if err := removeRefCaches(s, digest); err != nil { + t.Fatalf("removeRefCaches: %v", err) + } + for _, p := range []string{bundle, rootfs} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Fatalf("%s after removeRefCaches: %v, want IsNotExist", p, err) + } + } +} + +func TestDarwinRemoveRefCachesDetachesMountedBundle(t *testing.T) { + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("3", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + if err := os.MkdirAll(mnt, 0o755); err != nil { + t.Fatal(err) + } + var detached string + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(path string) error { + detached = path + return nil + }, + ) + + // No run holds run.lock, so the still-attached mount is stale: the sweep + // detaches it and the bundle is removed. + if err := removeRefCaches(s, digest); err != nil { + t.Fatalf("removeRefCaches: %v", err) + } + if detached != mnt { + t.Fatalf("detached = %q, want %q", detached, mnt) + } + if _, err := os.Stat(bundle); !os.IsNotExist(err) { + t.Fatalf("bundle after removeRefCaches: %v, want IsNotExist", err) + } +} + +func TestDarwinPruneCachesDropsOrphanAndLegacyCSBundles(t *testing.T) { + withDarwinCacheSeams(t, func(string) bool { return false }, nil) + s := openTestStore(t) + liveDigest := "sha256:" + strings.Repeat("4", 64) + orphanDigest := "sha256:" + strings.Repeat("5", 64) + if err := s.savePins(refPins{"live": liveDigest}); err != nil { + t.Fatal(err) + } + liveBundle, _ := csBundleDirForDigest(s.root, liveDigest) + orphanBundle, _ := csBundleDirForDigest(s.root, orphanDigest) + legacyBundle := filepath.Join(s.root, "cs", "legacy_ref") + for _, bundle := range []string{liveBundle, orphanBundle, legacyBundle} { + writeSparseBundleMarker(t, bundle) + } + + rep, err := s.pruneCaches(pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + if !sliceContains(rep.CacheDirs, orphanBundle) || !sliceContains(rep.CacheDirs, legacyBundle) { + t.Fatalf("pruneCaches dirs = %v, want orphan and legacy bundles", rep.CacheDirs) + } + if sliceContains(rep.CacheDirs, liveBundle) { + t.Fatalf("pruneCaches dropped live bundle %s: %v", liveBundle, rep.CacheDirs) + } + if _, err := os.Stat(orphanBundle); !os.IsNotExist(err) { + t.Fatalf("orphan bundle after prune: %v, want IsNotExist", err) + } + if _, err := os.Stat(legacyBundle); !os.IsNotExist(err) { + t.Fatalf("legacy bundle after prune: %v, want IsNotExist", err) + } + if _, err := os.Stat(liveBundle); err != nil { + t.Fatalf("live bundle after prune: %v, want present", err) + } +} + +// TestDarwinPruneCSBundleDryRunDoesNotSweepOrDelete pins that a dry-run makes +// the same decisions as a real prune (report the clones a real sweep would +// reap, skip a busy bundle) while never detaching or removing anything. The +// busy check is a real (non-blocking) probe of the bundle locks. +func TestDarwinPruneCSBundleDryRunDoesNotSweepOrDelete(t *testing.T) { + key := filepath.Join("sha256", strings.Repeat("6", 64)) + dryRun := func(t *testing.T, bundle string) pruneReport { + t.Helper() + rep, err := pruneCSBundle(pruneReport{}, bundle, key, nil, pruneOpts{cache: true, dryRun: true}) + if err != nil { + t.Fatalf("pruneCSBundle dry-run: %v", err) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("dry-run removed bundle: %v", err) + } + return rep + } + noDetach := func(t *testing.T, isMount func(string) bool) { + t.Helper() + withDarwinCacheSeams(t, isMount, func(string) error { + t.Fatal("detachForce called during dry-run") + return nil + }) + } + + t.Run("unmounted bundle reported", func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + noDetach(t, func(string) bool { return false }) + + rep := dryRun(t, bundle) + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != bundle { + t.Fatalf("dry-run dirs = %v, want [%s]", rep.CacheDirs, bundle) + } + }) + + t.Run("sweepable clone reported", func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + mnt := filepath.Join(bundle, "mnt") + orphan := filepath.Join(mnt, "run-42-1") + if err := os.MkdirAll(orphan, 0o755); err != nil { + t.Fatal(err) + } + noDetach(t, func(path string) bool { return path == mnt }) + + rep := dryRun(t, bundle) + if len(rep.CacheDirs) != 2 || rep.CacheDirs[0] != orphan || rep.CacheDirs[1] != bundle { + t.Fatalf("dry-run dirs = %v, want [%s %s]", rep.CacheDirs, orphan, bundle) + } + if rep.Bytes == 0 { + t.Fatal("dry-run Bytes = 0, want the bundle's on-disk size counted") + } + if _, err := os.Stat(orphan); err != nil { + t.Fatalf("dry-run reaped %s, want read-only: %v", orphan, err) + } + }) + + t.Run("busy bundle skipped", func(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + mnt := filepath.Join(bundle, "mnt") + noDetach(t, func(path string) bool { return path == mnt }) + holdRunLock(t, bundle) // a live run holds run.lock + + rep := dryRun(t, bundle) + if len(rep.CacheDirs) != 0 { + t.Fatalf("dry-run dirs = %v, want empty for a busy bundle", rep.CacheDirs) + } + if rep.Bytes != 0 { + t.Fatalf("dry-run Bytes = %d, want 0 for a busy bundle", rep.Bytes) + } + }) +} + +// TestDarwinSweepCSBundleIdleReapsAndHoldsLocks pins that an idle bundle +// (no run holds run.lock) is swept: a mounted volume's sweepable clones are +// reaped, the stale mount detached, and the returned unlock releases the +// bundle locks the sweep held for the caller's removal. +func TestDarwinSweepCSBundleIdleReapsAndHoldsLocks(t *testing.T) { + unmounted := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(unmounted, 0o755); err != nil { + t.Fatal(err) + } + withDarwinCacheSeams(t, func(string) bool { return false }, func(string) error { + t.Fatal("detachForce called for non-mount") + return nil + }) + reaped, busy, unlock, err := sweepCSBundle(unmounted) + if err != nil { + t.Fatalf("sweepCSBundle no mount: %v", err) + } + if busy || len(reaped) != 0 { + t.Fatalf("sweepCSBundle no mount = (reaped %v, busy %v), want idle empty", reaped, busy) + } + // While the sweep holds the locks, a would-be run's exclusive probe fails. + if _, err := acquireFlock(runLockPath(unmounted), syscall.LOCK_EX|syscall.LOCK_NB); !errorsIsCacheBusy(err) { + t.Fatalf("run.lock during sweep err = %v, want held", err) + } + unlock() + if free, err := acquireFlock(runLockPath(unmounted), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + t.Fatalf("run.lock after unlock err = %v, want free", err) + } else { + free.Close() + } + + bundle := filepath.Join(t.TempDir(), "bundle") + mnt := filepath.Join(bundle, "mnt") + clone := filepath.Join(mnt, "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + var detached string + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(path string) error { detached = path; return nil }, + ) + reaped, busy, unlock, err = sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle mounted: %v", err) + } + defer unlock() + if busy { + t.Fatal("sweepCSBundle mounted-but-idle reported busy") + } + if len(reaped) != 1 || reaped[0] != clone { + t.Fatalf("mounted reaped = %v, want [%s]", reaped, clone) + } + if detached != mnt { + t.Fatalf("detached = %q, want %q", detached, mnt) + } + if _, err := os.Stat(clone); !os.IsNotExist(err) { + t.Fatalf("sweepable clone not reaped: %v", err) + } +} + +// TestDarwinSweepCSBundleBusySkips pins that a bundle whose run.lock a live +// run holds is reported busy and NOT force-detached. +func TestDarwinSweepCSBundleBusySkips(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + mnt := filepath.Join(bundle, "mnt") + if err := os.MkdirAll(mnt, 0o755); err != nil { + t.Fatal(err) + } + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(string) error { + t.Fatal("detachForce called although a live run holds the volume") + return nil + }, + ) + holdRunLock(t, bundle) + + reaped, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle: %v", err) + } + defer unlock() + if !busy { + t.Fatal("sweepCSBundle did not report busy for a live run") + } + if len(reaped) != 0 { + t.Fatalf("reaped = %v, want empty", reaped) + } +} + +// TestDarwinPruneCSBundleSkipsLivePinnedBeforeSweep pins the guard order: a +// still-pinned digest's bundle is skipped by a non---all prune before any +// sweep runs, so an active run's mount is never probed or detached. +func TestDarwinPruneCSBundleSkipsLivePinnedBeforeSweep(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + writeSparseBundleMarker(t, bundle) + key := filepath.Join("sha256", strings.Repeat("7", 64)) + withDarwinCacheSeams(t, + func(string) bool { + t.Fatal("isMountPoint probed for a live pinned bundle") + return false + }, + func(string) error { + t.Fatal("detachForce called for a live pinned bundle") + return nil + }, + ) + + rep, err := pruneCSBundle(pruneReport{}, bundle, key, map[string]bool{key: true}, pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneCSBundle: %v", err) + } + if len(rep.CacheDirs) != 0 || rep.Bytes != 0 { + t.Fatalf("live pinned bundle was touched: %+v", rep) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("live pinned bundle missing after prune: %v", err) + } +} + +// TestDarwinPruneCSBundleLeavesBusyBundle pins that even when the sweep runs +// (e.g. --all), a bundle whose volume hosts a live run is left in place. +func TestDarwinPruneCSBundleLeavesBusyBundle(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + mnt := filepath.Join(bundle, "mnt") + writeSparseBundleMarker(t, bundle) + key := filepath.Join("sha256", strings.Repeat("8", 64)) + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(string) error { + t.Fatal("detachForce called although a live run holds the volume") + return nil + }, + ) + holdRunLock(t, bundle) + + rep, err := pruneCSBundle(pruneReport{}, bundle, key, map[string]bool{key: true}, pruneOpts{cache: true, all: true}) + if err != nil { + t.Fatalf("pruneCSBundle: %v", err) + } + if len(rep.CacheDirs) != 0 { + t.Fatalf("busy bundle reported as pruned: %v", rep.CacheDirs) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("busy bundle missing after prune: %v", err) + } +} + +// TestDarwinRemoveRefCachesRefusesLiveRun pins the rmi --force guard: a +// volume whose run.lock a live run holds must refuse cache removal instead of +// force-detaching the guest's rootfs. +func TestDarwinRemoveRefCachesRefusesLiveRun(t *testing.T) { + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("7", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + if err := os.MkdirAll(mnt, 0o755); err != nil { + t.Fatal(err) + } + detached := false + withDarwinCacheSeams(t, + func(path string) bool { return path == mnt }, + func(string) error { + detached = true + return nil + }, + ) + holdRunLock(t, bundle) + + err = removeRefCaches(s, digest) + if err == nil || !strings.Contains(err.Error(), "in use by a live run") { + t.Fatalf("removeRefCaches err = %v, want live-run refusal", err) + } + if detached { + t.Fatal("removeRefCaches force-detached a live run's volume") + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("bundle after refusal: %v, want untouched", err) + } +} + +// TestDarwinSweepCSBundleRejectsSymlinkedMnt pins S8: a tampered store whose +// mnt is a symlink at an unrelated volume must be refused before any mount +// probe, clone reap, or detach, so the sweep cannot force-detach or wipe that +// volume. provisionCaseSensitive guards its attach path the same way. +func TestDarwinSweepCSBundleRejectsSymlinkedMnt(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(bundle, 0o755); err != nil { + t.Fatal(err) + } + victim := filepath.Join(t.TempDir(), "unrelated-volume") + if err := os.MkdirAll(victim, 0o755); err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + if err := os.Symlink(victim, mnt); err != nil { + t.Fatal(err) + } + withDarwinCacheSeams(t, + func(string) bool { + t.Fatal("isMountPoint probed a symlinked mnt") + return false + }, + func(string) error { + t.Fatal("detachForce followed a symlinked mnt") + return nil + }, + ) + + _, _, unlock, err := sweepCSBundle(bundle) + if unlock != nil { + unlock() + } + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("sweepCSBundle symlinked mnt err = %v, want symlink refusal", err) + } + if _, err := os.Lstat(mnt); err != nil { + t.Fatalf("symlink mnt disturbed: %v", err) + } +} + +// TestDarwinRemoveRefCachesBusyPlainRootfsLeavesBundle pins #16: when a digest +// has both cache forms and a live --plain-rootfs run holds the plain rootfs +// lock, removeRefCaches must refuse before deleting either form, so the +// sparsebundle is not left half-removed under a still-live pin. The bundle +// survives and the plain rootfs is untouched. +func TestDarwinRemoveRefCachesBusyPlainRootfsLeavesBundle(t *testing.T) { + withDarwinCacheSeams(t, + func(string) bool { return false }, + func(string) error { + t.Fatal("detachForce called although the plain rootfs run is live") + return nil + }, + ) + s := &store{root: t.TempDir()} + digest := "sha256:" + strings.Repeat("9", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + writeSparseBundleMarker(t, bundle) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + // A live --plain-rootfs run holds the plain rootfs lock shared. + hold, err := acquireRootfsRunLock(rootfs) + if err != nil { + t.Fatal(err) + } + defer hold.Close() + + err = removeRefCaches(s, digest) + if err == nil || !strings.Contains(err.Error(), "in use by a live run") { + t.Fatalf("removeRefCaches err = %v, want live-run refusal", err) + } + if _, err := os.Stat(bundle); err != nil { + t.Fatalf("sparsebundle removed despite refusal (half-deleted cache): %v", err) + } + if _, err := os.Stat(rootfs); err != nil { + t.Fatalf("plain rootfs removed despite refusal: %v", err) + } +} + +// errorsIsCacheBusy reports whether err is the bundle-busy sentinel. +func errorsIsCacheBusy(err error) bool { return isCacheBusy(err) } diff --git a/cmd/elfuse-oci/cache_key.go b/cmd/elfuse-oci/cache_key.go index a59597c8..792890af 100644 --- a/cmd/elfuse-oci/cache_key.go +++ b/cmd/elfuse-oci/cache_key.go @@ -20,8 +20,11 @@ const ( // legacyCacheNameForRef reproduces the pre-digest cache naming scheme, which // flattened the ref itself into a single (intentionally lossy) path -// component. New caches are keyed by digest (cacheKeyForDigest); this helper -// survives to document the old layout. +// component. New caches are keyed by digest (cacheKeyForDigest). prune +// --cache recognizes legacy caches purely by their top-level directory name +// (anything under rootfs/ or cs/ that is not "sha256"), not through this +// helper; it survives to document the old layout and is exercised only by +// tests that fabricate legacy caches. func legacyCacheNameForRef(ref string) string { return strings.NewReplacer("/", "_", ":", "_", "@", "_").Replace(ref) } diff --git a/cmd/elfuse-oci/cache_other.go b/cmd/elfuse-oci/cache_other.go new file mode 100644 index 00000000..4e095657 --- /dev/null +++ b/cmd/elfuse-oci/cache_other.go @@ -0,0 +1,54 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !darwin + +package main + +import "os" + +// On non-Darwin the case-sensitive sparsebundle path is unavailable (no APFS, +// no hdiutil, no clonefile), so an unpacked cache is only ever the plain +// rootfs// directory. The lifecycle primitives (rmi, prune) touch +// caches through cacheExists / removeRefCaches / pruneCaches so the pure-Go +// blob GC and the rootfs cache sweep build and test on Linux CI; the darwin +// sparsebundle sweep lives in cache_darwin.go. + +// cacheHasKeptData reports whether digest's cache holds run --keep retained +// output. On non-Darwin the only cache is a plain rootfs directory with no +// per-run COW clones, so there is never retained output to protect: rmi always +// reclaims it. +func cacheHasKeptData(root, digest string) (bool, error) { + return false, nil +} + +// cacheExists reports whether digest has an unpacked cache under the store. On +// non-Darwin this is just the plain rootfs directory. +func cacheExists(root, digest string) bool { + rootfs, err := defaultRootfsForDigest(root, digest) + if err != nil { + return false + } + if _, err := os.Stat(rootfs); err == nil { + return true + } + return false +} + +// removeRefCaches deletes digest's unpacked cache(s). On non-Darwin, the plain +// rootfs directory only, refusing while a live run holds its per-digest lock. +func removeRefCaches(s *store, digest string) error { + return removeRootfsCacheForDigest(s, digest) +} + +// pruneCaches drops elfuse's unpacked caches. Without opts.all, only caches for +// refs no longer pinned (orphan caches) are dropped; with opts.all, every +// cache. On non-Darwin only plain rootfs// directories exist; the +// rootfs sweep is shared via pruneRootfsCaches. +func (s *store) pruneCaches(opts pruneOpts) (pruneReport, error) { + live, err := s.liveCacheKeys() + if err != nil { + return pruneReport{}, err + } + return pruneRootfsCaches(s, live, opts) +} diff --git a/cmd/elfuse-oci/clone_sweep.go b/cmd/elfuse-oci/clone_sweep.go new file mode 100644 index 00000000..08d31d7e --- /dev/null +++ b/cmd/elfuse-oci/clone_sweep.go @@ -0,0 +1,145 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "syscall" +) + +// Per-run COW clones of the warm base tree live inside an attached sparsebundle +// volume as run-- directories (see csrun.go). A crashed unpack +// can also leave a rootfs.tmp- staging directory behind (see +// unpackImage). Liveness is decided by the per-bundle advisory flocks +// (bundlelock.go), never by pids: a sweep only reaps once it holds run.lock +// exclusively, which proves no run is executing out of the volume, so every +// clone it finds is abandoned by construction. Pid parsing is gone; pid +// reuse can no longer make a still-wanted clone look dead. +// +// A run started with --keep records a keep sidecar for its clone so the sweep +// preserves it; without the sidecar every run-* clone is reapable. +// +// These helpers are pure path/lock logic with no darwin-specific calls, so +// they build and unit-test on Linux; sweepCSBundle (which needs isMountPoint + +// detachForce) is darwin-only and lives in cache_darwin.go. + +// keepDirName is a mount-root directory holding one marker file per kept clone, +// named by clone directory name. It lives in the mount root, BESIDE the clones, +// never inside any clone: a clone directory is a guest's /, so a marker there +// would collide with an image that ships its own /.elfuse-keep and could be +// forged by the guest at runtime, either of which would make an ordinary run's +// clone masquerade as kept and leak past every sweep. The mount root is outside +// every guest's --sysroot view, so markers here are writable only by +// elfuse-oci. A sweep skips a clone whose marker is present; only +// whole-bundle removal (prune of an unpinned/--all bundle, rmi --force) +// reclaims a kept clone, and that deletes this directory with it. +const keepDirName = ".elfuse-keep" + +// keepDirPath returns the mount-root keep-marker directory. +func keepDirPath(mountPath string) string { + return filepath.Join(mountPath, keepDirName) +} + +// cloneKeepMarkerPath returns the keep marker path for the clone named +// cloneName under mountPath. +func cloneKeepMarkerPath(mountPath, cloneName string) string { + return filepath.Join(keepDirPath(mountPath), cloneName) +} + +// writeKeepMarker records that the COW clone at cloneDir should survive sweeps, +// via a marker in the mount-root keep directory, so a later sweep preserves it +// after the creating run has exited and released run.lock. +func writeKeepMarker(cloneDir string) error { + mountPath := filepath.Dir(cloneDir) + if err := os.MkdirAll(keepDirPath(mountPath), 0o755); err != nil { + return err + } + f, err := os.OpenFile(cloneKeepMarkerPath(mountPath, filepath.Base(cloneDir)), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + return f.Close() +} + +// isRunCloneDir reports whether a directory entry name is a per-run COW clone +// (run--). The pid is not parsed; only the naming matters. +func isRunCloneDir(name string) bool { + return strings.HasPrefix(name, "run-") +} + +// isUnpackTempDir reports whether name is a leftover unpack staging directory +// (rootfs.tmp-) from a crashed unpackImage into the volume. +func isUnpackTempDir(name string) bool { + return strings.HasPrefix(name, "rootfs.tmp-") +} + +// listSweepableClones returns the reapable directories inside mountPath: every +// run-- clone WITHOUT a keep marker, plus any rootfs.tmp-* unpack +// leftover. The caller must hold run.lock exclusively (no live run), so a +// clone lacking a keep marker is guaranteed abandoned. Removes nothing; it is +// the read-only detection half shared with prune --dry-run. +func listSweepableClones(mountPath string) []string { + entries, err := os.ReadDir(mountPath) + if err != nil { + return nil + } + var sweepable []string + for _, e := range entries { + name := e.Name() + switch { + case isRunCloneDir(name): + if !e.IsDir() { + continue + } + // Preserve the clone unless its keep marker is DEFINITIVELY + // absent: err == nil means present, and a transient non-ENOENT + // error (e.g. EIO on the mounted volume) must not cause a + // --keep clone to be reaped; fail safe toward preservation. + if _, err := os.Stat(cloneKeepMarkerPath(mountPath, name)); !os.IsNotExist(err) { + continue + } + sweepable = append(sweepable, filepath.Join(mountPath, name)) + case isUnpackTempDir(name): + sweepable = append(sweepable, filepath.Join(mountPath, name)) + } + } + return sweepable +} + +// reapSweepableClones removes the directories listSweepableClones names. +// Removal is best-effort: a busy entry (e.g. still unmounting) is skipped and +// not reported. Returns the directories that were removed. The caller must +// hold run.lock exclusively. +func reapSweepableClones(mountPath string) []string { + var reaped []string + for _, dir := range listSweepableClones(mountPath) { + if err := os.RemoveAll(dir); err != nil { + continue // busy or evaporating; leave it for next time + } + reaped = append(reaped, dir) + } + return reaped +} + +// csBundleBusy reports whether a live run holds the bundle via a non-blocking +// exclusive probe of run.lock. It is the read-only busy check used by prune +// --dry-run and diagnostics; it acquires and immediately releases, mutating +// nothing. It deliberately does NOT touch attach.lock: attach.lock is a +// lifecycle lock whose holders (provision, sweep, a run's last-one-out Close) +// each own the mount's detach fate, and a read-only prober transiently +// holding it would fool a concurrent Close into skipping its detach (leaving +// the volume attached with no owner). Any error other than a clean +// acquisition fails closed (busy) so a dry-run never advertises a reap it +// could not safely perform. +func csBundleBusy(bundle string) bool { + r, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + return true + } + r.Close() + return false +} diff --git a/cmd/elfuse-oci/commands.go b/cmd/elfuse-oci/commands.go index 54539a4f..f30e37a5 100644 --- a/cmd/elfuse-oci/commands.go +++ b/cmd/elfuse-oci/commands.go @@ -7,12 +7,65 @@ import ( "errors" "fmt" "os" + + "github.com/google/go-containerregistry/pkg/v1" ) // The four subcommands share common-flag parsing (common.go) and the OCI // image-layout store (store.go). pull/unpack/inspect are pure store ops; run // additionally resolves the runspec and execs elfuse (run.go, csrun.go). +// afterImageResolve fires just after resolveImageForUse returns, while the +// caller holds the per-digest reference lock but no store lock. Production +// no-op; tests inject a concurrent repull here to prove the caller keeps using +// the image it resolved (keyed by the locked digest) and not a repinned one. +var afterImageResolve = func(digest string) {} + +// resolveImageForUse resolves ref to its pinned image and returns it with the +// manifest digest and a held per-digest reference lock (the digest's plain +// rootfs run lock, taken shared). Resolution and lock acquisition happen in +// one store-locked critical section, so the digest returned is exactly the +// digest locked: a concurrent repull (which needs the store lock to move the +// pin) cannot slip between the two, and the caller can key digest-scoped +// caches without a later re-resolution poisoning them. The reference lock +// then marks the image in use for the caller's lifetime: rmi probes it (and +// refuses) before dropping the last pin's descriptor and blobs, even before +// any cache dir exists, so a cold run that has resolved but not yet unpacked +// is never GC'd out from under. +// +// The SH acquisition cannot block: the only EX takers (rmi, prune) hold the +// store lock, which this holds, so no EX holder can exist meanwhile. +func resolveImageForUse(s *store, ref string) (v1.Image, string, *flockFile, error) { + var img v1.Image + var digest string + var lock *flockFile + err := s.withLock(func() error { + var err error + img, err = s.image(ref) + if err != nil { + return err + } + // Digest reads the manifest blob; do it under the lock so the blob + // cannot vanish mid-read. + d, err := img.Digest() + if err != nil { + return err + } + digest = d.String() + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return err + } + lock, err = acquireRootfsRunLock(rootfs) + return err + }) + if err != nil { + return nil, "", nil, err + } + afterImageResolve(digest) + return img, digest, lock, nil +} + // cmdPull implements `elfuse-oci pull [--store] [--platform] `. func cmdPull(args []string) error { cf, ref, err := parsePullArgs(args) @@ -46,21 +99,28 @@ func cmdUnpack(args []string) error { if err != nil { return err } - digest, err := s.digestFor(ref) + // resolveImageForUse pairs the image with the digest lock, so the cache + // keyed below by this digest is filled from this image even if the ref is + // repulled mid-unpack, and a concurrent rmi/prune cannot reclaim the tree + // (or the blobs the unpack still reads) mid-merge. + img, digest, lock, err := resolveImageForUse(s, ref) if err != nil { return err } + defer lock.Close() if rootfs == "" { rootfs, err = defaultRootfsForDigest(cf.store, digest) if err != nil { return err } } - fmt.Printf("Unpacking %s -> %s\n", ref, rootfs) - if err := unpackImage(s, ref, rootfs); err != nil { + // Progress goes to stderr, like every other report in this CLI; stdout + // is reserved for command output. + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rootfs) + if err := unpackImage(img, rootfs); err != nil { return err } - fmt.Printf("Unpacked %s\n", ref) + fmt.Fprintf(os.Stderr, "Unpacked %s\n", ref) return nil } @@ -115,23 +175,24 @@ func cmdRun(args []string) error { if err != nil { return err } - img, err := s.image(ref) - if err != nil { + // Resolve under the store lock and come back holding the per-digest run + // lock: from here to guest exit, rmi cannot reclaim this image's + // descriptor, blobs, or caches out from under the setup. + img, digestStr, refLock, err := resolveImageForUse(s, ref) + if errors.Is(err, errNotPulled) { // Auto-pull only when the ref is simply absent, so `run` is // self-sufficient on first use. Any other failure (corrupt refs.json, // unreadable layout) must surface rather than mask itself behind a // fresh network pull. - if !errors.Is(err, errNotPulled) { - return err - } if err := pullImage(cf, s, ref); err != nil { return err } - img, err = s.image(ref) - if err != nil { - return err - } + img, digestStr, refLock, err = resolveImageForUse(s, ref) + } + if err != nil { + return err } + defer refLock.Close() cfg, err := img.ConfigFile() if err != nil { return err @@ -149,12 +210,6 @@ func cmdRun(args []string) error { ref, got, want, want, ref) } } - digest, err := img.Digest() - if err != nil { - return err - } - digestStr := digest.String() - // Choose the rootfs path. Default: a case-sensitive APFS sparsebundle so // the guest's case-sensitive filenames don't collide on the host's // case-insensitive volume, with a per-run COW clone for isolation and @@ -163,15 +218,30 @@ func cmdRun(args []string) error { useCS := rf.rootfs == "" && !rf.plainRootfs if useCS { - return runCaseSensitive(cf, s, ref, digestStr, cfg, rf, tail) + // refLock stays held: runCaseSensitive keeps this process alive across + // the guest (spawnElfuseWait), so the reference lock lives for the + // whole run and rmi sees the digest in use even before the bundle + // exists. + return runCaseSensitive(cf, s, ref, img, digestStr, cfg, rf, tail) } - // Plain-directory path. + // Plain-directory path. The reference lock resolveImageForUse holds IS the + // per-digest run lock the store-managed cache is swept under, so a store + // cache run reuses it: it was taken before the existence probe, so the + // cache cannot be reclaimed between the probe and the guest's first read, + // and execElfuse threads its descriptor through the exec so the lock lives + // exactly as long as the guest. An explicit --rootfs is user-managed and + // never store-swept, so it takes no lock (release the reference lock: the + // image is fully resolved and an explicit rootfs is not digest-keyed). + var rootfsLock *flockFile if rf.rootfs == "" { rf.rootfs, err = defaultRootfsForDigest(cf.store, digestStr) if err != nil { return err } + rootfsLock = refLock + } else { + refLock.Close() } // Ensure the rootfs is unpacked before computing the spec, because // resolveUser reads /etc/passwd and /etc/group. Re-unpack only if @@ -181,7 +251,7 @@ func cmdRun(args []string) error { return err } fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rf.rootfs) - if err := unpackImage(s, ref, rf.rootfs); err != nil { + if err := unpackImage(img, rf.rootfs); err != nil { return err } } @@ -196,7 +266,7 @@ func cmdRun(args []string) error { if err := injectRuntimeFiles(rf.rootfs); err != nil { return err } - return execElfuseForRun(rf.rootfs, spec) + return execElfuseForRun(rf.rootfs, spec, rootfsLock) } func parseRunArgs(args []string) (commonFlags, runFlags, string, []string, error) { diff --git a/cmd/elfuse-oci/commands_integration_test.go b/cmd/elfuse-oci/commands_integration_test.go new file mode 100644 index 00000000..6db7ca47 --- /dev/null +++ b/cmd/elfuse-oci/commands_integration_test.go @@ -0,0 +1,578 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1" +) + +func withFakeCranePull(t *testing.T, fn func(string, ...crane.Option) (v1.Image, error)) { + t.Helper() + old := cranePull + cranePull = fn + t.Cleanup(func() { cranePull = old }) +} + +func withFakeExecElfuse(t *testing.T, fn func(string, *runSpec, *flockFile) error) { + t.Helper() + old := execElfuseForRun + execElfuseForRun = fn + t.Cleanup(func() { execElfuseForRun = old }) +} + +func TestCmdPullPinsImageOffline(t *testing.T) { + root := t.TempDir() + img := tinyImage(t) + wantDigest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + var gotRef string + var gotOptions int + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + gotRef = ref + gotOptions = len(opts) + return img, nil + }) + + stdout, stderr, err := captureOutput(t, func() error { + return cmdPull([]string{"--store", root, "--platform", "linux/amd64", "local:tiny"}) + }) + if err != nil { + t.Fatalf("cmdPull: %v", err) + } + if stdout != "" { + t.Fatalf("cmdPull stdout = %q, want empty", stdout) + } + if !strings.Contains(stderr, "Pulled local:tiny -> "+wantDigest.String()) { + t.Fatalf("cmdPull stderr = %q, want pull summary", stderr) + } + if gotRef != "local:tiny" || gotOptions != 1 { + t.Fatalf("fake crane.Pull got ref=%q options=%d, want local:tiny and platform option", gotRef, gotOptions) + } + + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + gotDigest, err := s.digestFor("local:tiny") + if err != nil { + t.Fatal(err) + } + if gotDigest != wantDigest.String() { + t.Fatalf("pin digest = %s, want %s", gotDigest, wantDigest) + } +} + +func TestCmdPullWrapsPullError(t *testing.T) { + root := t.TempDir() + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + return nil, errors.New("registry unavailable") + }) + + _, _, err := captureOutput(t, func() error { + return cmdPull([]string{"--store", root, "local:missing"}) + }) + if err == nil || !strings.Contains(err.Error(), "pull local:missing") || !strings.Contains(err.Error(), "registry unavailable") { + t.Fatalf("cmdPull error = %v, want wrapped pull error", err) + } +} + +func TestCmdListInspectRmiAndPruneWrappers(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + stdout, stderr, err := captureOutput(t, func() error { + return cmdList([]string{"--store", s.root}) + }) + if err != nil { + t.Fatalf("cmdList: %v", err) + } + if stderr != "" || !strings.Contains(stdout, "local:a") || !strings.Contains(stdout, "linux/arm64") { + t.Fatalf("cmdList stdout=%q stderr=%q, want list row", stdout, stderr) + } + listedDigest := shortDigest(manifest.String()) + if !strings.Contains(stdout, listedDigest) { + t.Fatalf("cmdList stdout=%q, want digest %s", stdout, listedDigest) + } + + stdout, stderr, err = captureOutput(t, func() error { + return cmdInspect([]string{"--store", s.root, "--json", "local:a"}) + }) + if err != nil { + t.Fatalf("cmdInspect --json: %v", err) + } + if stderr != "" || !strings.Contains(stdout, `"architecture": "arm64"`) || !strings.HasSuffix(stdout, "\n") { + t.Fatalf("cmdInspect stdout=%q stderr=%q, want JSON config with trailing newline", stdout, stderr) + } + + orphan := writeOrphanBlob(t, s.root, "command-prune-orphan") + stdout, stderr, err = captureOutput(t, func() error { + return cmdPrune([]string{"--store", s.root, "--dry-run"}) + }) + if err != nil { + t.Fatalf("cmdPrune --dry-run: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Would reclaim: 1 blob(s)") || !strings.Contains(stderr, orphan) { + t.Fatalf("cmdPrune dry-run stdout=%q stderr=%q, want dry-run summary", stdout, stderr) + } + if _, err := os.Stat(blobPath(s.root, orphan)); err != nil { + t.Fatalf("dry-run removed orphan blob: %v", err) + } + + stdout, stderr, err = captureOutput(t, func() error { + return cmdPrune([]string{"--store", s.root}) + }) + if err != nil { + t.Fatalf("cmdPrune: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Reclaimed: 1 blob(s)") { + t.Fatalf("cmdPrune stdout=%q stderr=%q, want reclaim summary", stdout, stderr) + } + if _, err := os.Stat(blobPath(s.root, orphan)); !os.IsNotExist(err) { + t.Fatalf("orphan blob after prune: %v, want IsNotExist", err) + } + + stdout, stderr, err = captureOutput(t, func() error { + return cmdRmi([]string{"--store", s.root, listedDigest}) + }) + if err != nil { + t.Fatalf("cmdRmi: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Removed local:a:") { + t.Fatalf("cmdRmi stdout=%q stderr=%q, want removal summary", stdout, stderr) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after cmdRmi") + } +} + +func TestCmdUnpackWrapperExplicitAndDefaultRootfs(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + digest, err := s.addImage("local:tiny", img) + if err != nil { + t.Fatal(err) + } + + explicit := filepath.Join(t.TempDir(), "explicit-rootfs") + stdout, stderr, err := captureOutput(t, func() error { + return cmdUnpack([]string{"--store", s.root, "--rootfs", explicit, "local:tiny"}) + }) + if err != nil { + t.Fatalf("cmdUnpack explicit: %v", err) + } + if stdout != "" || !strings.Contains(stderr, "Unpacking local:tiny -> "+explicit) || !strings.Contains(stderr, "Unpacked local:tiny") { + t.Fatalf("cmdUnpack explicit stdout=%q stderr=%q", stdout, stderr) + } + if b, err := os.ReadFile(filepath.Join(explicit, "hello")); err != nil || string(b) != "world" { + t.Fatalf("explicit rootfs hello = %q, err=%v; want world", b, err) + } + + defaultRootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + stdout, stderr, err = captureOutput(t, func() error { + return cmdUnpack([]string{"--store", s.root, "local:tiny"}) + }) + if err != nil { + t.Fatalf("cmdUnpack default: %v", err) + } + if stdout != "" || !strings.Contains(stderr, defaultRootfs) { + t.Fatalf("cmdUnpack default stdout=%q stderr=%q, want default rootfs path", stdout, stderr) + } + if b, err := os.ReadFile(filepath.Join(defaultRootfs, "hello")); err != nil || string(b) != "world" { + t.Fatalf("default rootfs hello = %q, err=%v; want world", b, err) + } +} + +func TestCmdRunPlainRootfsUnpacksInjectsAndExecs(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(t.TempDir(), "rootfs") + var gotRootfs string + var gotSpec *runSpec + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + gotRootfs = rootfs + gotSpec = spec + if b, err := os.ReadFile(filepath.Join(rootfs, "hello")); err != nil || string(b) != "world" { + t.Fatalf("rootfs hello = %q, err=%v; want world before exec", b, err) + } + for _, name := range []string{"hostname", "hosts", "resolv.conf"} { + if _, err := os.Stat(filepath.Join(rootfs, "etc", name)); err != nil { + t.Fatalf("runtime file %s missing before exec: %v", name, err) + } + } + return nil + }) + + stderrExpected := "Unpacking local:a -> " + rootfs + stdout, stderr, err := captureOutput(t, func() error { + return cmdRun([]string{ + "--store", s.root, + "--plain-rootfs", + "--rootfs", rootfs, + "--env", "A=2", + "local:a", + "/cli-cmd", "arg", + }) + }) + if err != nil { + t.Fatalf("cmdRun --plain-rootfs: %v", err) + } + if stdout != "" || !strings.Contains(stderr, stderrExpected) { + t.Fatalf("cmdRun stdout=%q stderr=%q, want unpack message %q", stdout, stderr, stderrExpected) + } + if gotRootfs != rootfs { + t.Fatalf("exec rootfs = %q, want %q", gotRootfs, rootfs) + } + if gotSpec == nil { + t.Fatal("exec spec was nil") + } + if !reflect.DeepEqual(gotSpec.Args, []string{"/cli-cmd", "arg"}) { + t.Fatalf("spec args = %v, want CLI tail", gotSpec.Args) + } + if !reflect.DeepEqual(gotSpec.Env, []string{"A=2", "PATH=" + defaultGuestPath}) { + t.Fatalf("spec env = %v, want [A=2] plus default PATH", gotSpec.Env) + } +} + +func TestCmdRunPlainRootfsEntrypointOverride(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(t.TempDir(), "rootfs") + var gotSpec *runSpec + withFakeExecElfuse(t, func(_ string, spec *runSpec, _ *flockFile) error { + gotSpec = spec + return nil + }) + + _, _, err := captureOutput(t, func() error { + return cmdRun([]string{ + "--store", s.root, "--plain-rootfs", "--rootfs", rootfs, + "--entrypoint", "/override", "local:a", "x", "y", + }) + }) + if err != nil { + t.Fatalf("cmdRun --entrypoint: %v", err) + } + if gotSpec == nil { + t.Fatal("exec spec was nil") + } + // --entrypoint replaces the image Entrypoint AND drops the image Cmd; the + // CLI tail becomes the new Cmd. + if want := []string{"/override", "x", "y"}; !reflect.DeepEqual(gotSpec.Args, want) { + t.Fatalf("spec args = %v, want %v", gotSpec.Args, want) + } +} + +func TestCmdRunPlainRootfsExistingSkipsUnpack(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + rootfs := filepath.Join(t.TempDir(), "existing-rootfs") + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + if _, err := os.Stat(filepath.Join(rootfs, "hello")); !os.IsNotExist(err) { + t.Fatalf("existing rootfs was unpacked over: stat hello = %v, want IsNotExist", err) + } + if !reflect.DeepEqual(spec.Args, []string{"/image-cmd"}) { + t.Fatalf("spec args = %v, want image cmd", spec.Args) + } + return nil + }) + + _, stderr, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", s.root, "--plain-rootfs", "--rootfs", rootfs, "local:a"}) + }) + if err != nil { + t.Fatalf("cmdRun existing rootfs: %v", err) + } + if strings.Contains(stderr, "Unpacking") { + t.Fatalf("existing rootfs stderr = %q, want no unpack message", stderr) + } +} + +func TestCmdRunPlainRootfsAutoPullsMissingImage(t *testing.T) { + root := t.TempDir() + rootfs := filepath.Join(t.TempDir(), "rootfs") + pullCalls := 0 + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + pullCalls++ + if ref != "local:pulled" { + t.Fatalf("pull ref = %q, want local:pulled", ref) + } + return buildImage(t, []string{"/pulled-cmd"}), nil + }) + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + if !reflect.DeepEqual(spec.Args, []string{"/pulled-cmd"}) { + t.Fatalf("spec args = %v, want pulled image cmd", spec.Args) + } + return nil + }) + + stdout, stderr, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", root, "--plain-rootfs", "--rootfs", rootfs, "local:pulled"}) + }) + if err != nil { + t.Fatalf("cmdRun auto-pull: %v", err) + } + if pullCalls != 1 { + t.Fatalf("pullCalls = %d, want 1", pullCalls) + } + // run's stdout belongs to the guest (callers capture it), so the pull + // summary must land on stderr with the unpack progress. + if strings.Contains(stdout, "Pulled local:pulled") { + t.Fatalf("cmdRun auto-pull stdout = %q, pull summary leaked into guest stdout", stdout) + } + if !strings.Contains(stderr, "Pulled local:pulled") || !strings.Contains(stderr, "Unpacking local:pulled") { + t.Fatalf("cmdRun auto-pull stderr = %q, want pull and unpack summaries", stderr) + } + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + if _, err := s.digestFor("local:pulled"); err != nil { + t.Fatalf("auto-pulled ref was not pinned: %v", err) + } +} + +func TestCommandWrappersReturnParseAndStoreErrors(t *testing.T) { + parseCases := []struct { + name string + fn func() error + }{ + {"pull", func() error { return cmdPull(nil) }}, + {"unpack", func() error { return cmdUnpack(nil) }}, + {"inspect", func() error { return cmdInspect(nil) }}, + {"run", func() error { return cmdRun(nil) }}, + {"list", func() error { return cmdList([]string{"extra"}) }}, + {"rmi", func() error { return cmdRmi(nil) }}, + {"prune", func() error { return cmdPrune([]string{"--all"}) }}, + } + for _, tc := range parseCases { + t.Run("parse "+tc.name, func(t *testing.T) { + if err := tc.fn(); err == nil { + t.Fatalf("%s parse error case succeeded, want error", tc.name) + } + }) + } + + storeFile := filepath.Join(t.TempDir(), "store-file") + if err := os.WriteFile(storeFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + storeCases := []struct { + name string + fn func() error + }{ + {"pull", func() error { return cmdPull([]string{"--store", storeFile, "local:a"}) }}, + {"unpack", func() error { return cmdUnpack([]string{"--store", storeFile, "local:a"}) }}, + {"inspect", func() error { return cmdInspect([]string{"--store", storeFile, "local:a"}) }}, + {"run", func() error { return cmdRun([]string{"--store", storeFile, "local:a"}) }}, + {"list", func() error { return cmdList([]string{"--store", storeFile}) }}, + {"rmi", func() error { return cmdRmi([]string{"--store", storeFile, "local:a"}) }}, + {"prune", func() error { return cmdPrune([]string{"--store", storeFile}) }}, + } + for _, tc := range storeCases { + t.Run("store "+tc.name, func(t *testing.T) { + if err := tc.fn(); err == nil { + t.Fatalf("%s store error case succeeded, want error", tc.name) + } + }) + } +} + +// TestCmdRunPlatformMismatchOnPinnedRef pins the --platform check: the store +// pins one digest per ref, so an explicit --platform that disagrees with the +// pinned image must fail instead of silently launching the wrong +// architecture. Without --platform the pinned image runs as-is. +func TestCmdRunPlatformMismatchOnPinnedRef(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/image-cmd"})); err != nil { + t.Fatal(err) + } + withFakeExecElfuse(t, func(string, *runSpec, *flockFile) error { return nil }) + rootfs := filepath.Join(t.TempDir(), "rootfs") + + _, _, err := captureOutput(t, func() error { + return cmdRun([]string{ + "--store", s.root, "--platform", "linux/amd64", + "--plain-rootfs", "--rootfs", rootfs, "local:a", + }) + }) + if err == nil || !strings.Contains(err.Error(), "pinned for linux/arm64") { + t.Fatalf("cmdRun --platform mismatch err = %v, want pinned-platform error", err) + } + + for _, args := range [][]string{ + {"--store", s.root, "--platform", "linux/arm64", "--plain-rootfs", "--rootfs", rootfs, "local:a"}, + {"--store", s.root, "--plain-rootfs", "--rootfs", rootfs, "local:a"}, + } { + if _, _, err := captureOutput(t, func() error { return cmdRun(args) }); err != nil { + t.Fatalf("cmdRun %v: %v", args, err) + } + } +} + +// TestCmdRunPlainRootfsLockDiscipline pins which runs hold the per-digest +// cache lock at exec time: a store-default rootfs arrives with the run lock +// held (a concurrent prune would see busy), while an explicit --rootfs is +// user-managed and locks nothing. +func TestCmdRunPlainRootfsLockDiscipline(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/image-cmd"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + var gotLock *flockFile + var busyAtExec bool + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, lock *flockFile) error { + gotLock = lock + busyAtExec = rootfsCacheBusy(rootfs) + return nil + }) + if _, _, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", s.root, "--plain-rootfs", "local:a"}) + }); err != nil { + t.Fatalf("cmdRun store-default: %v", err) + } + if gotLock == nil { + t.Error("store-default rootfs exec got nil lock, want held run lock") + } + if !busyAtExec { + t.Error("store-default cache not busy at exec time, want lock held") + } + + gotLock = nil + rootfs := filepath.Join(t.TempDir(), "explicit") + if _, _, err := captureOutput(t, func() error { + return cmdRun([]string{"--store", s.root, "--plain-rootfs", "--rootfs", rootfs, "local:a"}) + }); err != nil { + t.Fatalf("cmdRun explicit rootfs: %v", err) + } + if gotLock != nil { + t.Error("explicit --rootfs exec got a lock, want nil") + } +} + +// markedImage builds a single-layer image whose /marker file holds content, so +// a test can tell which image populated a digest-keyed cache. +func markedImage(t *testing.T, content string) v1.Image { + t.Helper() + return testImageWithLayers(t, testTarLayer(t, + tarEntry{header: regHeader("marker", 0o644, 0), body: content})) +} + +// TestCmdUnpackUsesResolvedImageDespiteRepull pins that unpack fills the +// digest-keyed cache from the image it resolved, not a re-resolution of the +// mutable ref: a repull that moves the tag to a different image mid-setup must +// not poison digest A's cache with image B's content (#8). The afterImageResolve +// seam fires in the exact window between resolution and unpack. +func TestCmdUnpackUsesResolvedImageDespiteRepull(t *testing.T) { + s := openTestStore(t) + imgA := markedImage(t, "image-A") + imgB := markedImage(t, "image-B") + digestA, err := s.addImage("local:tag", imgA) + if err != nil { + t.Fatal(err) + } + + old := afterImageResolve + t.Cleanup(func() { afterImageResolve = old }) + repinned := false + afterImageResolve = func(digest string) { + if repinned || digest != digestA { + return + } + repinned = true + // Move the tag to image B while the unpack still holds digest A's + // reference lock. addImage takes the store lock, which resolveImageForUse + // has already released by now. + if _, err := s.addImage("local:tag", imgB); err != nil { + t.Errorf("repull to image B: %v", err) + } + } + + if _, _, err := captureOutput(t, func() error { + return cmdUnpack([]string{"--store", s.root, "local:tag"}) + }); err != nil { + t.Fatalf("cmdUnpack: %v", err) + } + if !repinned { + t.Fatal("afterImageResolve never fired for digest A") + } + rootfsA, err := defaultRootfsForDigest(s.root, digestA) + if err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(filepath.Join(rootfsA, "marker")); err != nil || string(b) != "image-A" { + t.Fatalf("digest-A cache marker = %q, err=%v; want image-A (not poisoned by repull)", b, err) + } +} + +// TestRmiRefusesWhileReferenceLockHeld pins #6: a starting run holds the +// per-digest reference lock before any cache dir or bundle exists, so rmi of +// the last pin must refuse (even with --force) rather than GC blobs out from +// under it, and the pin plus its blobs survive the refusal. Once the lock is +// released the same rmi succeeds. +func TestRmiRefusesWhileReferenceLockHeld(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + digest, err := s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + // A cold run's reference lock: taken before the rootfs cache dir exists. + hold, err := acquireRootfsRunLock(rootfs) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Fatalf("cache dir exists before run unpacked it: %v", err) + } + + if _, err := s.rmi("local:a", true); err == nil { + t.Fatal("rmi --force succeeded while reference lock held, want refusal") + } + if _, err := s.image("local:a"); err != nil { + t.Fatalf("pin/blobs gone after refused rmi: %v", err) + } + + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi after lock released: %v", err) + } +} diff --git a/cmd/elfuse-oci/common_test.go b/cmd/elfuse-oci/common_test.go index a65d120d..0946abe3 100644 --- a/cmd/elfuse-oci/common_test.go +++ b/cmd/elfuse-oci/common_test.go @@ -135,16 +135,47 @@ func TestParseRunArgs(t *testing.T) { func TestParseCommandFlagErrors(t *testing.T) { cases := []struct { name string - err error + fn func() error }{ - {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }()}, - {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }()}, - {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }()}, - {"run missing ref", func() error { _, _, _, _, err := parseRunArgs([]string{"--env", "A=1"}); return err }()}, + {"malformed platform", func() error { _, _, err := parsePullArgs([]string{"--platform", "bogus", "alpine:3"}); return err }}, + {"unknown flag", func() error { _, _, err := parsePullArgs([]string{"--unknown", "alpine:3"}); return err }}, + {"missing flag value", func() error { _, _, _, err := parseUnpackArgs([]string{"--rootfs"}); return err }}, + {"run missing ref", func() error { _, _, _, _, err := parseRunArgs([]string{"--env", "A=1"}); return err }}, + {"list extra arg", func() error { _, _, err := parseListArgs([]string{"alpine:3"}); return err }}, + {"rmi missing ref", func() error { _, _, _, err := parseRmiArgs([]string{"--force"}); return err }}, + {"prune all without cache", func() error { _, _, err := parsePruneArgs([]string{"--all"}); return err }}, } for _, tc := range cases { - if tc.err == nil { - t.Errorf("%s: got nil error", tc.name) - } + t.Run(tc.name, func(t *testing.T) { + if err := tc.fn(); err == nil { + t.Errorf("%s: got nil error, want parse failure", tc.name) + } + }) + } +} + +func TestParseLifecycleArgs(t *testing.T) { + cf, asJSON, err := parseListArgs([]string{"--store", "/s", "--json"}) + if err != nil { + t.Fatal(err) + } + if cf.store != "/s" || !asJSON { + t.Fatalf("list parse = store %q json %v, want /s true", cf.store, asJSON) + } + + cf, force, ref, err := parseRmiArgs([]string{"--force", "alpine:3"}) + if err != nil { + t.Fatal(err) + } + if force != true || ref != "alpine:3" || cf.platform != defaultPlatform { + t.Fatalf("rmi parse = force %v ref %q platform %+v", force, ref, cf.platform) + } + + cf, opts, err := parsePruneArgs([]string{"--cache", "--all", "--dry-run"}) + if err != nil { + t.Fatal(err) + } + if !opts.cache || !opts.all || !opts.dryRun || cf.platform != defaultPlatform { + t.Fatalf("prune parse = opts %+v platform %+v", opts, cf.platform) } } diff --git a/cmd/elfuse-oci/csrun.go b/cmd/elfuse-oci/csrun.go index d1237eb2..64bd1190 100644 --- a/cmd/elfuse-oci/csrun.go +++ b/cmd/elfuse-oci/csrun.go @@ -21,10 +21,35 @@ var ( clonefileForRun = unix.Clonefile spawnElfuseWaitForRun = spawnElfuseWait cleanupCloneAndMountForRun = cleanupCloneAndMount + writeKeptSidecarForRun = writeKeptSidecar osExitForRun = os.Exit runNowUnixNano = func() int64 { return time.Now().UnixNano() } ) +// keptSidecarName marks a whole sparsebundle as holding run --keep retained +// output. It lives beside the bundle's sparsebundle image and flocks, outside +// the mounted volume, so a cold rmi can detect a deliberate keep without +// attaching a detached bundle to look for .elfuse-keep clones inside it. The +// only thing that removes a kept clone is whole-bundle removal (rmi --force, +// prune --cache --all), which deletes this sidecar along with it, so the marker +// never goes stale. +const keptSidecarName = "kept" + +func keptSidecarPath(bundle string) string { + return filepath.Join(bundle, keptSidecarName) +} + +// writeKeptSidecar records that m's bundle holds run --keep retained output. +// m.mountPath is /mnt, so the sidecar lands beside the bundle image. +func writeKeptSidecar(m *csMount) error { + f, err := os.OpenFile(keptSidecarPath(filepath.Dir(m.mountPath)), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + return f.Close() +} + // csBundleDirForDigest is /cs//: it holds the case-sensitive // sparsebundle image and the attach mount point for one pinned manifest digest. func csBundleDirForDigest(store, digest string) (string, error) { @@ -43,7 +68,7 @@ func csBundleDirForDigest(store, digest string) (string, error) { // The unpacked base tree persists in the sparsebundle image file across // attach/detach cycles, so warm re-runs skip the (slow) unpack and pay only the // attach. -func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { +func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref string, img v1.Image, digest, size string) (*csMount, string, error) { bundle, err := csBundleDirForDigest(cf.store, digest) if err != nil { return nil, "", err @@ -59,7 +84,11 @@ func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size strin return nil, "", errors.Join(err, closeMount(m)) } fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, rootfs) - if err := unpackImage(s, ref, rootfs); err != nil { + // Unpack the caller's already-resolved image, not a re-resolution of + // ref: the bundle is keyed by digest, so a repull moving the tag + // mid-setup must not fill this digest's sparsebundle with another + // image. + if err := unpackImage(img, rootfs); err != nil { return nil, "", errors.Join(err, closeMount(m)) } } @@ -78,8 +107,8 @@ func ensureCaseSensitiveRootfs(cf commonFlags, s *store, ref, digest, size strin // The clone lives in the same APFS volume as the base tree (clonefile is // intra-volume only), so it is instant and free until the guest writes (COW). // It isolates each run's mutations from the warm base, so re-runs stay clean. -func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { - m, baseRootfs, err := ensureCaseSensitiveRootfsForRun(cf, s, ref, digest, rf.sparseSize) +func runCaseSensitive(cf commonFlags, s *store, ref string, img v1.Image, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { + m, baseRootfs, err := ensureCaseSensitiveRootfsForRun(cf, s, ref, img, digest, rf.sparseSize) if err != nil { return err } @@ -89,11 +118,26 @@ func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.Conf // Per-run COW clone. --no-clone runs against the base tree directly (mutations // then persist into the warm tree; useful for debugging or when clonefile is - // unavailable). + // unavailable). Liveness no longer depends on a clone directory existing: + // this run holds the bundle's run.lock (via the csMount) for its whole + // lifetime, so prune --cache/rmi --force see the volume busy and leave it + // attached regardless of whether a clone was made, so --no-clone needs + // no placeholder directory. sysroot := baseRootfs - var cloneDir string + cloneDir := filepath.Join(m.mountPath, fmt.Sprintf("run-%d-%d", os.Getpid(), runNowUnixNano())) + if rf.keepRootfs { + // Record the bundle-level keep FIRST, before the per-clone marker, so + // the two keep records never disagree: if this succeeds but the + // per-clone marker below fails, a cold rmi still refuses to discard the + // bundle without --force. In the reverse order a failed sidecar write + // would leave a sweep-preserved clone that rmi silently discards. It + // also covers the --no-clone --keep case (mutations land in the base + // tree, no clone marker is written at all). + if err := writeKeptSidecarForRun(m); err != nil { + return errors.Join(err, closeMount(m)) + } + } if !rf.noClone { - cloneDir = filepath.Join(m.mountPath, fmt.Sprintf("run-%d-%d", os.Getpid(), runNowUnixNano())) if err := os.RemoveAll(cloneDir); err != nil { err = fmt.Errorf("remove stale COW clone %s: %w", cloneDir, err) return errors.Join(err, closeMount(m)) @@ -103,6 +147,14 @@ func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.Conf return errors.Join(err, closeMount(m)) } sysroot = cloneDir + if rf.keepRootfs { + // Mark the clone so a later prune/rmi sweep preserves it even + // after this run exits and releases run.lock: without the marker + // the sweep, which only runs when no run is live, would reap it. + if err := writeKeepMarker(cloneDir); err != nil { + return errors.Join(err, cleanupCloneAndMountForRun(cloneDir, rf.keepRootfs, m)) + } + } } spec, err := computeRunSpec(cfg, rf, sysroot, tail) @@ -124,8 +176,11 @@ func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.Conf // --keep leaves the clone and the mount in place for inspection. The // clone lives in the sparsebundle volume, so the mount must stay // attached for it to be reachable on the host; a later run reattaches - // (detaching this stale mount first) and the kept clone reappears. - if cloneDir != "" { + // (detaching this stale mount first) and the kept clone, protected + // by its keep marker from any intervening sweep, reappears. Under + // --no-clone there is no clone to keep (mutations landed in the base + // tree), only the still-attached mount. + if !rf.noClone { fmt.Fprintf(os.Stderr, "kept clone: %s\n", cloneDir) } fmt.Fprintf(os.Stderr, "mount stays attached: %s\n", m.mountPath) diff --git a/cmd/elfuse-oci/csrun_darwin_test.go b/cmd/elfuse-oci/csrun_darwin_test.go index 3516c767..d40579b2 100644 --- a/cmd/elfuse-oci/csrun_darwin_test.go +++ b/cmd/elfuse-oci/csrun_darwin_test.go @@ -6,6 +6,7 @@ package main import ( + "archive/tar" "fmt" "os" "path/filepath" @@ -25,6 +26,7 @@ func withCSRunSeams(t *testing.T) { oldClone := clonefileForRun oldSpawn := spawnElfuseWaitForRun oldCleanup := cleanupCloneAndMountForRun + oldSidecar := writeKeptSidecarForRun oldExit := osExitForRun oldNow := runNowUnixNano t.Cleanup(func() { @@ -32,6 +34,7 @@ func withCSRunSeams(t *testing.T) { clonefileForRun = oldClone spawnElfuseWaitForRun = oldSpawn cleanupCloneAndMountForRun = oldCleanup + writeKeptSidecarForRun = oldSidecar osExitForRun = oldExit runNowUnixNano = oldNow }) @@ -48,7 +51,7 @@ func TestRunCaseSensitiveCloneSpawnCleanupAndExit(t *testing.T) { runNowUnixNano = func() int64 { return 123 } expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-123", os.Getpid())) var clonedSrc, clonedDst string - ensureCaseSensitiveRootfsForRun = func(cf commonFlags, s *store, ref, digest, size string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, s *store, ref string, img v1.Image, digest, size string) (*csMount, string, error) { if cf.store != "store" || ref != "local:a" || digest != "sha256:"+strings.Repeat("7", 64) || size != "64m" { t.Fatalf("ensure args = store=%q ref=%q digest=%q size=%q", cf.store, ref, digest, size) } @@ -104,6 +107,7 @@ func TestRunCaseSensitiveCloneSpawnCleanupAndExit(t *testing.T) { commonFlags{store: "store"}, &store{}, "local:a", + nil, "sha256:"+strings.Repeat("7", 64), cfg, runFlags{sparseSize: "64m"}, @@ -119,7 +123,7 @@ func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { if err := os.MkdirAll(base, 0o755); err != nil { t.Fatal(err) } - ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { return &csMount{mountPath: mount}, base, nil } clonefileForRun = func(string, string, int) error { @@ -136,6 +140,13 @@ func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { t.Fatal("cleanup called with --keep") return nil } + // --no-clone --keep still records the keep beside the bundle so a cold rmi + // refuses to discard the mutated base tree without --force. + sidecarWritten := false + writeKeptSidecarForRun = func(*csMount) error { + sidecarWritten = true + return nil + } osExitForRun = func(code int) { panic(runExitCode(code)) } defer func() { @@ -144,8 +155,11 @@ func TestRunCaseSensitiveNoCloneKeepSkipsCleanup(t *testing.T) { if !ok || code != 0 { t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) } + if !sidecarWritten { + t.Error("--no-clone --keep did not write the kept sidecar") + } }() - err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("8", 64), + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("8", 64), &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, runFlags{noClone: true, keepRootfs: true}, nil) @@ -162,7 +176,7 @@ func TestRunCaseSensitiveSpecErrorCleansCloneAndMount(t *testing.T) { m := &csMount{mountPath: mount} runNowUnixNano = func() int64 { return 456 } expectedClone := filepath.Join(mount, fmt.Sprintf("run-%d-456", os.Getpid())) - ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, string, string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { return m, base, nil } clonefileForRun = func(src, dst string, flags int) error { @@ -185,7 +199,7 @@ func TestRunCaseSensitiveSpecErrorCleansCloneAndMount(t *testing.T) { } osExitForRun = func(code int) { t.Fatalf("exit called after spec error with code %d", code) } - err := runCaseSensitive(commonFlags{}, &store{}, "local:a", "sha256:"+strings.Repeat("9", 64), + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("9", 64), &v1.ConfigFile{Config: v1.Config{}}, runFlags{}, nil) @@ -211,8 +225,12 @@ func TestEnsureCaseSensitiveRootfsProvisionsUnpacksAndSkipsExisting(t *testing.T if err != nil { t.Fatal(err) } + img, err := s.image("local:tiny") + if err != nil { + t.Fatal(err) + } - m, rootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + m, rootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", img, digest, "32m") if err != nil { t.Fatalf("ensureCaseSensitiveRootfs: %v", err) } @@ -242,8 +260,12 @@ func TestEnsureCaseSensitiveRootfsProvisionsUnpacksAndSkipsExisting(t *testing.T if err != nil { t.Fatal(err) } + img, err := s.image("local:tiny") + if err != nil { + t.Fatal(err) + } - m, gotRootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", digest, "32m") + m, gotRootfs, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:tiny", img, digest, "32m") if err != nil { t.Fatalf("ensureCaseSensitiveRootfs existing: %v", err) } @@ -272,9 +294,21 @@ func TestEnsureCaseSensitiveRootfsClosesMountOnUnpackError(t *testing.T) { t.Setenv("HDIUTIL_DETACH_LOG", detachLog) s := openTestStore(t) - _, _, err := ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:missing", "sha256:"+strings.Repeat("a", 64), "32m") - if err == nil || !strings.Contains(err.Error(), "not pulled") { - t.Fatalf("ensureCaseSensitiveRootfs err = %v, want unpack not-pulled error", err) + // The image is resolved by the caller now, so an unpack failure has to come + // from the image content, not a missing ref (that is caught upstream in + // resolveImageForUse). A fifo entry is an unsupported special file, so + // unpackImage fails and ensureCaseSensitiveRootfs must still detach the + // mount it attached. + bad := testImageWithLayers(t, testTarLayer(t, + tarEntry{header: tar.Header{Name: "dev/fifo", Typeflag: tar.TypeFifo, Mode: 0o644}})) + digest, err := s.addImage("local:bad", bad) + if err != nil { + t.Fatal(err) + } + + _, _, err = ensureCaseSensitiveRootfs(commonFlags{store: s.root}, s, "local:bad", bad, digest, "32m") + if err == nil || !strings.Contains(err.Error(), "fifo") { + t.Fatalf("ensureCaseSensitiveRootfs err = %v, want unpack fifo error", err) } b, readErr := os.ReadFile(detachLog) if readErr != nil { @@ -328,7 +362,7 @@ func TestCmdRunDefaultCaseSensitivePath(t *testing.T) { t.Fatal(err) } runNowUnixNano = func() int64 { return 789 } - ensureCaseSensitiveRootfsForRun = func(cf commonFlags, _ *store, ref, digest, size string) (*csMount, string, error) { + ensureCaseSensitiveRootfsForRun = func(cf commonFlags, _ *store, ref string, img v1.Image, digest, size string) (*csMount, string, error) { if cf.store != s.root || ref != "local:a" || size != "" { t.Fatalf("ensure from cmdRun got store=%q ref=%q size=%q", cf.store, ref, size) } @@ -362,3 +396,118 @@ func TestCmdRunDefaultCaseSensitivePath(t *testing.T) { err := cmdRun([]string{"--store", s.root, "local:a"}) t.Fatalf("cmdRun returned %v, want exit panic", err) } + +// TestRunCaseSensitiveNoCloneCreatesNoMarkerDir pins that a --no-clone run +// leaves no run-- placeholder in the volume: liveness now rides on +// the bundle's run.lock (held via the csMount), not on a marker directory, so +// none is created and none is cleaned up. +func TestRunCaseSensitiveNoCloneCreatesNoMarkerDir(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 789 } + cloneName := filepath.Join(mount, fmt.Sprintf("run-%d-789", os.Getpid())) + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { + return m, base, nil + } + clonefileForRun = func(string, string, int) error { + t.Fatal("clonefile called with --no-clone") + return nil + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec) (int, error) { + if rootfs != base { + t.Fatalf("spawn rootfs = %q, want base rootfs", rootfs) + } + if _, err := os.Lstat(cloneName); !os.IsNotExist(err) { + t.Fatalf("--no-clone created a placeholder %q: %v, want none", cloneName, err) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(clone string, keep bool, cm *csMount) error { + return removeClone(clone, keep) + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) + } + }() + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("6", 64), + &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, + runFlags{noClone: true}, + nil) + t.Fatalf("runCaseSensitive returned %v, want exit panic", err) +} + +// TestRunCaseSensitiveKeepWritesKeepMarker pins that a --keep run records a +// keep sidecar beside (not inside) its COW clone so a later sweep preserves the +// clone after this run exits and releases run.lock, and so image content or a +// guest cannot forge the keep by writing /.elfuse-keep in the clone. +func TestRunCaseSensitiveKeepWritesKeepMarker(t *testing.T) { + withCSRunSeams(t) + mount := t.TempDir() + base := filepath.Join(mount, "rootfs") + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + m := &csMount{mountPath: mount} + runNowUnixNano = func() int64 { return 321 } + cloneDir := filepath.Join(mount, fmt.Sprintf("run-%d-321", os.Getpid())) + ensureCaseSensitiveRootfsForRun = func(commonFlags, *store, string, v1.Image, string, string) (*csMount, string, error) { + return m, base, nil + } + clonefileForRun = func(src, dst string, flags int) error { + return os.MkdirAll(dst, 0o755) + } + spawnElfuseWaitForRun = func(rootfs string, spec *runSpec) (int, error) { + if rootfs != cloneDir { + t.Fatalf("spawn rootfs = %q, want clone %q", rootfs, cloneDir) + } + if _, err := os.Stat(cloneKeepMarkerPath(mount, filepath.Base(cloneDir))); err != nil { + t.Fatalf("keep marker during run: %v, want present", err) + } + // The keep record must live OUTSIDE the clone (the guest's /), so + // image content or the guest cannot forge it. + if _, err := os.Stat(filepath.Join(cloneDir, ".elfuse-keep")); !os.IsNotExist(err) { + t.Fatalf("keep record found inside the clone: %v, want only the sidecar", err) + } + return 0, nil + } + cleanupCloneAndMountForRun = func(string, bool, *csMount) error { + t.Fatal("cleanup called with --keep") + return nil + } + sidecarWritten := false + writeKeptSidecarForRun = func(*csMount) error { + sidecarWritten = true + return nil + } + osExitForRun = func(code int) { panic(runExitCode(code)) } + + defer func() { + r := recover() + code, ok := r.(runExitCode) + if !ok || code != 0 { + t.Fatalf("runCaseSensitive panic = %T %v, want exit code 0", r, r) + } + // The kept clone with its marker survives: a later sweep skips it. + if listed := listSweepableClones(mount); len(listed) != 0 { + t.Fatalf("listSweepableClones = %v, want the kept clone skipped", listed) + } + if !sidecarWritten { + t.Error("--keep did not write the kept sidecar") + } + }() + err := runCaseSensitive(commonFlags{}, &store{}, "local:a", nil, "sha256:"+strings.Repeat("6", 64), + &v1.ConfigFile{Config: v1.Config{Cmd: []string{"/cmd"}}}, + runFlags{keepRootfs: true}, + nil) + t.Fatalf("runCaseSensitive returned %v, want exit panic", err) +} diff --git a/cmd/elfuse-oci/csrun_other.go b/cmd/elfuse-oci/csrun_other.go index b16d9b68..972310af 100644 --- a/cmd/elfuse-oci/csrun_other.go +++ b/cmd/elfuse-oci/csrun_other.go @@ -17,6 +17,6 @@ import ( // reports a clear error and directs the user at --plain-rootfs. This stub // exists so elfuse-oci compiles on Linux, where pull/inspect/unpack and // conformance/interop tests run. -func runCaseSensitive(cf commonFlags, s *store, ref, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { +func runCaseSensitive(cf commonFlags, s *store, ref string, img v1.Image, digest string, cfg *v1.ConfigFile, rf runFlags, tail []string) error { return fmt.Errorf("case-sensitive sparsebundle rootfs requires macOS; pass --plain-rootfs for a plain directory") } diff --git a/cmd/elfuse-oci/gc.go b/cmd/elfuse-oci/gc.go new file mode 100644 index 00000000..75f7087c --- /dev/null +++ b/cmd/elfuse-oci/gc.go @@ -0,0 +1,415 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// rmReport summarizes a reachability-GC pass: the blob digests removed (or that +// would be removed, under --dry-run) and the bytes reclaimed. CacheDropped +// records whether rmi also reclaimed the image's unpacked cache, so the command +// can report it rather than deleting a large warm tree silently. +type rmReport struct { + Ref string + Blobs []string + Bytes int64 + CacheDropped bool +} + +// gc runs a reachability pass over the OCI image-layout store and removes any +// sha256 blob that is not reachable from an index.json manifest descriptor. +// Reachability follows manifest/index descriptors recursively, then marks image +// manifests, configs, and layers live. When dryRun is set, gc reports what it +// would reclaim and deletes nothing. +// +// gc is the shared engine behind `rmi` (called after a manifest descriptor is +// removed from index.json, so the dropped image's config/layers surface as +// unreachable) and `prune` (a standalone sweep that reclaims retag/partial-pull +// orphans). It also reclaims stale temporary blob files left by interrupted +// writes under blobs/sha256/; those filenames are not valid sha256 digests and +// make layout.Path.GarbageCollect abort before it can sweep anything. +// +// It first reconciles index.json against the pin set (pruneUnpinnedDescriptors) +// so a re-pulled mutable tag's orphaned prior descriptor stops keeping its +// blobs live; the blob sweep then reclaims them. A dry run reconciles nothing +// and reports against the current index. +func (s *store) gc(dryRun bool) (rmReport, error) { + if !dryRun { + if err := s.pruneUnpinnedDescriptors(); err != nil { + return rmReport{}, fmt.Errorf("gc: reconcile descriptors: %w", err) + } + } + live, err := s.liveBlobDigests() + if err != nil { + return rmReport{}, fmt.Errorf("gc: compute reachability: %w", err) + } + var rep rmReport + blobs, err := s.localBlobFiles() + if err != nil { + return rep, err + } + for _, b := range blobs { + if !b.malformed && live[b.digest] { + continue + } + fi, err := os.Stat(b.path) + if os.IsNotExist(err) { + continue // raced or already removed + } + if err != nil { + return rep, err + } + rep.Blobs = append(rep.Blobs, b.digest) + rep.Bytes += fi.Size() + if !dryRun { + err := b.remove(s) + if err != nil && !os.IsNotExist(err) { + return rep, err + } + } + } + return rep, nil +} + +type localBlob struct { + path string + digest string + hash v1.Hash + malformed bool +} + +func (b localBlob) remove(s *store) error { + if b.malformed { + return os.Remove(b.path) + } + return s.path.RemoveBlob(b.hash) +} + +// liveBlobDigests roots reachability at the refs.json pin set, not at every +// index.json descriptor: an unpinned descriptor (a re-pulled tag's orphaned +// prior manifest, or a pull that appended a manifest but crashed before +// pinning) must not keep its blobs live. Each pinned digest's manifest, +// config, and layers are marked live. A real gc calls pruneUnpinnedDescriptors +// first so index.json never keeps referencing a manifest whose blobs this +// sweep reclaims; a dry run skips that but, rooting here at pins too, still +// reports exactly the blobs a real run would reclaim. +func (s *store) liveBlobDigests() (map[string]bool, error) { + pins, err := s.loadPins() + if err != nil { + return nil, err + } + live := map[string]bool{} + seen := map[string]bool{} + for _, digest := range pins { + if seen[digest] { + continue // shared manifest already marked + } + seen[digest] = true + h, err := v1.NewHash(digest) + if err != nil { + return nil, err + } + img, err := s.path.Image(h) + if err != nil { + return nil, fmt.Errorf("gc: read pinned manifest %s: %w", digest, err) + } + if err := markLiveImage(img, live); err != nil { + return nil, err + } + } + return live, nil +} + +func markLiveImage(image v1.Image, live map[string]bool) error { + h, err := image.Digest() + if err != nil { + return err + } + live[h.String()] = true + + h, err = image.ConfigName() + if err != nil { + return err + } + live[h.String()] = true + + layers, err := image.Layers() + if err != nil { + return err + } + for _, layer := range layers { + h, err := layer.Digest() + if err != nil { + return err + } + live[h.String()] = true + } + return nil +} + +func (s *store) localBlobFiles() ([]localBlob, error) { + base := filepath.Join(s.root, "blobs", "sha256") + entries, err := os.ReadDir(base) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + blobs := make([]localBlob, 0, len(entries)) + for _, e := range entries { + if e.IsDir() { + continue + } + info, err := e.Info() + if err != nil { + // The entry vanished between ReadDir and Info (a concurrent + // rmi/prune already reclaimed it); skip it rather than aborting + // the whole GC pass, matching the IsNotExist tolerance elsewhere + // in this file. + if os.IsNotExist(err) { + continue + } + return nil, err + } + if !info.Mode().IsRegular() { + continue + } + + name := e.Name() + path := filepath.Join(base, name) + if validSHA256Hex(name) { + h := v1.Hash{Algorithm: "sha256", Hex: name} + blobs = append(blobs, localBlob{path: path, digest: h.String(), hash: h}) + continue + } + blobs = append(blobs, localBlob{path: path, digest: "sha256:" + name, malformed: true}) + } + return blobs, nil +} + +func validSHA256Hex(s string) bool { + return len(s) == 64 && isLowerHex(s) +} + +// isLowerHex reports whether s contains only lowercase hex digits. Callers +// bound the length themselves (the empty string passes). +func isLowerHex(s string) bool { + for _, c := range s { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + return false + } + } + return true +} + +// rewriteIndexManifests rewrites index.json to the descriptors keep returns +// true for, writing atomically and durably (writeFileDurable), not through the +// layout package's in-place os.WriteFile: rmi commits an index.json change +// before dropping the pin from refs.json, and that ordering only survives a +// crash if each write is individually durable. A plain truncate-in-place write +// could leave a torn index.json (store unreadable) or a non-durable change a +// crash reverts after the fsynced pin drop, stranding a descriptor and every +// blob it keeps live. It reports whether anything changed so callers can skip +// the write (and its fsync) when the index already matches. +func (s *store) rewriteIndexManifests(keep func(v1.Descriptor) bool) (bool, error) { + idx, err := s.path.ImageIndex() + if err != nil { + return false, fmt.Errorf("rewrite index: read index: %w", err) + } + im, err := idx.IndexManifest() + if err != nil { + return false, fmt.Errorf("rewrite index: parse index: %w", err) + } + kept := make([]v1.Descriptor, 0, len(im.Manifests)) + for _, desc := range im.Manifests { + if keep(desc) { + kept = append(kept, desc) + } + } + if len(kept) == len(im.Manifests) { + return false, nil + } + next := *im + next.Manifests = kept + b, err := json.Marshal(&next) + if err != nil { + return false, fmt.Errorf("rewrite index: marshal index: %w", err) + } + if err := writeFileDurable(filepath.Join(s.root, "index.json"), b, 0o644); err != nil { + return false, err + } + return true, nil +} + +// removeManifestDescriptor removes the manifest descriptor with the given digest +// from index.json. It does not touch the manifest's config or layer blobs; a +// subsequent gc pass reclaims them once they are unreachable from every +// remaining descriptor, so a blob shared with another still-pinned image is +// kept. +func (s *store) removeManifestDescriptor(digest string) error { + h, err := v1.NewHash(digest) + if err != nil { + return fmt.Errorf("remove manifest descriptor: %w", err) + } + _, err = s.rewriteIndexManifests(func(desc v1.Descriptor) bool { + return desc.Digest != h + }) + return err +} + +// pruneUnpinnedDescriptors removes index.json manifest descriptors that no +// refs.json pin references. Reachability GC roots liveness at index.json +// descriptors, so a re-pull that moves a mutable tag to a new digest would +// otherwise leak the prior image forever: its descriptor stays in the index +// (keeping every blob live) even though no ref pins it, and rmi cannot target +// a digest it can no longer resolve through a pin. Reconciling the index to +// the pin set before the blob sweep makes that orphan reclaimable. The store +// only ever appends single-platform image manifests (addImage), and pins point +// at them directly, so a top-level descriptor absent from the pin set is +// genuinely unreferenced. +func (s *store) pruneUnpinnedDescriptors() error { + pins, err := s.loadPins() + if err != nil { + return err + } + pinned := make(map[string]bool, len(pins)) + for _, digest := range pins { + pinned[digest] = true + } + _, err = s.rewriteIndexManifests(func(desc v1.Descriptor) bool { + return pinned[desc.Digest.String()] + }) + return err +} + +// liveCacheKeys returns the set of digest cache keys for every currently pinned +// ref. pruneCaches uses it to keep digest-keyed rootfs/sparsebundle caches that +// are still reachable through refs.json. +func (s *store) liveCacheKeys() (map[string]bool, error) { + pins, err := s.loadPins() + if err != nil { + return nil, err + } + m := make(map[string]bool, len(pins)) + for _, digest := range pins { + key, err := cacheKeyForDigest(digest) + if err != nil { + return nil, err + } + m[key] = true + } + return m, nil +} + +// rmi removes one selected ref from the store. The target may be an exact ref +// or a unique sha256 digest prefix. If other refs still pin the same manifest +// digest, only the resolved pin is dropped: the shared descriptor, blobs, and +// digest-keyed caches stay live through the remaining refs. If this was the last +// pin for the digest, rmi removes the manifest descriptor, GCs the +// now-unreachable blobs, and reclaims the image's unpacked cache: the cache is +// derived state subordinate to the image, so it goes with it instead of being +// left as an orphan only prune --cache could reap. Two safety rules survive +// force: a cache whose volume a live run still uses is never dropped (even with +// force), and a cache holding run --keep retained output refuses without force +// so a deliberate keep is not discarded silently. +func (s *store) rmi(target string, force bool) (rmReport, error) { + // The store lock spans the whole resolve-modify-GC sequence so an rmi + // cannot interleave with a concurrent pull's check-append-pin (or another + // rmi) and lose one side's refs.json/index.json update. + unlock, err := s.lock() + if err != nil { + return rmReport{}, err + } + defer unlock() + pins, err := s.loadPins() + if err != nil { + return rmReport{}, err + } + ref, digest, err := resolvePinnedTarget(pins, target) + if err != nil { + return rmReport{}, err + } + + lastPin := true + for otherRef, otherDigest := range pins { + if otherRef != ref && otherDigest == digest { + lastPin = false + break + } + } + + var cacheDropped bool + if lastPin { + // A live or starting run holds the digest's reference lock (the plain + // rootfs run lock, taken by resolveImageForUse for every run path + // before any cache dir or bundle exists). Probe it directly rather + // than inferring liveness from cacheExists: a cold run that has + // resolved the image but not yet unpacked has no cache to detect, yet + // its descriptor and blobs must survive. rmi holds the store lock + // throughout, and a run claims the reference lock only while holding + // that same store lock (resolveImageForUse), so the set of holders is + // frozen here: a busy probe cannot be a run that is about to appear. + // Refuse regardless of force; force discards derived state, it does + // not evict a running guest. + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return rmReport{}, err + } + if rootfsCacheBusy(rootfs) { + return rmReport{}, fmt.Errorf("rmi: %q is in use by a live run; stop it before removing the image", ref) + } + // A run --keep clone is deliberately retained output living in the + // cache; refuse to discard it without force. Everything else in the + // cache is derived state reclaimed with the image below. + kept, err := cacheHasKeptData(s.root, digest) + if err != nil { + return rmReport{}, fmt.Errorf("rmi: inspect cache for %q: %w", ref, err) + } + if kept && !force { + return rmReport{}, fmt.Errorf("rmi: %q has retained run --keep output; pass --force to discard it, or inspect it with a fresh run then 'elfuse-oci prune --cache'", ref) + } + // Reclaim the unpacked cache as part of removing the image. removeRefCaches + // fails closed if a live run still holds the volume (even with force), so + // this never yanks a rootfs out from under a running guest. + if cacheExists(s.root, digest) { + if err := removeRefCaches(s, digest); err != nil { + return rmReport{}, fmt.Errorf("rmi: drop cache for %q: %w", ref, err) + } + cacheDropped = true + } + } + + // On a last-pin removal, update index.json BEFORE committing the pin + // removal to refs.json. In the opposite order, a failure between the two + // writes strands the manifest: the ref is gone from refs.json (so rmi can + // no longer resolve it) while the descriptor keeps every blob live, and + // prune never removes descriptors; the image becomes unreclaimable. In + // this order the same crash window leaves a stale pin over a removed + // descriptor, which a retried rmi resolves and finishes + // (RemoveDescriptors is a filter, so re-removing is a no-op). + if lastPin { + if err := s.removeManifestDescriptor(digest); err != nil { + return rmReport{}, fmt.Errorf("rmi: remove manifest descriptor for %q: %w", ref, err) + } + } + delete(pins, ref) + if err := s.savePins(pins); err != nil { + return rmReport{}, err + } + if !lastPin { + return rmReport{Ref: ref}, nil + } + rep, err := s.gc(false) + rep.Ref = ref + rep.CacheDropped = cacheDropped + return rep, err +} diff --git a/cmd/elfuse-oci/inspect.go b/cmd/elfuse-oci/inspect.go index 9679925b..1b334abe 100644 --- a/cmd/elfuse-oci/inspect.go +++ b/cmd/elfuse-oci/inspect.go @@ -13,6 +13,16 @@ import ( // inspect prints a stored image's manifest + config. With --json the raw config // JSON is emitted (one object); otherwise a human-readable summary. func inspect(w io.Writer, s *store, ref string, asJSON bool) error { + // Hold the store lock across resolution and every metadata read, as list + // does: ConfigFile/Layers are lazy blob reads, so a concurrent rmi on the + // last ref could delete the config/layer blobs between resolving the pin + // and reading them, failing inspect partway through. + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + img, err := s.image(ref) if err != nil { return err @@ -45,9 +55,13 @@ func inspect(w io.Writer, s *store, ref string, asJSON bool) error { bw := bufio.NewWriter(w) w = bw + platform := cfg.OS + "/" + cfg.Architecture + if cfg.Variant != "" { + platform += "/" + cfg.Variant + } fmt.Fprintf(w, "%-12s %s\n", "Ref:", ref) fmt.Fprintf(w, "%-12s %s\n", "Digest:", d) - fmt.Fprintf(w, "%-12s %s/%s\n", "Platform:", cfg.OS, cfg.Architecture) + fmt.Fprintf(w, "%-12s %s\n", "Platform:", platform) if !cfg.Created.IsZero() { fmt.Fprintf(w, "%-12s %s\n", "Created:", cfg.Created.UTC().Format("2006-01-02T15:04:05Z")) } diff --git a/cmd/elfuse-oci/inspect_test.go b/cmd/elfuse-oci/inspect_test.go index da7cd956..916c9e28 100644 --- a/cmd/elfuse-oci/inspect_test.go +++ b/cmd/elfuse-oci/inspect_test.go @@ -121,6 +121,80 @@ func TestInspectHumanIncludesCreatedEnvAndLayers(t *testing.T) { } } +// TestInspectHoldsStoreLock pins #2: inspect resolves and reads image metadata +// under the store lock, so a concurrent rmi cannot delete blobs mid-inspect. +// The proof is that inspect blocks while another holder keeps the lock and +// completes once it is released. +func TestInspectHoldsStoreLock(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:tiny", tinyImage(t)); err != nil { + t.Fatal(err) + } + unlock, err := s.lock() + if err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { + var buf bytes.Buffer + done <- inspect(&buf, s, "local:tiny", false) + }() + + select { + case <-done: + unlock() + t.Fatal("inspect completed while the store lock was held; it does not take the lock") + case <-time.After(150 * time.Millisecond): + } + + unlock() + select { + case err := <-done: + if err != nil { + t.Fatalf("inspect after unlock: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("inspect did not complete after the store lock was released") + } +} + +// TestInspectAndListAgreeOnVariant pins #13: an image with a platform variant +// (linux/arm/v7) must show the full os/arch/variant in inspect, matching list, +// not the truncated os/arch. +func TestInspectAndListAgreeOnVariant(t *testing.T) { + s := openTestStore(t) + img := tinyImage(t) + cfg, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + cfg.Architecture = "arm" + cfg.Variant = "v7" + img, err = mutate.ConfigFile(img, cfg) + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:v7", img); err != nil { + t.Fatal(err) + } + + var ins bytes.Buffer + if err := inspect(&ins, s, "local:v7", false); err != nil { + t.Fatal(err) + } + if !strings.Contains(ins.String(), "linux/arm/v7") { + t.Fatalf("inspect platform missing variant:\n%s", ins.String()) + } + var lst bytes.Buffer + if err := list(&lst, s, false); err != nil { + t.Fatal(err) + } + if !strings.Contains(lst.String(), "linux/arm/v7") { + t.Fatalf("list platform missing variant:\n%s", lst.String()) + } +} + func TestInspectMissingRefAndConfigErrors(t *testing.T) { s := openTestStore(t) if err := inspect(&bytes.Buffer{}, s, "local:missing", false); err == nil || !strings.Contains(err.Error(), "not pulled") { diff --git a/cmd/elfuse-oci/lifecycle_darwin_test.go b/cmd/elfuse-oci/lifecycle_darwin_test.go new file mode 100644 index 00000000..70110c56 --- /dev/null +++ b/cmd/elfuse-oci/lifecycle_darwin_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build darwin + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The darwin-only sparsebundle lifecycle (prune --cache detaching a stale mount +// and reaping abandoned COW clones, rmi --force dropping a mounted bundle) +// cannot run in hosted CI: it needs hdiutil + APFS + a real attach. The +// cross-platform pieces it rests on (listSweepableClones, csBundleBusy, the +// bundle flocks, pruneCaches, pruneRootfsCaches) are covered in +// lifecycle_test.go and bundlelock_test.go. This file adds the darwin-only +// round-trip behind ELFUSE_OCI_DARWIN_CS=1 so a Mac operator can opt in; +// without the flag it skips, so `go test ./cmd/elfuse-oci/` stays green +// everywhere by default. + +// TestDarwinCSSweep exercises the crash-recovery path prune --cache runs per +// sparsebundle bundle: while a live run holds the bundle's run.lock the volume +// is reported busy and stays attached; once that run is gone the next sweep +// reaps the abandoned (unmarked) clone, preserves a --keep-marked clone, and +// detaches, after which removeRefCaches drops the whole bundle. Gated because +// it provisions a real APFS sparsebundle via hdiutil. +func TestDarwinCSSweep(t *testing.T) { + if os.Getenv("ELFUSE_OCI_DARWIN_CS") == "" { + t.Skip("set ELFUSE_OCI_DARWIN_CS=1 to exercise the darwin sparsebundle sweep (needs hdiutil + APFS)") + } + + s := openTestStore(t) + digest := "sha256:" + strings.Repeat("1", 64) + bundle, err := csBundleDirForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + mnt := filepath.Join(bundle, "mnt") + // provision returns a mount holding run.lock shared; this stands in for + // the live run. Drop owned so Close does not detach; we release the + // liveness lock explicitly to simulate the run exiting. + m, err := provisionCaseSensitive(bundle, mnt, "32m") + if err != nil { + t.Fatalf("provision: %v", err) + } + m.owned = false + t.Cleanup(func() { + if isMountPoint(mnt) { + _ = detachForce(mnt) + } + }) + + // Plant an abandoned clone (reapable) and a --keep-marked clone (preserved). + reapClone := filepath.Join(mnt, "run-1-1") + keepClone := filepath.Join(mnt, "run-2-2") + for _, p := range []string{reapClone, keepClone} { + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + } + if err := writeKeepMarker(keepClone); err != nil { + t.Fatal(err) + } + if !isMountPoint(mnt) { + t.Fatalf("mnt %s not attached after provision", mnt) + } + + // First sweep: the live run still holds run.lock, so the bundle is busy; + // nothing is reaped and the volume stays attached. + reaped, busy, unlock, err := sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle: %v", err) + } + unlock() + if !busy { + t.Fatal("sweepCSBundle busy = false while a live run holds run.lock") + } + if len(reaped) != 0 { + t.Fatalf("sweepCSBundle reaped = %v while busy, want none", reaped) + } + if !isMountPoint(mnt) { + t.Fatal("volume detached although a live run remains") + } + + // The live run exits: release its run.lock. + if err := m.runLock.Close(); err != nil { + t.Fatal(err) + } + + // Second sweep, idle now: the unmarked clone is reaped, the kept clone + // preserved, and the stale mount detached. + reaped, busy, unlock, err = sweepCSBundle(bundle) + if err != nil { + t.Fatalf("sweepCSBundle after run exit: %v", err) + } + unlock() + if busy { + t.Fatalf("second sweep reported busy, want idle") + } + if len(reaped) != 1 || reaped[0] != reapClone { + t.Fatalf("second sweep reaped = %v, want [%s]", reaped, reapClone) + } + if isMountPoint(mnt) { + t.Errorf("mnt still attached after idle sweepCSBundle, want detached") + } + + // removeRefCaches should now delete the whole bundle directory, kept clone + // and all. + if err := removeRefCaches(s, digest); err != nil { + t.Fatalf("removeRefCaches: %v", err) + } + if _, err := os.Stat(bundle); !os.IsNotExist(err) { + t.Errorf("bundle dir after removeRefCaches: %v, want IsNotExist", err) + } +} diff --git a/cmd/elfuse-oci/lifecycle_test.go b/cmd/elfuse-oci/lifecycle_test.go new file mode 100644 index 00000000..846ae1f5 --- /dev/null +++ b/cmd/elfuse-oci/lifecycle_test.go @@ -0,0 +1,1217 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" +) + +func firstLayerDigest(t *testing.T, img v1.Image) v1.Hash { + t.Helper() + ls, err := img.Layers() + if err != nil || len(ls) == 0 { + t.Fatalf("layers: %v (n=%d)", err, len(ls)) + } + d, err := ls[0].Digest() + if err != nil { + t.Fatal(err) + } + return d +} + +func indexManifestCount(t *testing.T, root string) int { + t.Helper() + b, err := os.ReadFile(filepath.Join(root, "index.json")) + if err != nil { + t.Fatal(err) + } + var idx struct { + Manifests []json.RawMessage `json:"manifests"` + } + if err := json.Unmarshal(b, &idx); err != nil { + t.Fatalf("index.json unmarshal: %v", err) + } + return len(idx.Manifests) +} + +func rootfsForDigest(t *testing.T, s *store, digest string) string { + t.Helper() + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + t.Fatal(err) + } + return rootfs +} + +func rootfsForImage(t *testing.T, s *store, img v1.Image) string { + t.Helper() + d, err := img.Digest() + if err != nil { + t.Fatal(err) + } + return rootfsForDigest(t, s, d.String()) +} + +// TestPruneReclaimsRepulledTagPriorImage pins #14: re-pulling a mutable tag to +// a new digest must not leak the prior image. Reachability GC roots at the pin +// set, and gc reconciles index.json to it, so after the tag moves A->B a prune +// removes A's orphaned descriptor and its unique blobs (manifest, config) while +// keeping the layer blob B still shares, and B stays intact. +func TestPruneReclaimsRepulledTagPriorImage(t *testing.T) { + s := openTestStore(t) + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + + manifestA, err := imgA.Digest() + if err != nil { + t.Fatal(err) + } + configA, err := imgA.ConfigName() + if err != nil { + t.Fatal(err) + } + sharedLayer := firstLayerDigest(t, imgA) + if got := firstLayerDigest(t, imgB); got != sharedLayer { + t.Fatalf("test images do not share a layer: A=%s B=%s", sharedLayer, got) + } + + if _, err := s.addImage("local:tag", imgA); err != nil { + t.Fatal(err) + } + digestB, err := s.addImage("local:tag", imgB) // repin tag A->B; A now orphaned + if err != nil { + t.Fatal(err) + } + if n := indexManifestCount(t, s.root); n != 2 { + t.Fatalf("index has %d descriptors after repull, want 2 (A leaked, B live)", n) + } + + if err := cmdPrune([]string{"--store", s.root}); err != nil { + t.Fatalf("prune: %v", err) + } + + if n := indexManifestCount(t, s.root); n != 1 { + t.Fatalf("index has %d descriptors after prune, want 1 (A reclaimed)", n) + } + for _, gone := range []v1.Hash{manifestA, configA} { + if _, err := os.Stat(blobPath(s.root, gone.String())); !os.IsNotExist(err) { + t.Errorf("orphaned blob %s still present after prune: %v", gone, err) + } + } + if _, err := os.Stat(blobPath(s.root, sharedLayer.String())); err != nil { + t.Errorf("shared layer %s reclaimed although B still references it: %v", sharedLayer, err) + } + got, err := s.digestFor("local:tag") + if err != nil || got != digestB { + t.Fatalf("tag resolves to %q err=%v; want B %s", got, err, digestB) + } + if _, err := s.image("local:tag"); err != nil { + t.Fatalf("image B unreadable after prune: %v", err) + } +} + +// --- list ------------------------------------------------------------------- + +func TestListEmpty(t *testing.T) { + s := openTestStore(t) + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + if buf.Len() != 0 { + t.Errorf("human list of empty store produced output %q, want none", buf.String()) + } + var jbuf bytes.Buffer + if err := list(&jbuf, s, true); err != nil { + t.Fatal(err) + } + if strings.TrimSpace(jbuf.String()) != "[]" { + t.Errorf("json list of empty store = %q, want []", jbuf.String()) + } +} + +func TestListShape(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + out := buf.String() + for _, want := range []string{"local:a", "linux/arm64"} { + if !strings.Contains(out, want) { + t.Errorf("human list missing %q in:\n%s", want, out) + } + } + + var jbuf bytes.Buffer + if err := list(&jbuf, s, true); err != nil { + t.Fatal(err) + } + var entries []listEntry + if err := json.Unmarshal(jbuf.Bytes(), &entries); err != nil { + t.Fatalf("json list unmarshal: %v (raw %q)", err, jbuf.String()) + } + if len(entries) != 1 || entries[0].Ref != "local:a" { + t.Fatalf("json list entries = %+v, want one local:a", entries) + } + e := entries[0] + if !strings.HasPrefix(e.Digest, "sha256:") { + t.Errorf("json digest = %q, want sha256: prefix", e.Digest) + } + if e.Platform != "linux/arm64" { + t.Errorf("json platform = %q, want linux/arm64", e.Platform) + } + if e.Layers != 1 { + t.Errorf("json layers = %d, want 1", e.Layers) + } + if e.Size <= 0 { + t.Errorf("json size = %d, want > 0 (compressed layer size)", e.Size) + } +} + +func TestListCorruptConfigErrors(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + config, err := img.ConfigName() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if err := os.Remove(blobPath(s.root, config.String())); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err = list(&buf, s, false) + if err == nil || !strings.Contains(err.Error(), "list: local:a: config") { + t.Fatalf("list err = %v, want local:a config error", err) + } +} + +func TestListMultipleRefsSorted(t *testing.T) { + s := openTestStore(t) + for _, ref := range []string{"local:b", "local:a"} { + if _, err := s.addImage(ref, buildImage(t, []string{ref})); err != nil { + t.Fatal(err) + } + } + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + out := buf.String() + if i, j := strings.Index(out, "local:a"), strings.Index(out, "local:b"); i < 0 || j < 0 || i > j { + t.Errorf("list not sorted by ref:\n%s", out) + } +} + +// --- rmi -------------------------------------------------------------------- + +func TestRmiDropsPinAndBlobs(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + config, _ := img.ConfigName() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi: %v", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("digestFor after rmi succeeded, want not-pulled error") + } + for _, d := range []string{manifest.String(), config.String(), layer.String()} { + if _, err := os.Stat(blobPath(s.root, d)); !os.IsNotExist(err) { + t.Errorf("blob %s after rmi: %v, want IsNotExist", d, err) + } + } + if n := indexManifestCount(t, s.root); n != 0 { + t.Errorf("index.json after rmi has %d manifest descriptors, want 0", n) + } +} + +func TestRmiByRefDropsStaleTempBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.rmi("local:a", false) + if err != nil { + t.Fatalf("rmi by ref with stale temp blob: %v", err) + } + if !sliceContains(rep.Blobs, stale) { + t.Fatalf("rmi report blobs = %v, want stale temp blob %s", rep.Blobs, stale) + } + if _, err := os.Stat(blobPath(s.root, stale)); !os.IsNotExist(err) { + t.Fatalf("stale temp blob after rmi by ref: %v, want IsNotExist", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after rmi by ref") + } +} + +func TestRmiKeepsSharedBlobs(t *testing.T) { + s := openTestStore(t) + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + sharedLayer := firstLayerDigest(t, imgA) // same content -> same digest as imgB's layer + if _, err := s.addImage("local:a", imgA); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", imgB); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi local:a: %v", err) + } + if _, err := os.Stat(blobPath(s.root, sharedLayer.String())); err != nil { + t.Errorf("shared layer blob after rmi local:a: %v, want present (still reachable via local:b)", err) + } + if _, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after rmi local:a: %v", err) + } + if img, err := s.image("local:b"); err != nil { + t.Errorf("s.image(local:b) after rmi local:a: %v", err) + } else if ls, _ := img.Layers(); len(ls) != 1 { + t.Errorf("local:b layers after rmi local:a = %d, want 1", len(ls)) + } +} + +func TestRmiKeepsSameDigestPinnedByOtherRef(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + config, _ := img.ConfigName() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", img); err != nil { + t.Fatal(err) + } + if n := indexManifestCount(t, s.root); n != 1 { + t.Fatalf("index manifest count before rmi = %d, want 1", n) + } + + rep, err := s.rmi("local:a", false) + if err != nil { + t.Fatalf("rmi local:a: %v", err) + } + if len(rep.Blobs) != 0 || rep.Bytes != 0 { + t.Fatalf("rmi local:a report = %+v, want no blobs removed", rep) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("local:a pin still present after rmi, want gone") + } + if got, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after rmi local:a: %v", err) + } else if got != manifest.String() { + t.Fatalf("local:b digest = %s, want %s", got, manifest) + } + if n := indexManifestCount(t, s.root); n != 1 { + t.Fatalf("index manifest count after rmi local:a = %d, want 1", n) + } + for _, d := range []string{manifest.String(), config.String(), layer.String()} { + if _, err := os.Stat(blobPath(s.root, d)); err != nil { + t.Errorf("blob %s after rmi local:a: %v, want present", d, err) + } + } + if _, err := s.image("local:b"); err != nil { + t.Fatalf("s.image(local:b) after rmi local:a: %v", err) + } +} + +func TestRmiAbsentRefErrors(t *testing.T) { + s := openTestStore(t) + _, err := s.rmi("local:never", false) + if err == nil || !strings.Contains(err.Error(), "not pulled") { + t.Errorf("rmi absent ref err = %v, want an error mentioning \"not pulled\"", err) + } +} + +func TestRmiByDigestDropsStaleTempBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.rmi(shortDigest(manifest.String()), false) + if err != nil { + t.Fatalf("rmi by digest with stale temp blob: %v", err) + } + if rep.Ref != "local:a" { + t.Fatalf("rmi report ref = %q, want local:a", rep.Ref) + } + if !sliceContains(rep.Blobs, stale) { + t.Fatalf("rmi report blobs = %v, want stale temp blob %s", rep.Blobs, stale) + } + if _, err := os.Stat(blobPath(s.root, stale)); !os.IsNotExist(err) { + t.Fatalf("stale temp blob after rmi by digest: %v, want IsNotExist", err) + } +} + +func TestRmiAcceptsDigestFromList(t *testing.T) { + for _, tc := range []struct { + name string + target func(string) string + }{ + {"short hex", shortDigest}, + {"full digest", func(d string) string { return d }}, + {"qualified prefix", func(d string) string { return "sha256:" + shortDigest(d) }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + + rep, err := s.rmi(tc.target(manifest.String()), false) + if err != nil { + t.Fatalf("rmi by digest: %v", err) + } + if rep.Ref != "local:a" { + t.Fatalf("rmi report ref = %q, want local:a", rep.Ref) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after digest rmi") + } + if _, err := os.Stat(blobPath(s.root, manifest.String())); !os.IsNotExist(err) { + t.Fatalf("manifest blob after digest rmi: %v, want IsNotExist", err) + } + }) + } +} + +func TestCmdRmiAcceptsDigestPrintedByList(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + fields := strings.Fields(buf.String()) + if len(fields) < 7 { + t.Fatalf("list output has too few fields:\n%s", buf.String()) + } + listedDigest := fields[6] + + if err := cmdRmi([]string{"--store", s.root, listedDigest}); err != nil { + t.Fatalf("cmdRmi by listed digest %q: %v", listedDigest, err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after cmdRmi by listed digest") + } +} + +func TestRmiDigestPrefixAmbiguous(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", img); err != nil { + t.Fatal(err) + } + + _, err := s.rmi(shortDigest(manifest.String()), false) + if err == nil || !strings.Contains(err.Error(), "ambiguous") || !strings.Contains(err.Error(), "local:a, local:b") { + t.Fatalf("rmi ambiguous digest err = %v, want both refs listed", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Fatalf("local:a pin lost after ambiguous rmi: %v", err) + } + if _, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after ambiguous rmi: %v", err) + } +} + +func TestRmiKeepsCacheForSameDigestPinnedByOtherRef(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", img); err != nil { + t.Fatal(err) + } + rootfs := rootfsForImage(t, s, img) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi local:a with shared cache: %v", err) + } + if _, err := os.Stat(rootfs); err != nil { + t.Fatalf("shared digest rootfs after rmi local:a: %v, want present", err) + } + if _, err := s.digestFor("local:b"); err != nil { + t.Fatalf("local:b pin lost after rmi local:a: %v", err) + } + + // Removing the last ref reclaims the now-unshared cache with the image. + rep, err := s.rmi("local:b", false) + if err != nil { + t.Fatalf("rmi final shared-cache ref: %v", err) + } + if !rep.CacheDropped { + t.Error("rmi last shared-cache ref did not report dropping the cache") + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Fatalf("shared digest rootfs after final rmi: %v, want removed", err) + } +} + +func TestRmiForceDropsCache(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + rootfs := rootfsForImage(t, s, img) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", true); err != nil { + t.Fatalf("rmi --force: %v", err) + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Errorf("rootfs cache after rmi --force: %v, want IsNotExist", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("pin still present after rmi --force, want gone") + } +} + +// TestRmiDropsColdCacheWithoutForce pins the fixed behavior: a plain rmi +// reclaims a cold unpacked cache as part of removing the image, so the natural +// run -> rmi lifecycle needs no --force and leaves no orphan cache. +func TestRmiDropsColdCacheWithoutForce(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + rootfs := rootfsForImage(t, s, img) + if err := os.MkdirAll(rootfs, 0o755); err != nil { + t.Fatal(err) + } + + rep, err := s.rmi("local:a", false) + if err != nil { + t.Fatalf("rmi cold cache without --force: %v", err) + } + if !rep.CacheDropped { + t.Error("rmi did not report dropping the cold cache") + } + if _, err := os.Stat(rootfs); !os.IsNotExist(err) { + t.Errorf("cold cache after rmi: %v, want removed", err) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Error("pin present after rmi, want gone") + } +} + +func TestDigestCachePathChangesWhenRefDigestChanges(t *testing.T) { + s := openTestStore(t) + ref := "local:tag" + + imgA := buildImage(t, []string{"/a"}) + if _, err := s.addImage(ref, imgA); err != nil { + t.Fatal(err) + } + rootfsA := rootfsForImage(t, s, imgA) + + imgB := buildImage(t, []string{"/b"}) + if _, err := s.addImage(ref, imgB); err != nil { + t.Fatal(err) + } + rootfsB := rootfsForImage(t, s, imgB) + + if rootfsA == rootfsB { + t.Fatalf("digest-keyed rootfs path did not change across repull: %s", rootfsA) + } + if got := filepath.Dir(rootfsB); filepath.Base(got) != "sha256" { + t.Fatalf("rootfs path %s not under rootfs/sha256/", rootfsB) + } +} + +func TestDigestCachePathsAvoidRefEncodingCollisions(t *testing.T) { + s := openTestStore(t) + refA := "local/a:b" + refB := "local:a/b" + if legacyCacheNameForRef(refA) != legacyCacheNameForRef(refB) { + t.Fatalf("test refs no longer collide under legacy encoding: %q vs %q", refA, refB) + } + + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + if _, err := s.addImage(refA, imgA); err != nil { + t.Fatal(err) + } + if _, err := s.addImage(refB, imgB); err != nil { + t.Fatal(err) + } + + rootfsA := rootfsForImage(t, s, imgA) + rootfsB := rootfsForImage(t, s, imgB) + if rootfsA == rootfsB { + t.Fatalf("digest-keyed refs collided at %s", rootfsA) + } +} + +// --- prune ------------------------------------------------------------------ + +// writeOrphanBlob writes an unreferenced file under blobs/sha256/ named with a +// valid sha256 digest so the local GC treats it as an unreachable blob. +func writeOrphanBlob(t *testing.T, root, content string) string { + t.Helper() + sum := sha256.Sum256([]byte(content)) + hex := hex.EncodeToString(sum[:]) + p := filepath.Join(root, "blobs", "sha256", hex) + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return "sha256:" + hex +} + +func writeStaleTempBlob(t *testing.T, root, baseDigest string) string { + t.Helper() + name := strings.TrimPrefix(baseDigest, "sha256:") + "1072211852" + p := filepath.Join(root, "blobs", "sha256", name) + if err := os.WriteFile(p, []byte("stale temp blob"), 0o644); err != nil { + t.Fatal(err) + } + return "sha256:" + name +} + +func TestPruneSweepsOrphanBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + manifest, _ := img.Digest() + config, _ := img.ConfigName() + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "orphan-bytes") + + rep, err := s.gc(false) + if err != nil { + t.Fatalf("gc: %v", err) + } + if len(rep.Blobs) != 1 || rep.Blobs[0] != orphan { + t.Errorf("gc removed = %v, want [%s]", rep.Blobs, orphan) + } + if _, err := os.Stat(blobPath(s.root, orphan)); !os.IsNotExist(err) { + t.Errorf("orphan blob after prune: %v, want gone", err) + } + for _, d := range []string{manifest.String(), config.String(), layer.String()} { + if _, err := os.Stat(blobPath(s.root, d)); err != nil { + t.Errorf("referenced blob %s lost after prune: %v", d, err) + } + } +} + +func TestPruneSweepsOrphanAndStaleTempBlobs(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "orphan-bytes") + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.gc(false) + if err != nil { + t.Fatalf("gc with stale temp blob: %v", err) + } + for _, want := range []string{orphan, stale} { + if !sliceContains(rep.Blobs, want) { + t.Fatalf("gc removed = %v, want %s", rep.Blobs, want) + } + if _, err := os.Stat(blobPath(s.root, want)); !os.IsNotExist(err) { + t.Fatalf("blob %s after prune: %v, want gone", want, err) + } + } + if _, err := os.Stat(blobPath(s.root, layer.String())); err != nil { + t.Fatalf("live layer blob after prune: %v, want present", err) + } +} + +func TestPruneDryRunDeletesNothing(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "orphan-bytes") + + rep, err := s.gc(true) + if err != nil { + t.Fatalf("gc --dry-run: %v", err) + } + if len(rep.Blobs) != 1 || rep.Blobs[0] != orphan { + t.Errorf("dry-run gc reported = %v, want [%s]", rep.Blobs, orphan) + } + if _, err := os.Stat(blobPath(s.root, orphan)); err != nil { + t.Errorf("orphan blob deleted under --dry-run: %v, want present", err) + } +} + +func TestPruneDryRunKeepsStaleTempBlob(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + layer := firstLayerDigest(t, img) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + stale := writeStaleTempBlob(t, s.root, layer.String()) + + rep, err := s.gc(true) + if err != nil { + t.Fatalf("gc --dry-run with stale temp blob: %v", err) + } + if !sliceContains(rep.Blobs, stale) { + t.Fatalf("dry-run gc reported = %v, want stale temp blob %s", rep.Blobs, stale) + } + if _, err := os.Stat(blobPath(s.root, stale)); err != nil { + t.Fatalf("stale temp blob deleted under --dry-run: %v, want present", err) + } +} + +func TestPruneCacheDropsUnpulledRootfs(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + // Legacy cache for an unpulled ref "local:b", an orphan cache under the + // digest-keyed layout. + orphanCache := legacyRootfsForRef(s.root, "local:b") + if err := os.MkdirAll(orphanCache, 0o755); err != nil { + t.Fatal(err) + } + + rep, err := s.pruneCaches(pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != orphanCache { + t.Errorf("pruneCaches dropped = %v, want [%s]", rep.CacheDirs, orphanCache) + } + if _, err := os.Stat(orphanCache); !os.IsNotExist(err) { + t.Errorf("orphan rootfs cache after prune --cache: %v, want gone", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Errorf("local:a pin lost after prune --cache: %v", err) + } +} + +func TestPruneCacheAllDropsPulledRootfs(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + liveCache := rootfsForImage(t, s, img) + if err := os.MkdirAll(liveCache, 0o755); err != nil { + t.Fatal(err) + } + + rep, err := s.pruneCaches(pruneOpts{cache: true, all: true}) + if err != nil { + t.Fatalf("pruneCaches --all: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != liveCache { + t.Errorf("pruneCaches --all dropped = %v, want [%s]", rep.CacheDirs, liveCache) + } + if _, err := os.Stat(liveCache); !os.IsNotExist(err) { + t.Errorf("live rootfs cache after prune --cache --all: %v, want gone", err) + } + // --all drops the cache only; the store (pin + blobs) is untouched. + if _, err := s.digestFor("local:a"); err != nil { + t.Errorf("local:a pin lost after prune --cache --all: %v", err) + } +} + +func TestRmiThenPruneIdempotent(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/hello"})); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi: %v", err) + } + rep, err := s.gc(false) + if err != nil { + t.Fatalf("gc after rmi: %v", err) + } + if len(rep.Blobs) != 0 { + t.Errorf("gc after rmi removed %v, want nothing (already reclaimed)", rep.Blobs) + } +} + +// --- sweepable clone detection ---------------------------------------------- + +// TestListSweepableClonesSkipsKeepMarkerAndNonClones pins the reap set the +// bundle sweep uses once it holds run.lock exclusively: every run-- +// clone WITHOUT a keep marker plus any rootfs.tmp-* unpack leftover, and +// nothing else. Pids are irrelevant now; the exclusive lock already proves +// no run is live. +func TestListSweepableClonesSkipsKeepMarkerAndNonClones(t *testing.T) { + dir := t.TempDir() + reapClone := filepath.Join(dir, "run-1234-1") + keepClone := filepath.Join(dir, "run-5678-2") + tmpUnpack := filepath.Join(dir, "rootfs.tmp-abcd") + rootfs := filepath.Join(dir, "rootfs") + for _, p := range []string{reapClone, keepClone, tmpUnpack, rootfs} { + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + } + if err := writeKeepMarker(keepClone); err != nil { + t.Fatal(err) + } + + got := listSweepableClones(dir) + want := map[string]bool{reapClone: true, tmpUnpack: true} + if len(got) != len(want) { + t.Fatalf("listSweepableClones = %v, want %v", got, want) + } + for _, g := range got { + if !want[g] { + t.Fatalf("listSweepableClones included %q, want only %v", g, want) + } + } + + // The read-only lister removes nothing. + for _, p := range []string{reapClone, keepClone, tmpUnpack, rootfs} { + if _, err := os.Stat(p); err != nil { + t.Errorf("listSweepableClones removed %s, want read-only: %v", p, err) + } + } + + // reapSweepableClones removes exactly the sweepable set, keeping the + // marked clone, the base rootfs, and the keep marker's clone dir. + reaped := reapSweepableClones(dir) + if len(reaped) != 2 { + t.Fatalf("reapSweepableClones = %v, want 2 entries", reaped) + } + if _, err := os.Stat(reapClone); !os.IsNotExist(err) { + t.Errorf("unmarked clone not reaped: %v", err) + } + if _, err := os.Stat(tmpUnpack); !os.IsNotExist(err) { + t.Errorf("unpack leftover not reaped: %v", err) + } + if _, err := os.Stat(keepClone); err != nil { + t.Errorf("kept clone was reaped, want preserved: %v", err) + } + if _, err := os.Stat(rootfs); err != nil { + t.Errorf("base rootfs was reaped, want left alone: %v", err) + } +} + +// TestListSweepableClonesPreservesOnMarkerStatError pins the fail-safe: if the +// keep-marker stat returns a transient non-ENOENT error (here EACCES from an +// unreadable keep directory), the clone is preserved rather than reaped; +// reaping a --keep clone on a flaky read would be data loss. +func TestListSweepableClonesPreservesOnMarkerStatError(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission bits do not bind as root") + } + dir := t.TempDir() + clone := filepath.Join(dir, "run-1-1") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + // 0o000 on the keep dir makes os.Stat(.elfuse-keep/run-1-1) fail with + // EACCES (cannot traverse .elfuse-keep), not ENOENT, while dir itself stays + // readable so the clone is still discovered. + if err := os.MkdirAll(keepDirPath(dir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(keepDirPath(dir), 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(keepDirPath(dir), 0o755) }) + + if got := listSweepableClones(dir); len(got) != 0 { + t.Fatalf("listSweepableClones = %v, want empty (fail safe on marker stat error)", got) + } +} + +// TestListSweepableClonesReapsImageShippedKeepFile pins #10/S9: a clone whose +// own contents include /.elfuse-keep (an image that ships that path, or a guest +// that wrote it) is NOT preserved. The keep record lives in the mount-root keep +// directory, outside every guest's view, so only elfuse-oci's --keep can +// set it; forged in-clone files are ignored and the clone is reaped. +func TestListSweepableClonesReapsImageShippedKeepFile(t *testing.T) { + dir := t.TempDir() + clone := filepath.Join(dir, "run-7-7") + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + // The image (or guest) planted /.elfuse-keep inside the clone. + if err := os.WriteFile(filepath.Join(clone, ".elfuse-keep"), nil, 0o644); err != nil { + t.Fatal(err) + } + + got := listSweepableClones(dir) + if len(got) != 1 || got[0] != clone { + t.Fatalf("listSweepableClones = %v, want the clone reapable despite in-clone /.elfuse-keep", got) + } + + // A genuine --keep record in the mount-root keep dir does preserve it. + if err := writeKeepMarker(clone); err != nil { + t.Fatal(err) + } + if got := listSweepableClones(dir); len(got) != 0 { + t.Fatalf("listSweepableClones = %v, want kept clone skipped after writeKeepMarker", got) + } +} + +// TestCSBundleBusy pins the read-only busy probe used by prune --dry-run: a +// bundle whose run.lock is held (a live run) reports busy; an idle one does +// not. +func TestCSBundleBusy(t *testing.T) { + bundle := t.TempDir() + if csBundleBusy(bundle) { + t.Fatal("csBundleBusy(idle) = true, want false") + } + hold, err := acquireFlock(runLockPath(bundle), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + if !csBundleBusy(bundle) { + t.Fatal("csBundleBusy(live run) = false, want true") + } + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if csBundleBusy(bundle) { + t.Fatal("csBundleBusy after release = true, want false") + } +} + +func TestListMissingImage(t *testing.T) { + s := openTestStore(t) + missingDigest := "sha256:" + strings.Repeat("9", 64) + if err := s.savePins(refPins{"missing": missingDigest}); err != nil { + t.Fatal(err) + } + if err := list(&bytes.Buffer{}, s, false); err == nil || !strings.Contains(err.Error(), "list: missing: image") { + t.Fatalf("list missing image err = %v, want image error", err) + } +} + +func TestShortDigest(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"short hex passthrough", "abcdef", "abcdef"}, + {"exactly twelve", "123456789012", "123456789012"}, + {"truncated to twelve", "123456789012345", "123456789012"}, + {"sha256 prefix stripped and truncated", "sha256:abcdef1234567890abcdef00", "abcdef123456"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := shortDigest(tc.in); got != tc.want { + t.Fatalf("shortDigest(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// TestListShowsPlatformVariant pins that a variant-qualified platform is not +// truncated to os/arch in list output. +func TestListShowsPlatformVariant(t *testing.T) { + s := openTestStore(t) + img, err := mutate.ConfigFile(buildImage(t, []string{"/hello"}), &v1.ConfigFile{ + Architecture: "arm64", + OS: "linux", + Variant: "v8", + }) + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:variant", img); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := list(&buf, s, false); err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "linux/arm64/v8") { + t.Fatalf("list output %q missing linux/arm64/v8", buf.String()) + } +} + +// --- plain-rootfs run lock --------------------------------------------------- + +func TestRootfsCacheBusy(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cache") + if rootfsCacheBusy(dir) { + t.Fatal("rootfsCacheBusy(no lock file) = true, want false") + } + hold, err := acquireRootfsRunLock(dir) + if err != nil { + t.Fatal(err) + } + if !rootfsCacheBusy(dir) { + t.Fatal("rootfsCacheBusy(live run) = false, want true") + } + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if rootfsCacheBusy(dir) { + t.Fatal("rootfsCacheBusy after release = true, want false") + } +} + +// TestPruneCacheHonorsRootfsRunLock pins the busy semantics of the plain +// cache sweep: a digest cache whose run lock is held by a live run survives +// prune --cache --all and is never advertised by a dry run; an idle cache +// (including one with a stale lock file left by a dead run) is reclaimed, +// lock file included. +func TestPruneCacheHonorsRootfsRunLock(t *testing.T) { + cases := []struct { + name string + holdLock bool + staleLock bool + dryRun bool + wantGone bool + wantInRep bool + }{ + {name: "live run skipped", holdLock: true}, + {name: "live run hidden from dry-run", holdLock: true, dryRun: true}, + {name: "idle reclaimed", wantGone: true, wantInRep: true}, + {name: "stale lock file reclaimed", staleLock: true, wantGone: true, wantInRep: true}, + {name: "idle dry-run reported only", dryRun: true, wantInRep: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + cache := rootfsForImage(t, s, img) + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + if tc.holdLock { + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + defer hold.Close() + } + if tc.staleLock { + if err := os.WriteFile(rootfsRunLockPath(cache), nil, 0o644); err != nil { + t.Fatal(err) + } + } + + rep, err := s.pruneCaches(pruneOpts{cache: true, all: true, dryRun: tc.dryRun}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + + inRep := sliceContains(rep.CacheDirs, cache) + if inRep != tc.wantInRep { + t.Errorf("reported = %v, want %v (dirs %v)", inRep, tc.wantInRep, rep.CacheDirs) + } + _, statErr := os.Stat(cache) + gone := os.IsNotExist(statErr) + wantGone := tc.wantGone && !tc.dryRun + if gone != wantGone { + t.Errorf("cache gone = %v, want %v (stat err %v)", gone, wantGone, statErr) + } + if wantGone { + if _, err := os.Lstat(rootfsRunLockPath(cache)); !os.IsNotExist(err) { + t.Errorf("lock file after reclaim: %v, want gone", err) + } + } + }) + } +} + +// TestPruneCacheHonorsLockForStagingDir pins that the sweep guards a +// staging dir (.tmp-, unpackImage's pre-rename workspace) by +// the DIGEST's run lock, which the unpacker holds for the whole unpack: a +// mid-flight unpack survives prune --cache --all and never appears in a dry +// run, while a crashed unpack's leftover (digest lock free) is reclaimed. +func TestPruneCacheHonorsLockForStagingDir(t *testing.T) { + cases := []struct { + name string + holdLock bool + dryRun bool + wantGone bool + wantInRep bool + }{ + {name: "live unpack skipped", holdLock: true}, + {name: "live unpack hidden from dry-run", holdLock: true, dryRun: true}, + {name: "crashed leftover reclaimed", wantGone: true, wantInRep: true}, + {name: "crashed leftover dry-run reported only", dryRun: true, wantInRep: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + cache := rootfsForImage(t, s, img) + staging := cache + rootfsStagingSuffix + "123456" + if err := os.MkdirAll(staging, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(staging, "f"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if tc.holdLock { + // The digest's lock, exactly what a live unpacker holds; a + // lock named after the staging dir itself never exists. + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + defer hold.Close() + } + + rep, err := s.pruneCaches(pruneOpts{cache: true, all: true, dryRun: tc.dryRun}) + if err != nil { + t.Fatalf("pruneCaches: %v", err) + } + + inRep := sliceContains(rep.CacheDirs, staging) + if inRep != tc.wantInRep { + t.Errorf("reported = %v, want %v (dirs %v)", inRep, tc.wantInRep, rep.CacheDirs) + } + _, statErr := os.Stat(staging) + gone := os.IsNotExist(statErr) + if gone != tc.wantGone { + t.Errorf("staging gone = %v, want %v (stat err %v)", gone, tc.wantGone, statErr) + } + }) + } +} + +// TestRemoveRootfsCacheMissingDirRespectsLock pins the lock-first rule for +// a cache dir that is already gone: a staging dir vanishes exactly when its +// unpack renames it into place, and that holder still owns the digest lock, +// so removal must refuse rather than unlink the lock out from under it. +// With no holder the stale lock file is swept, and a path with no store +// structure at all is a plain no-op. +func TestRemoveRootfsCacheMissingDirRespectsLock(t *testing.T) { + cache := filepath.Join(t.TempDir(), "sha256", strings.Repeat("ab", 32)) + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + staging := cache + rootfsStagingSuffix + "123456" + + if err := removeRootfsCache(staging); !errors.Is(err, errCacheBusy) { + t.Fatalf("removeRootfsCache(vanished staging, lock held) = %v, want errCacheBusy", err) + } + if _, err := os.Lstat(rootfsRunLockPath(cache)); err != nil { + t.Errorf("digest lock file unlinked by refused removal: %v", err) + } + + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if err := removeRootfsCache(staging); err != nil { + t.Fatalf("removeRootfsCache(vanished staging, idle) = %v", err) + } + if _, err := os.Lstat(rootfsRunLockPath(cache)); !os.IsNotExist(err) { + t.Errorf("stale digest lock survived idle removal: %v", err) + } + + if err := removeRootfsCache(filepath.Join(t.TempDir(), "nope", "sha256", "feed")); err != nil { + t.Fatalf("removeRootfsCache(no store structure) = %v, want nil", err) + } +} + +// TestRmiRefusesLiveRootfsRun pins rmi's fail-closed rule for the plain +// cache: even --force refuses while a live run holds the per-digest lock, +// and the pin survives the refusal; once the run exits the same rmi +// succeeds and reclaims cache and lock file. +func TestRmiRefusesLiveRootfsRun(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + if _, err := s.addImage("local:a", img); err != nil { + t.Fatal(err) + } + cache := rootfsForImage(t, s, img) + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + hold, err := acquireRootfsRunLock(cache) + if err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", true); err == nil || + !strings.Contains(err.Error(), "in use by a live run") { + t.Fatalf("rmi under live run err = %v, want live-run refusal", err) + } + if _, err := s.digestFor("local:a"); err != nil { + t.Errorf("pin lost after refused rmi: %v", err) + } + if _, err := os.Stat(cache); err != nil { + t.Errorf("cache damaged after refused rmi: %v", err) + } + + if err := hold.Close(); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:a", true); err != nil { + t.Fatalf("rmi after run exit: %v", err) + } + if _, err := os.Stat(cache); !os.IsNotExist(err) { + t.Errorf("cache after rmi: %v, want gone", err) + } + if _, err := os.Lstat(rootfsRunLockPath(cache)); !os.IsNotExist(err) { + t.Errorf("lock file after rmi: %v, want gone", err) + } +} diff --git a/cmd/elfuse-oci/list.go b/cmd/elfuse-oci/list.go new file mode 100644 index 00000000..5ce89322 --- /dev/null +++ b/cmd/elfuse-oci/list.go @@ -0,0 +1,146 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/google/go-containerregistry/pkg/v1" +) + +// listEntry is one row of `elfuse-oci list --json`. +type listEntry struct { + Ref string `json:"ref"` + Digest string `json:"digest"` + Platform string `json:"platform"` + Created string `json:"created,omitempty"` + Size int64 `json:"size"` + Layers int `json:"layers"` +} + +// list prints every ref pinned in the store with its manifest digest, platform, +// creation time, total compressed layer size, and layer count. With asJSON it +// emits a JSON array of listEntry; otherwise a human-readable table sorted by +// ref. The size is the sum of the layers' compressed descriptor sizes +// (layer.Size()), matching `docker images` and the blob bytes rmi or a plain +// prune would reclaim, not the uncompressed size, which would require +// streaming every layer and defeat a fast listing. (prune --cache reports a +// different pool entirely: the on-disk allocation of the unpacked caches.) +func list(w io.Writer, s *store, asJSON bool) error { + // Snapshot pins and manifests under the store lock: a concurrent rmi + // deletes both under the same lock, and reading unlocked could observe a + // pin whose descriptor or blobs are already gone and fail mid-listing. + unlock, err := s.lock() + if err != nil { + return err + } + defer unlock() + + pins, err := s.loadPins() + if err != nil { + return err + } + refs := make([]string, 0, len(pins)) + for ref := range pins { + refs = append(refs, ref) + } + sort.Strings(refs) + + entries := make([]listEntry, 0, len(refs)) + for _, ref := range refs { + e := listEntry{Ref: ref, Digest: pins[ref]} + h, err := v1.NewHash(pins[ref]) + if err != nil { + return fmt.Errorf("list: %s: digest %q: %w", ref, pins[ref], err) + } + img, err := s.path.Image(h) + if err != nil { + return fmt.Errorf("list: %s: image: %w", ref, err) + } + cfg, err := img.ConfigFile() + if err != nil { + return fmt.Errorf("list: %s: config: %w", ref, err) + } + e.Platform = cfg.OS + "/" + cfg.Architecture + if cfg.Variant != "" { + e.Platform += "/" + cfg.Variant + } + if !cfg.Created.IsZero() { + e.Created = cfg.Created.UTC().Format("2006-01-02T15:04:05Z") + } + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("list: %s: layers: %w", ref, err) + } + e.Layers = len(layers) + for i, l := range layers { + sz, err := l.Size() + if err != nil { + return fmt.Errorf("list: %s: layer %d size: %w", ref, i, err) + } + e.Size += sz + } + entries = append(entries, e) + } + + if asJSON { + b, err := json.MarshalIndent(entries, "", " ") + if err != nil { + return err + } + fmt.Fprintf(w, "%s\n", b) + return nil + } + if len(entries) == 0 { + return nil + } + fmt.Fprintf(w, "%-40s %-12s %-14s %12s %s\n", "REF", "DIGEST", "PLATFORM", "SIZE", "LAYERS") + for _, e := range entries { + fmt.Fprintf(w, "%-40s %s %-14s %12d %d\n", e.Ref, shortDigest(e.Digest), e.Platform, e.Size, e.Layers) + } + return nil +} + +// shortDigest returns the first 12 hex characters of a "sha256:..." digest. +func shortDigest(d string) string { + if i := strings.Index(d, ":"); i >= 0 { + d = d[i+1:] + } + if len(d) > 12 { + return d[:12] + } + return d +} + +// cmdList implements `elfuse-oci list [--store] [--json]` (alias: images). +func cmdList(args []string) error { + cf, asJSON, err := parseListArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + return list(os.Stdout, s, asJSON) +} + +func parseListArgs(args []string) (commonFlags, bool, error) { + var cf commonFlags + var asJSON bool + fs := newCommandFlagSet("list", &cf) + fs.BoolVar(&asJSON, "json", false, "emit a JSON array of {ref,digest,platform,created,size,layers}") + if err := fs.Parse(args); err != nil { + return cf, false, err + } + if err := noArgs("list", fs.Args()); err != nil { + return cf, false, err + } + return cf, asJSON, nil +} diff --git a/cmd/elfuse-oci/main.go b/cmd/elfuse-oci/main.go index 15407a27..3535bd3a 100644 --- a/cmd/elfuse-oci/main.go +++ b/cmd/elfuse-oci/main.go @@ -18,10 +18,14 @@ // elfuse-oci run [--store DIR] [--entrypoint E] [--env K=V]... // [--user UID[:GID]] [--workdir DIR] [--platform ...] // [args...] +// elfuse-oci list [--store DIR] [--json] (alias: images) +// elfuse-oci rmi [--store DIR] [--force] +// elfuse-oci prune [--store DIR] [--cache] [--all] [--dry-run] // // is an OCI image reference (docker.io/library/alpine:3, ghcr.io/..., -// localhost:5000/foo:tag, or name@sha256:...). The default store is -// $ELFUSE_OCI_STORE or ~/.local/share/elfuse/oci. +// localhost:5000/foo:tag, or name@sha256:...). `rmi` also accepts a unique +// sha256 digest prefix from `list`. The default store is $ELFUSE_OCI_STORE or +// ~/.local/share/elfuse/oci. package main import ( @@ -46,6 +50,9 @@ commands: unpack Unpack a stored image's layers into a rootfs directory inspect Print a stored image's manifest + config run Pull + unpack + exec the image's entrypoint under elfuse + list List refs pinned in the store with digest/platform/size (alias: images) + rmi Remove a ref or unique digest and garbage-collect unreachable blobs + prune Garbage-collect unreachable blobs; --cache also drops rootfs/sparsebundle caches help Show this help version Print the elfuse-oci version @@ -72,6 +79,18 @@ run flags: clone (mutations persist; macOS only) --keep Keep the per-run COW clone and mount for inspection (macOS only) + +list flags: + --json Emit a JSON array of {ref,digest,platform,created,size,layers} + +rmi flags: + --force Also drop a last-ref unpacked cache (rootfs/, cs/ sparsebundle) + before removing; otherwise rmi refuses if a cache is present + +prune flags: + --cache Also drop elfuse unpacked caches (rootfs/ and, on macOS, cs/) + --all With --cache, drop caches even for still-pulled refs + --dry-run Report what would be reclaimed without deleting `) } @@ -108,6 +127,12 @@ func dispatch(cmd string, rest []string) error { return cmdInspect(rest) case "run": return cmdRun(rest) + case "list", "images": + return cmdList(rest) + case "rmi": + return cmdRmi(rest) + case "prune": + return cmdPrune(rest) default: usage() return fmt.Errorf("unknown command: %s", cmd) diff --git a/cmd/elfuse-oci/main_command_test.go b/cmd/elfuse-oci/main_command_test.go index 9a8636da..7a761c07 100644 --- a/cmd/elfuse-oci/main_command_test.go +++ b/cmd/elfuse-oci/main_command_test.go @@ -5,8 +5,12 @@ package main import ( "os/exec" + "path/filepath" "strings" "testing" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1" ) func TestRunDispatchHelpVersionAndErrors(t *testing.T) { @@ -82,3 +86,68 @@ func TestMainSubprocessExitBehavior(t *testing.T) { t.Fatalf("main no-arg stderr = %q, want formatted error", stderr) } } + +func TestRunDispatchesAllSubcommands(t *testing.T) { + root := t.TempDir() + img := tinyImage(t) + withFakeCranePull(t, func(ref string, opts ...crane.Option) (v1.Image, error) { + if ref != "local:tiny" { + t.Fatalf("pull ref = %q, want local:tiny", ref) + } + return img, nil + }) + + stdout, stderr, err := captureOutput(t, func() error { + return run([]string{"pull", "--store", root, "local:tiny"}) + }) + if err != nil || !strings.Contains(stderr, "Pulled local:tiny") { + t.Fatalf("run pull stderr=%q err=%v, want pull summary", stderr, err) + } + + for _, cmd := range []string{"list", "images"} { + stdout, _, err = captureOutput(t, func() error { + return run([]string{cmd, "--store", root}) + }) + if err != nil || !strings.Contains(stdout, "local:tiny") { + t.Fatalf("run %s stdout=%q err=%v, want list row", cmd, stdout, err) + } + } + + stdout, _, err = captureOutput(t, func() error { + return run([]string{"inspect", "--store", root, "local:tiny"}) + }) + if err != nil || !strings.Contains(stdout, "Ref: local:tiny") { + t.Fatalf("run inspect stdout=%q err=%v, want inspect output", stdout, err) + } + + unpackRoot := filepath.Join(t.TempDir(), "unpack-rootfs") + _, stderr, err = captureOutput(t, func() error { + return run([]string{"unpack", "--store", root, "--rootfs", unpackRoot, "local:tiny"}) + }) + if err != nil || !strings.Contains(stderr, "Unpacked local:tiny") { + t.Fatalf("run unpack stderr=%q err=%v, want unpack summary", stderr, err) + } + + withFakeExecElfuse(t, func(rootfs string, spec *runSpec, _ *flockFile) error { return nil }) + runRoot := filepath.Join(t.TempDir(), "run-rootfs") + _, _, err = captureOutput(t, func() error { + return run([]string{"run", "--store", root, "--plain-rootfs", "--rootfs", runRoot, "local:tiny"}) + }) + if err != nil { + t.Fatalf("run run --plain-rootfs: %v", err) + } + + _, stderr, err = captureOutput(t, func() error { + return run([]string{"prune", "--store", root, "--dry-run"}) + }) + if err != nil || !strings.Contains(stderr, "Would reclaim") { + t.Fatalf("run prune stderr=%q err=%v, want dry-run summary", stderr, err) + } + + _, stderr, err = captureOutput(t, func() error { + return run([]string{"rmi", "--store", root, "local:tiny"}) + }) + if err != nil || !strings.Contains(stderr, "Removed local:tiny") { + t.Fatalf("run rmi stderr=%q err=%v, want rmi summary", stderr, err) + } +} diff --git a/cmd/elfuse-oci/prune.go b/cmd/elfuse-oci/prune.go new file mode 100644 index 00000000..7ddba94f --- /dev/null +++ b/cmd/elfuse-oci/prune.go @@ -0,0 +1,225 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "syscall" +) + +// pruneOpts controls a prune pass. +type pruneOpts struct { + cache bool // also drop elfuse rootfs/sparsebundle caches + all bool // with cache: drop caches even for still-pulled refs + dryRun bool // report only; delete nothing +} + +// pruneReport summarizes a prune pass: blobs reclaimed, cache dirs dropped, +// and the approximate bytes freed. The total mixes two accountings: blob +// bytes are logical file sizes, cache-dir bytes are on-disk allocation +// (st_blocks, the honest figure for sparse files and APFS clones), so it is +// an estimate, not one uniform metric. +type pruneReport struct { + Blobs []string + CacheDirs []string + Bytes int64 +} + +// cmdPrune implements `elfuse-oci prune [--store] [--cache] [--all] [--dry-run]`. +// +// Without --cache, prune runs a reachability GC over blobs/sha256/ and reclaims +// any blob not reachable from an index.json manifest descriptor (retag or +// partial-pull orphans). With --cache it additionally drops elfuse's unpacked +// caches: the plain rootfs// directories and, on darwin, the +// case-sensitive sparsebundle bundles (cs//). By default only +// digest-keyed caches no longer reachable from refs.json are dropped, plus any +// legacy ref-named caches; --all drops every cache, including for still-pulled +// refs. --dry-run reports what would be removed without deleting. --all +// requires --cache (it has no meaning for the blob GC, which is already +// unconditional). +func cmdPrune(args []string) error { + cf, opts, err := parsePruneArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + + // The whole sweep runs under the store lock: the GC's reachability scan + // must not race a concurrent pull, whose fresh blobs land before the + // index descriptor that makes them reachable and would otherwise be + // reclaimed in the window between the two writes. rmi already runs its + // own gc under this lock. Reporting below happens after the lock drops. + var rep pruneReport + err = s.withLock(func() error { + gcr, err := s.gc(opts.dryRun) + if err != nil { + return err + } + rep.Blobs = gcr.Blobs + rep.Bytes += gcr.Bytes + + if opts.cache { + cr, err := s.pruneCaches(opts) + if err != nil { + return err + } + rep.CacheDirs = cr.CacheDirs + rep.Bytes += cr.Bytes + } + return nil + }) + if err != nil { + return err + } + + verb := "Reclaimed" + if opts.dryRun { + verb = "Would reclaim" + } + fmt.Fprintf(os.Stderr, "%s: %d blob(s), %d cache dir(s), ~%d bytes\n", + verb, len(rep.Blobs), len(rep.CacheDirs), rep.Bytes) + for _, b := range rep.Blobs { + fmt.Fprintf(os.Stderr, " blob %s\n", b) + } + for _, d := range rep.CacheDirs { + fmt.Fprintf(os.Stderr, " cache %s\n", d) + } + return nil +} + +func parsePruneArgs(args []string) (commonFlags, pruneOpts, error) { + var cf commonFlags + var opts pruneOpts + fs := newCommandFlagSet("prune", &cf) + fs.BoolVar(&opts.cache, "cache", false, "also drop unpacked caches (rootfs/ and, on macOS, cs/ sparsebundles)") + fs.BoolVar(&opts.all, "all", false, "with --cache, drop caches even for still-pulled refs") + fs.BoolVar(&opts.dryRun, "dry-run", false, "report what would be reclaimed without deleting") + if err := fs.Parse(args); err != nil { + return cf, opts, err + } + if err := noArgs("prune", fs.Args()); err != nil { + return cf, opts, err + } + if opts.all && !opts.cache { + return cf, opts, fmt.Errorf("prune: --all requires --cache") + } + return cf, opts, nil +} + +// pruneRootfsCaches drops plain rootfs// cache directories. Without +// opts.all, only dirs whose digest key is not live are dropped; with opts.all, +// every digest cache is dropped. Pre-digest rootfs/ caches are +// always treated as orphaned legacy caches because they are no longer used by +// run/unpack. Shared across platforms; the darwin-only sparsebundle sweep lives +// in cache_darwin.go. +func pruneRootfsCaches(s *store, live map[string]bool, opts pruneOpts) (pruneReport, error) { + var rep pruneReport + base := filepath.Join(s.root, rootfsCacheDirName) + entries, err := os.ReadDir(base) + if err != nil { + if os.IsNotExist(err) { + return rep, nil + } + return rep, err + } + for _, e := range entries { + if !e.IsDir() { + continue + } + top := filepath.Join(base, e.Name()) + if e.Name() == "sha256" { + children, err := os.ReadDir(top) + if err != nil { + return rep, err + } + for _, child := range children { + if !child.IsDir() { + continue + } + key := filepath.Join("sha256", child.Name()) + if !opts.all && live[key] { + continue + } + dir := filepath.Join(top, child.Name()) + // A cache whose per-digest run lock is held belongs to a live + // --plain-rootfs guest: skip it, and never advertise it in a + // dry run, the way the bundle sweep skips a busy sparsebundle. + // This also covers the non---all case where a re-pull moved + // the pin off a digest a guest is still running from. + if opts.dryRun { + if rootfsCacheBusy(dir) { + continue + } + rep.Bytes += dirSize(dir) + rep.CacheDirs = append(rep.CacheDirs, dir) + continue + } + size := dirSize(dir) + if err := removeRootfsCache(dir); err != nil { + if errors.Is(err, errCacheBusy) { + continue + } + return rep, err + } + rep.Bytes += size + rep.CacheDirs = append(rep.CacheDirs, dir) + } + continue + } + + // Legacy ref-named rootfs caches are no longer live under the + // digest-keyed scheme; prune --cache reclaims them as orphan caches. + // No run ever locks a legacy path, but the same locked removal keeps + // one code path. + dir := top + if opts.dryRun { + rep.Bytes += dirSize(dir) + rep.CacheDirs = append(rep.CacheDirs, dir) + continue + } + size := dirSize(dir) + if err := removeRootfsCache(dir); err != nil { + if errors.Is(err, errCacheBusy) { + continue + } + return rep, err + } + rep.Bytes += size + rep.CacheDirs = append(rep.CacheDirs, dir) + } + return rep, nil +} + +// dirSize reports the on-disk allocation (stat block count, not logical file +// size) of a tree, so a sparse APFS sparsebundle image is measured by the bytes +// it actually occupies rather than its 16g virtual ceiling. Best-effort: walk +// errors are ignored so a busy/evaporating entry does not abort the sweep. +func dirSize(path string) int64 { + var total int64 + _ = filepath.WalkDir(path, func(_ string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + if info, err := d.Info(); err == nil { + total += diskUsage(info) + } + return nil + }) + return total +} + +// diskUsage returns the bytes actually allocated to a file (Blocks * 512) when +// the platform exposes stat blocks, falling back to logical size otherwise. +func diskUsage(fi os.FileInfo) int64 { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + return int64(st.Blocks) * 512 + } + return fi.Size() +} diff --git a/cmd/elfuse-oci/rmi.go b/cmd/elfuse-oci/rmi.go new file mode 100644 index 00000000..171ef6fe --- /dev/null +++ b/cmd/elfuse-oci/rmi.go @@ -0,0 +1,59 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" +) + +// cmdRmi implements `elfuse-oci rmi [--store] [--force] `. +// +// rmi drops the selected ref's pin. The target may be an exact ref or a unique +// sha256 digest prefix from `list`. If no remaining ref pins the same manifest +// digest, it removes that descriptor from index.json, garbage-collects the +// now-unreachable blobs, and reclaims the image's unpacked cache (rootfs/ or cs/ +// sparsebundle); the cache is derived state that goes with the image. A blob, +// descriptor, or cache still reachable through another pinned ref is kept. rmi +// refuses when a live run still uses the cache (never overridable) or when the +// cache holds run --keep retained output (then --force discards it). An absent +// ref/digest is an error. +func cmdRmi(args []string) error { + cf, force, ref, err := parseRmiArgs(args) + if err != nil { + return err + } + s, err := cf.openResolvedStore() + if err != nil { + return err + } + rep, err := s.rmi(ref, force) + if err != nil { + return err + } + removed := ref + if rep.Ref != "" { + removed = rep.Ref + } + fmt.Fprintf(os.Stderr, "Removed %s: %d blob(s), %d bytes\n", removed, len(rep.Blobs), rep.Bytes) + for _, b := range rep.Blobs { + fmt.Fprintf(os.Stderr, " blob %s\n", b) + } + if rep.CacheDropped { + fmt.Fprintf(os.Stderr, " dropped unpacked cache\n") + } + return nil +} + +func parseRmiArgs(args []string) (commonFlags, bool, string, error) { + var cf commonFlags + var force bool + fs := newCommandFlagSet("rmi", &cf) + fs.BoolVar(&force, "force", false, "discard run --keep retained output when removing an image that has it") + if err := fs.Parse(args); err != nil { + return cf, false, "", err + } + ref, err := oneArg("rmi", fs.Args(), "") + return cf, force, ref, err +} diff --git a/cmd/elfuse-oci/rootfslock.go b/cmd/elfuse-oci/rootfslock.go new file mode 100644 index 00000000..98abec08 --- /dev/null +++ b/cmd/elfuse-oci/rootfslock.go @@ -0,0 +1,168 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + + "golang.org/x/sys/unix" +) + +// Per-digest plain-rootfs cache locks. +// +// The plain digest-keyed rootfs cache (/rootfs///) is the +// same kind of shared mutable state as a sparsebundle bundle: a +// --plain-rootfs run executes a guest out of it while prune --cache and rmi +// want to RemoveAll it. Liveness follows the bundlelock.go discipline (a +// held flock proves a live holder regardless of pids) with one lock +// instead of two: there is no mount lifecycle to serialize (no attach.lock +// analog), and concurrent cold unpacks already reconcile through +// unpackImage's stage-and-rename. +// +// The lock is a SIBLING file (.lock next to the cache dir), not a file +// inside it: the cache dir is the guest's /, so a lock inside would appear +// at the guest's / in every run, and the dir's very existence is the +// "fully unpacked" signal that unpackImage publishes by atomic rename. +// Prune's sweep enumerations skip non-directories, so the lock file is +// invisible to them. +// +// - A run (and a store-cache unpack) holds the lock SHARED from before the +// cache-existence probe until the guest exits. The plain run path execs +// elfuse in place, so PreserveAcrossExec threads the descriptor through +// the exec: the kernel releases the flock exactly when the elfuse +// process exits, SIGKILL included, the same no-leaked-liveness property +// run.lock gives the sparsebundle path. +// - prune --cache takes it EXCLUSIVE non-blocking and skips a busy cache; +// rmi refuses a busy cache outright. The remover deletes the lock file +// while still holding the lock; acquireFlock's stat-after-lock guard +// already handles a racing acquirer. +// +// Ordering stays acyclic: runs take the rootfs lock while holding no store +// lock, and prune/rmi (which do hold the store lock) only ever probe the +// rootfs lock non-blocking. + +func rootfsRunLockPath(dir string) string { return dir + ".lock" } + +// rootfsCacheLockPath returns the lock file guarding a cache dir in the +// prune sweep. A published cache is guarded by its sibling .lock. +// A staging dir .tmp- (unpackImage's pre-rename workspace, a +// temp sibling in the same parent) is guarded by the SAME .lock: the +// unpacker holds that lock from before the cache-existence probe until the +// guest exits, so a probe on a lock named after the staging dir itself +// proves nothing and would let the sweep reclaim a tree a live unpack is +// still writing. A staging dir whose digest lock is free is a crashed +// unpack's leftover and remains reclaimable. +func rootfsCacheLockPath(dir string) string { + if hex, _, isStaging := strings.Cut(filepath.Base(dir), rootfsStagingSuffix); isStaging { + return rootfsRunLockPath(filepath.Join(filepath.Dir(dir), hex)) + } + return rootfsRunLockPath(dir) +} + +// acquireRootfsRunLock takes dir's run lock shared, blocking: if a prune is +// mid-removal the run waits it out and then re-unpacks the reclaimed cache. +// The parent directory is created first so a cold store can take the lock +// before its first unpack. +func acquireRootfsRunLock(dir string) (*flockFile, error) { + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return nil, err + } + return acquireFlock(rootfsRunLockPath(dir), syscall.LOCK_SH) +} + +// rootfsCacheBusy reports whether a live run holds dir's run lock. Used by +// prune's dry-run so it never advertises a reap the real pass would refuse. +// A missing lock file means no holder (holders create it), so the probe +// creates no state; any other failure fails closed to busy. +func rootfsCacheBusy(dir string) bool { + f, err := os.OpenFile(rootfsCacheLockPath(dir), os.O_RDWR, 0) + if err != nil { + return !os.IsNotExist(err) + } + defer f.Close() + if err := flockRetryIntr(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return true + } + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return false +} + +// lockRootfsCacheForRemoval takes dir's run lock exclusively, non-blocking. +// busy=true means a live run holds it and the cache must be left alone. On +// success the caller removes the cache dir and the lock file, then unlocks, +// the same unlock-after-removal rule sweepCSBundle follows, so a blocked +// acquirer never sees a half-removed tree. +func lockRootfsCacheForRemoval(dir string) (unlock func(), busy bool, err error) { + l, err := acquireFlock(rootfsCacheLockPath(dir), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + if errors.Is(err, errCacheBusy) { + return func() {}, true, nil + } + return func() {}, false, err + } + return func() { l.Close() }, false, nil +} + +// removeRootfsCache removes dir and its guarding lock file under an +// exclusive run lock, returning errCacheBusy (wrapped) when a live holder +// pins the cache: a running guest for a published dir, a mid-flight unpack +// for a staging dir (both hold the digest's lock, see rootfsCacheLockPath). +// The lock comes first even when dir is already gone: a vanished staging +// dir usually means its unpack just renamed it into place, and the holder +// still owns the digest lock, which an unlocked cleanup here would unlink +// out from under it. A missing lock parent means nothing was ever unpacked +// or locked, so an rmi of a run-less image stays a no-op that conjures no +// store structure. +func removeRootfsCache(dir string) error { + unlock, busy, err := lockRootfsCacheForRemoval(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer unlock() + if busy { + return fmt.Errorf("%s: %w", dir, errCacheBusy) + } + if err := os.RemoveAll(dir); err != nil { + return err + } + if err := os.Remove(rootfsCacheLockPath(dir)); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// removeRootfsCacheForDigest is the shared plain-cache half of the darwin +// and non-darwin removeRefCaches: delete digest's unpacked plain rootfs, +// refusing while a live run uses it. +func removeRootfsCacheForDigest(s *store, digest string) error { + rootfs, err := defaultRootfsForDigest(s.root, digest) + if err != nil { + return err + } + if err := removeRootfsCache(rootfs); err != nil { + if errors.Is(err, errCacheBusy) { + return fmt.Errorf( + "cache for %s is in use by a live run; stop it before removing the image", + digest) + } + return err + } + return nil +} + +// PreserveAcrossExec clears FD_CLOEXEC on the lock's descriptor so the flock +// rides through syscall.Exec into the replacement process and is released by +// the kernel exactly when that process exits, SIGKILL included. +func (l *flockFile) PreserveAcrossExec() error { + _, err := unix.FcntlInt(l.f.Fd(), unix.F_SETFD, 0) + return err +} diff --git a/cmd/elfuse-oci/run.go b/cmd/elfuse-oci/run.go index 50aaeb3b..efa859f5 100644 --- a/cmd/elfuse-oci/run.go +++ b/cmd/elfuse-oci/run.go @@ -9,6 +9,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "runtime" "syscall" ) @@ -59,7 +60,13 @@ func elfuseArgv(rootfs string, spec *runSpec) []string { // becomes elfuse in place, so the invoking shell reaps the same pid and // terminal signals such as Ctrl-C go straight to elfuse rather than through a // Go middleman. -func execElfuse(rootfs string, spec *runSpec) error { +// +// A non-nil lock is the store cache's per-digest run lock. Its descriptor is +// made exec-survivable so the kernel holds the flock for exactly the elfuse +// process's lifetime (SIGKILL included) and releases it at guest exit; the +// one inherited fd is the price of not leaving a Go wrapper alive to babysit +// the lock. +func execElfuse(rootfs string, spec *runSpec, lock *flockFile) error { bin, err := elfuseBin() if err != nil { return err @@ -67,10 +74,17 @@ func execElfuse(rootfs string, spec *runSpec) error { if _, err := os.Stat(bin); err != nil { return fmt.Errorf("elfuse binary not found at %s (set $ELFUSE_BIN): %w", bin, err) } - if err := syscall.Exec(bin, elfuseArgv(rootfs, spec), os.Environ()); err != nil { - return fmt.Errorf("exec %s: %w", bin, err) + if lock != nil { + if err := lock.PreserveAcrossExec(); err != nil { + return fmt.Errorf("preserve rootfs lock across exec: %w", err) + } } - return nil // unreachable + err = syscall.Exec(bin, elfuseArgv(rootfs, spec), os.Environ()) + // Unreachable on success. Keep the lock's *os.File live until the exec + // verdict so its finalizer cannot close the fd (dropping the flock) in + // the window before the process image is replaced. + runtime.KeepAlive(lock) + return fmt.Errorf("exec %s: %w", bin, err) } // spawnElfuseWait runs elfuse as a child and waits for it, returning the exit diff --git a/cmd/elfuse-oci/run_test.go b/cmd/elfuse-oci/run_test.go index b8b9a2a7..911a8a63 100644 --- a/cmd/elfuse-oci/run_test.go +++ b/cmd/elfuse-oci/run_test.go @@ -120,7 +120,7 @@ func TestElfuseBinEnvAndFallback(t *testing.T) { func TestExecElfuseMissingBinary(t *testing.T) { t.Setenv("ELFUSE_BIN", filepath.Join(t.TempDir(), "missing-elfuse")) - err := execElfuse(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}) + err := execElfuse(t.TempDir(), &runSpec{Args: []string{"/bin/true"}, Workdir: "/", UID: 0, GID: 0}, nil) if err == nil || !strings.Contains(err.Error(), "elfuse binary not found") { t.Fatalf("execElfuse missing binary err = %v, want not found", err) } diff --git a/cmd/elfuse-oci/sparsebundle_test.go b/cmd/elfuse-oci/sparsebundle_test.go index 01b027c8..8d106232 100644 --- a/cmd/elfuse-oci/sparsebundle_test.go +++ b/cmd/elfuse-oci/sparsebundle_test.go @@ -11,27 +11,10 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" ) -// withDarwinCacheSeams swaps the darwin mount-probe and force-detach seams -// for a test and restores them on cleanup. -func withDarwinCacheSeams(t *testing.T, isMount func(string) bool, detach func(string) error) { - t.Helper() - oldIsMount := isMountPointFn - oldDetach := detachForce - if isMount != nil { - isMountPointFn = isMount - } - if detach != nil { - detachForce = detach - } - t.Cleanup(func() { - isMountPointFn = oldIsMount - detachForce = oldDetach - }) -} - // hdiutil attach -plist output is an array of system entity dictionaries. We // only care about the mount-point. This fixture is a trimmed-down capture of the // real shape (system-entities -> dict -> mount-point key/string). @@ -396,6 +379,138 @@ func TestProvisionCaseSensitiveWithFakeHdiutilFailures(t *testing.T) { } } +// withMountSeam overrides the mount-point probe directly (not via any shared +// seam helper) so these lock-behavior tests are self-contained. +func withMountSeam(t *testing.T, fn func(string) bool) { + t.Helper() + old := isMountPointFn + isMountPointFn = fn + t.Cleanup(func() { isMountPointFn = old }) +} + +// TestProvisionSharesLiveMount pins the F1 fix: when live runs of the digest +// hold run.lock, a new provision must share the already-attached volume, not +// force-detach it out from under the running guests. +func TestProvisionSharesLiveMount(t *testing.T) { + installFakeHdiutil(t) + bundle := t.TempDir() + requested := filepath.Join(t.TempDir(), "mnt") + withMountSeam(t, func(p string) bool { return p == requested }) + oldDetach := detachForce + detachForce = func(p string) error { + t.Errorf("detachForce(%s) called although a live run holds the volume", p) + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + holder, err := acquireFlock(runLockPath(bundle), syscall.LOCK_SH) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + + m, err := provisionCaseSensitive(bundle, requested, "32m") + if err != nil { + t.Fatalf("provisionCaseSensitive with live run: %v", err) + } + if m.mountPath != requested || !m.owned { + t.Fatalf("mount = %+v, want shared attach at requested mount", m) + } + + // The new run holds run.lock shared: an exclusive probe must report busy. + if _, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("run.lock probe err = %v, want errCacheBusy while run is live", err) + } + + // Close with the other holder still live: last-one-out must NOT detach + // (the detachForce seam above fails the test if it does). + if err := m.Close(); err != nil { + t.Fatalf("Close with surviving run: %v", err) + } +} + +// TestProvisionRejectsSymlinkedMountPath pins the G4 fix: a symlinked mount +// path must be refused before any mount-status probe or force-detach, so a +// tampered cache cannot trick provision into detaching an unrelated volume. +func TestProvisionRejectsSymlinkedMountPath(t *testing.T) { + installFakeHdiutil(t) + t.Setenv("HDIUTIL_MOUNT", t.TempDir()) + bundle := t.TempDir() + requested := filepath.Join(t.TempDir(), "mnt") + if err := os.Symlink(t.TempDir(), requested); err != nil { + t.Fatal(err) + } + // Even if the path reads as a mount point, the symlink guard must win and + // no detach may run. + withMountSeam(t, func(string) bool { return true }) + oldDetach := detachForce + detachForce = func(p string) error { + t.Errorf("detachForce(%s) called on a symlinked mount path", p) + return nil + } + t.Cleanup(func() { detachForce = oldDetach }) + + _, err := provisionCaseSensitive(bundle, requested, "32m") + if err == nil || !strings.Contains(err.Error(), "is a symlink") { + t.Fatalf("provisionCaseSensitive(symlink) err = %v, want 'is a symlink'", err) + } +} + +// TestProvisionDetachesStaleMountAndHoldsRunLock pins the crash-recovery +// path: with no live run holding run.lock, a leftover attached mount is +// provably stale: provision detaches it, re-attaches cleanly, and the +// returned mount holds run.lock shared until Close, whose last-one-out probe +// then detaches. +func TestProvisionDetachesStaleMountAndHoldsRunLock(t *testing.T) { + installFakeHdiutil(t) + bundle := t.TempDir() + requested := filepath.Join(t.TempDir(), "requested-mnt") + actualMount := filepath.Join(t.TempDir(), "actual-mount") + if err := os.MkdirAll(actualMount, 0o755); err != nil { + t.Fatal(err) + } + detachLog := filepath.Join(t.TempDir(), "detach.log") + t.Setenv("HDIUTIL_MOUNT", actualMount) + t.Setenv("HDIUTIL_DETACH_LOG", detachLog) + // The stale leftover: the requested mount point reads as attached. + withMountSeam(t, func(p string) bool { return p == requested }) + + m, err := provisionCaseSensitive(bundle, requested, "32m") + if err != nil { + t.Fatalf("provisionCaseSensitive: %v", err) + } + b, err := os.ReadFile(detachLog) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), requested) { + t.Fatalf("detach log = %q, want stale mount %s detached", b, requested) + } + if m.mountPath != actualMount { + t.Fatalf("mountPath = %q, want re-attached %q", m.mountPath, actualMount) + } + + // Liveness is held from provision until Close. + if _, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) { + t.Fatalf("run.lock probe err = %v, want errCacheBusy while mount is open", err) + } + if err := m.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + b, err = os.ReadFile(detachLog) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), actualMount) { + t.Fatalf("detach log = %q, want last-one-out detach of %s", b, actualMount) + } + free, err := acquireFlock(runLockPath(bundle), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + t.Fatalf("run.lock probe after Close err = %v, want free", err) + } + free.Close() +} + // TestParseMountpointDecodesXMLEntities pins entity decoding: hdiutil's plist // escapes XML-special characters in the mount path (a store path may carry // "&" or "'"), and the raw escaped form would make every later use of the diff --git a/cmd/elfuse-oci/store.go b/cmd/elfuse-oci/store.go index 5dc4ab2c..871d1794 100644 --- a/cmd/elfuse-oci/store.go +++ b/cmd/elfuse-oci/store.go @@ -8,6 +8,8 @@ import ( "fmt" "os" "path/filepath" + "sort" + "strings" "syscall" "github.com/google/go-containerregistry/pkg/v1" @@ -61,7 +63,47 @@ func writeIfAbsent(path string, data []byte) error { } else if !os.IsNotExist(err) { return err } - return os.WriteFile(path, data, 0o644) + // Durable even on cold-store bootstrap: a crash mid-write must not leave a + // truncated oci-layout or index.json that later opens fail to parse. + return writeFileDurable(path, data, 0o644) +} + +// writeFileDurable writes data to path atomically and durably: a uniquely +// named temp sibling is written, fsynced, and renamed into place, then the +// parent directory is fsynced so the rename itself survives a crash. A crash +// leaves either the prior file or the complete new one, never a truncated +// mix. This is the store's shared durability primitive for refs.json and +// index.json; rmi's crash-ordering rule (index.json committed before +// refs.json) holds only if each write is individually durable, which the +// layout package's plain os.WriteFile is not. +func writeFileDurable(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + // A unique temp name: a fixed name would let two writers clobber each + // other's half-written temp even before the rename race. + tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) // no-op once the rename succeeds + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(perm); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmp.Name(), path); err != nil { + return err + } + return fsyncDir(dir) } // refPins maps an image reference to its manifest digest ("sha256:..."). @@ -89,48 +131,85 @@ func (s *store) savePins(p refPins) error { if err != nil { return err } - // Unique temp name: a fixed name would let two writers clobber each - // other's half-written temp even before the rename race. - tmp, err := os.CreateTemp(s.root, ".refs.json.*") + // Durability, not just atomicity: rmi's crash-ordering argument (commit the + // index.json descriptor removal before dropping the pin) only holds if the + // new refs.json cannot revert to an old pin after a crash. + return writeFileDurable(filepath.Join(s.root, "refs.json"), b, 0o644) +} + +// fsyncDir flushes a directory's entries (renames, unlinks) to stable +// storage. +func fsyncDir(dir string) error { + f, err := os.Open(dir) if err != nil { return err } - defer os.Remove(tmp.Name()) // no-op once the rename succeeds - if _, err := tmp.Write(b); err != nil { - tmp.Close() - return err - } - if err := tmp.Chmod(0o644); err != nil { - tmp.Close() + defer f.Close() + return f.Sync() +} + +// fsyncFile flushes an existing file's data to stable storage. Used to make a +// blob durable before the pin that references it is committed. +func fsyncFile(path string) error { + f, err := os.Open(path) + if err != nil { return err } - // Durability, not just atomicity: rmi's crash-ordering argument (drop the - // pin before dropping the descriptor) only holds if the new refs.json - // cannot revert to an old pin after a crash. Sync the temp file before the - // rename and the directory after it, so the rename is durable once this - // returns. - if err := tmp.Sync(); err != nil { - tmp.Close() + defer f.Close() + return f.Sync() +} + +// syncAppendedImage makes img's on-disk state durable after AppendImage but +// before the pin is committed: the config and layer blobs, then index.json, +// then the store directory. Without this the pin (refs.json) is fsynced while +// the blobs and index it references may still sit in the page cache, so a +// crash could leave a durable pin over content that never reached disk, which +// every later resolve of the ref then fails on. The layout package writes +// blobs and index.json with plain os.WriteFile, so the durability is ours to +// add here. +func (s *store) syncAppendedImage(img v1.Image) error { + digests, err := imageBlobDigests(img) + if err != nil { return err } - if err := tmp.Close(); err != nil { - return err + for _, h := range digests { + p := filepath.Join(s.root, "blobs", h.Algorithm, h.Hex) + if err := fsyncFile(p); err != nil { + return err + } } - if err := os.Rename(tmp.Name(), filepath.Join(s.root, "refs.json")); err != nil { + if err := fsyncFile(filepath.Join(s.root, "index.json")); err != nil { return err } return fsyncDir(s.root) } -// fsyncDir flushes a directory's entries (renames, unlinks) to stable -// storage. -func fsyncDir(dir string) error { - f, err := os.Open(dir) +// imageBlobDigests returns the hashes of every blob img introduces: its +// manifest, config, and layers. +func imageBlobDigests(img v1.Image) ([]v1.Hash, error) { + var hs []v1.Hash + mh, err := img.Digest() if err != nil { - return err + return nil, err } - defer f.Close() - return f.Sync() + hs = append(hs, mh) + ch, err := img.ConfigName() + if err != nil { + return nil, err + } + hs = append(hs, ch) + layers, err := img.Layers() + if err != nil { + return nil, err + } + for _, l := range layers { + lh, err := l.Digest() + if err != nil { + return nil, err + } + hs = append(hs, lh) + } + return hs, nil } // lock takes an exclusive advisory flock on /.lock and returns the @@ -199,6 +278,56 @@ func (s *store) digestFor(ref string) (string, error) { return d, nil } +// resolvePinnedTarget resolves an exact pulled ref, or a unique sha256 digest +// prefix such as the 12-character digest printed by `list`. +func resolvePinnedTarget(pins refPins, target string) (string, string, error) { + if d, ok := pins[target]; ok { + return target, d, nil + } + + prefix, ok := digestPrefix(target) + if !ok { + return "", "", fmt.Errorf("store: %q %w (run `elfuse-oci pull %s` first)", target, errNotPulled, target) + } + + var matches []string + matchDigest := "" + for ref, digest := range pins { + h, err := v1.NewHash(digest) + if err != nil { + return "", "", fmt.Errorf("store: pinned digest for %q: %w", ref, err) + } + if h.Algorithm == "sha256" && strings.HasPrefix(h.Hex, prefix) { + matches = append(matches, ref) + matchDigest = digest + } + } + sort.Strings(matches) + + switch len(matches) { + case 0: + return "", "", fmt.Errorf("store: digest %q %w", target, errNotPulled) + case 1: + return matches[0], matchDigest, nil + default: + return "", "", fmt.Errorf("store: digest %q is ambiguous; matches refs: %s", target, strings.Join(matches, ", ")) + } +} + +func digestPrefix(target string) (string, bool) { + if i := strings.IndexByte(target, ':'); i >= 0 { + if target[:i] != "sha256" { + return "", false + } + target = target[i+1:] + } + target = strings.ToLower(target) + if len(target) < 12 || len(target) > 64 || !isLowerHex(target) { + return "", false + } + return target, true +} + // addImage appends img to the layout index if its manifest is not already // present (dedup by digest), and pins ref to that digest. Returns the digest. // The store lock covers the whole check-append-pin sequence: index.json is @@ -224,6 +353,12 @@ func (s *store) addImage(ref string, img v1.Image) (string, error) { if err := s.path.AppendImage(img); err != nil { return fmt.Errorf("store: append image: %w", err) } + // Make the blobs and index.json durable before the pin that will + // point at them, so a crash never strands a fsynced pin over + // content still in the page cache. + if err := s.syncAppendedImage(img); err != nil { + return fmt.Errorf("store: sync appended image: %w", err) + } } return s.pinLocked(ref, d.String()) }) diff --git a/cmd/elfuse-oci/store_test.go b/cmd/elfuse-oci/store_test.go index 9b9073c2..6bfa1a4e 100644 --- a/cmd/elfuse-oci/store_test.go +++ b/cmd/elfuse-oci/store_test.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "errors" "fmt" "os" @@ -11,6 +12,7 @@ import ( "strings" "sync" "testing" + "time" ) // TestDigestForErrorKinds pins the distinction cmdRun's auto-pull relies on: @@ -83,6 +85,80 @@ func TestPinConcurrentWritersKeepAllEntries(t *testing.T) { } } +// TestRmiKeepsPinWhenDescriptorRemovalFails pins the rmi write ordering: +// index.json must be updated before the pin is dropped from refs.json. In the +// reverse order a failure between the writes strands the manifest: the ref +// no longer resolves while the descriptor keeps all blobs live, and prune +// never removes descriptors. With the correct order the pin survives the +// failure and a retried rmi completes. +func TestRmiKeepsPinWhenDescriptorRemovalFails(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("running as root: a read-only store dir cannot induce the write failure") + } + s := openTestStore(t) + if _, err := s.addImage("local:stuck", buildImage(t, []string{"/a"})); err != nil { + t.Fatal(err) + } + // index.json is now written atomically (temp + rename), so a read-only + // index.json no longer blocks the write: a rename replaces the file + // regardless of its mode. Make the store directory read-only instead, so + // the descriptor removal's temp create fails before refs.json is touched. + if err := os.Chmod(s.root, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(s.root, 0o755) }) + + if _, err := s.rmi("local:stuck", false); err == nil { + t.Fatal("rmi succeeded although the store directory is read-only") + } + pins, err := s.loadPins() + if err != nil { + t.Fatal(err) + } + if _, ok := pins["local:stuck"]; !ok { + t.Fatal("pin dropped although descriptor removal failed; image is stranded") + } + + if err := os.Chmod(s.root, 0o755); err != nil { + t.Fatal(err) + } + if _, err := s.rmi("local:stuck", false); err != nil { + t.Fatalf("retried rmi after transient failure: %v", err) + } + pins, err = s.loadPins() + if err != nil { + t.Fatal(err) + } + if _, ok := pins["local:stuck"]; ok { + t.Fatal("pin still present after successful retried rmi") + } +} + +// TestGCReclaimsOrphanKeepsLive exercises store.gc directly (the lifecycle tests +// otherwise only reach it through rmi/prune): an unreferenced blob is reclaimed +// while a still-pinned image's own manifest/config/layers survive. +func TestGCReclaimsOrphanKeepsLive(t *testing.T) { + s := openTestStore(t) + if _, err := s.addImage("local:a", buildImage(t, []string{"/a"})); err != nil { + t.Fatal(err) + } + orphan := writeOrphanBlob(t, s.root, "gc-direct-orphan") + + rep, err := s.gc(false) + if err != nil { + t.Fatal(err) + } + if !sliceContains(rep.Blobs, orphan) { + t.Fatalf("gc did not report orphan %s; blobs=%v", orphan, rep.Blobs) + } + if _, err := os.Stat(blobPath(s.root, orphan)); !os.IsNotExist(err) { + t.Fatalf("orphan blob still present after gc: %v", err) + } + if _, err := s.image("local:a"); err != nil { + t.Fatalf("live image unreadable after gc (live blobs reclaimed): %v", err) + } +} + func TestDefaultStoreFromEnvAndResolveStore(t *testing.T) { want := filepath.Join(t.TempDir(), "store") t.Setenv("ELFUSE_OCI_STORE", want) @@ -194,6 +270,43 @@ func TestLoadPinsCorruptNullAndPinError(t *testing.T) { } } +func TestInvalidPinnedDigestErrors(t *testing.T) { + s := openTestStore(t) + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte(`{"bad":"not-a-digest"}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.image("bad"); err == nil { + t.Fatal("image with invalid pinned digest succeeded, want error") + } + var buf bytes.Buffer + if err := list(&buf, s, false); err == nil || !strings.Contains(err.Error(), `digest "not-a-digest"`) { + t.Fatalf("list err = %v, want invalid digest error", err) + } + if _, err := s.liveCacheKeys(); err == nil { + t.Fatal("liveCacheKeys with invalid pinned digest succeeded, want error") + } +} + +func TestRemoveManifestDescriptorAndGCErrors(t *testing.T) { + s := openTestStore(t) + if err := s.removeManifestDescriptor("not-a-digest"); err == nil { + t.Fatal("removeManifestDescriptor invalid digest succeeded, want error") + } + + // Pin a ref so gc's descriptor reconciliation and reachability both have to + // read the (now corrupt) index.json rather than short-circuiting on an + // empty pin set. + if _, err := s.addImage("local:a", buildImage(t, []string{"/a"})); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(s.root, "index.json"), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.gc(false); err == nil || !strings.Contains(err.Error(), "index") { + t.Fatalf("gc corrupt index err = %v, want an index parse error", err) + } +} + func TestCacheKeyForDigestRejectsInvalidAndUnsupported(t *testing.T) { if _, err := cacheKeyForDigest("not-a-digest"); err == nil { t.Fatal("cacheKeyForDigest accepted malformed digest") @@ -206,3 +319,193 @@ func TestCacheKeyForDigestRejectsInvalidAndUnsupported(t *testing.T) { t.Fatalf("cacheKeyForDigest(%q) succeeded, want rejection", unsupported) } } + +func TestPruneRootfsCachesKeepsLiveDropsOrphanAndDryRun(t *testing.T) { + s := &store{root: t.TempDir()} + liveHex := strings.Repeat("a", 64) + orphanHex := strings.Repeat("b", 64) + liveDir := filepath.Join(s.root, "rootfs", "sha256", liveHex) + orphanDir := filepath.Join(s.root, "rootfs", "sha256", orphanHex) + for _, dir := range []string{liveDir, orphanDir} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "file"), []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(s.root, "rootfs", "sha256", "not-a-dir"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + live := map[string]bool{filepath.Join("sha256", liveHex): true} + rep, err := pruneRootfsCaches(s, live, pruneOpts{cache: true, dryRun: true}) + if err != nil { + t.Fatalf("dry-run pruneRootfsCaches: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != orphanDir { + t.Fatalf("dry-run cache dirs = %v, want [%s]", rep.CacheDirs, orphanDir) + } + if _, err := os.Stat(orphanDir); err != nil { + t.Fatalf("dry-run removed orphan cache: %v", err) + } + + rep, err = pruneRootfsCaches(s, live, pruneOpts{cache: true}) + if err != nil { + t.Fatalf("pruneRootfsCaches: %v", err) + } + if len(rep.CacheDirs) != 1 || rep.CacheDirs[0] != orphanDir { + t.Fatalf("cache dirs = %v, want [%s]", rep.CacheDirs, orphanDir) + } + if _, err := os.Stat(orphanDir); !os.IsNotExist(err) { + t.Fatalf("orphan cache after prune: %v, want IsNotExist", err) + } + if _, err := os.Stat(liveDir); err != nil { + t.Fatalf("live cache removed: %v", err) + } +} + +func TestPruneRootfsCachesMissingRootAndDiskUsageFallback(t *testing.T) { + s := &store{root: t.TempDir()} + rep, err := pruneRootfsCaches(s, nil, pruneOpts{cache: true}) + if err != nil { + t.Fatalf("missing rootfs prune: %v", err) + } + if len(rep.CacheDirs) != 0 || rep.Bytes != 0 { + t.Fatalf("missing rootfs report = %+v, want empty", rep) + } + + if got := diskUsage(fakeFileInfo{size: 123}); got != 123 { + t.Fatalf("diskUsage fallback = %d, want logical size 123", got) + } +} + +func TestPruneCachesErrorsOnInvalidLivePin(t *testing.T) { + s := openTestStore(t) + if err := os.WriteFile(filepath.Join(s.root, "refs.json"), []byte(`{"bad":"not-a-digest"}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.pruneCaches(pruneOpts{cache: true}); err == nil { + t.Fatal("pruneCaches with invalid live pin succeeded, want error") + } +} + +func TestDigestPrefix(t *testing.T) { + cases := []struct { + in string + want string + wantOK bool + }{ + {"abcdef123456", "abcdef123456", true}, + {"sha256:ABCDEF123456", "abcdef123456", true}, + {"abcdef12345", "", false}, + {strings.Repeat("a", 65), "", false}, + {"sha512:" + strings.Repeat("a", 64), "", false}, + {"not-hex-12345", "", false}, + } + for _, tc := range cases { + got, ok := digestPrefix(tc.in) + if ok != tc.wantOK || got != tc.want { + t.Fatalf("digestPrefix(%q) = (%q, %v), want (%q, %v)", tc.in, got, ok, tc.want, tc.wantOK) + } + } +} + +func TestResolvePinnedTarget(t *testing.T) { + digestA := "sha256:" + strings.Repeat("a", 64) + digestB := "sha256:" + strings.Repeat("b", 64) + digestAmbiguous := "sha256:" + strings.Repeat("a", 12) + strings.Repeat("c", 52) + base := refPins{ + "local:a": digestA, + "local:b": digestB, + } + ambiguous := refPins{ + "local:a": digestA, + "local:b": digestB, + "local:ambiguous": digestAmbiguous, + } + + cases := []struct { + name string + pins refPins + target string + wantRef string + wantDigest string + wantErr string // when set, expect an error containing this and ignore wantRef/wantDigest + }{ + {name: "exact ref", pins: base, target: "local:a", wantRef: "local:a", wantDigest: digestA}, + {name: "unique prefix", pins: base, target: strings.Repeat("b", 12), wantRef: "local:b", wantDigest: digestB}, + {name: "uppercase prefix", pins: base, target: "sha256:" + strings.Repeat("B", 12), wantRef: "local:b", wantDigest: digestB}, + {name: "missing digest", pins: base, target: strings.Repeat("d", 12), wantErr: "not pulled"}, + {name: "invalid target", pins: base, target: "not-a-ref", wantErr: "not pulled"}, + {name: "ambiguous prefix", pins: ambiguous, target: strings.Repeat("a", 12), wantErr: "ambiguous"}, + {name: "invalid pinned digest", pins: refPins{"bad": "not-a-digest"}, target: strings.Repeat("e", 12), wantErr: "pinned digest"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ref, digest, err := resolvePinnedTarget(tc.pins, tc.target) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want substring %q", err, tc.wantErr) + } + return + } + if err != nil || ref != tc.wantRef || digest != tc.wantDigest { + t.Fatalf("resolve = ref=%q digest=%q err=%v, want %s %s", ref, digest, err, tc.wantRef, tc.wantDigest) + } + }) + } + + // The ambiguous case must list the colliding refs in sorted order. + if _, _, err := resolvePinnedTarget(ambiguous, strings.Repeat("a", 12)); err == nil || + !strings.Contains(err.Error(), "local:a, local:ambiguous") { + t.Fatalf("ambiguous err = %v, want sorted ambiguous refs", err) + } +} + +func TestRmiByDigestPrefixReportsResolvedRef(t *testing.T) { + s := openTestStore(t) + img := buildImage(t, []string{"/hello"}) + digest, err := s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + prefix := strings.TrimPrefix(digest, "sha256:")[:12] + rep, err := s.rmi(prefix, false) + if err != nil { + t.Fatalf("rmi by prefix: %v", err) + } + if rep.Ref != "local:a" { + t.Fatalf("rmi report ref = %q, want local:a", rep.Ref) + } + if _, err := s.digestFor("local:a"); err == nil { + t.Fatal("local:a pin still present after rmi by digest prefix") + } + + s = openTestStore(t) + digest, err = s.addImage("local:a", img) + if err != nil { + t.Fatal(err) + } + prefix = strings.TrimPrefix(digest, "sha256:")[:12] + _, stderr, err := captureOutput(t, func() error { + return cmdRmi([]string{"--store", s.root, prefix}) + }) + if err != nil { + t.Fatalf("cmdRmi by prefix: %v", err) + } + if !strings.Contains(stderr, "Removed local:a:") || strings.Contains(stderr, "Removed "+prefix+":") { + t.Fatalf("cmdRmi stderr = %q, want resolved ref in summary", stderr) + } +} + +type fakeFileInfo struct { + size int64 +} + +func (f fakeFileInfo) Name() string { return "fake" } +func (f fakeFileInfo) Size() int64 { return f.size } +func (f fakeFileInfo) Mode() os.FileMode { return 0o644 } +func (f fakeFileInfo) ModTime() time.Time { return time.Time{} } +func (f fakeFileInfo) IsDir() bool { return false } +func (f fakeFileInfo) Sys() any { return nil } diff --git a/cmd/elfuse-oci/test_helpers_test.go b/cmd/elfuse-oci/test_helpers_test.go index e07c21d0..98ac62ba 100644 --- a/cmd/elfuse-oci/test_helpers_test.go +++ b/cmd/elfuse-oci/test_helpers_test.go @@ -39,7 +39,7 @@ func TestMain(m *testing.M) { UID: 1, GID: 2, } - if err := execElfuse(os.Getenv("ELFUSE_EXEC_ROOTFS"), spec); err != nil { + if err := execElfuse(os.Getenv("ELFUSE_EXEC_ROOTFS"), spec, nil); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(98) } diff --git a/cmd/elfuse-oci/unpack.go b/cmd/elfuse-oci/unpack.go index a8cccd63..0f037d6a 100644 --- a/cmd/elfuse-oci/unpack.go +++ b/cmd/elfuse-oci/unpack.go @@ -16,9 +16,19 @@ import ( "github.com/google/go-containerregistry/pkg/v1" ) -// unpackImage extracts every layer of the stored image (base first) into a -// plain directory rootfs. Each layer is a tar stream (crane decompresses gzip -// and zstd transparently via layer.Uncompressed). +// rootfsStagingSuffix marks unpackImage's pre-rename staging siblings, +// . rootfsCacheLockPath parses it to map a staging +// dir back to its digest's lock, so the two must never drift apart. +const rootfsStagingSuffix = ".tmp-" + +// unpackImage extracts every layer of img (base first) into a plain directory +// rootfs. Each layer is a tar stream (crane decompresses gzip and zstd +// transparently via layer.Uncompressed). +// +// img is the caller's already-resolved image, not a ref re-resolved here: the +// caller keys caches by the digest it resolved, and a re-resolution could +// observe a different pin (a concurrent repull) and fill a digest-keyed cache +// with another image's content. // // Layer application implements the OCI whiteout conventions: // - `.wh.` in a directory removes `` (from this and lower layers). @@ -33,11 +43,7 @@ import ( // // Ownership is not applied here: elfuse runs as the host user and overrides // identity at runtime via --user, so the rootfs carries only mode bits. -func unpackImage(s *store, ref, dest string) error { - img, err := s.image(ref) - if err != nil { - return err - } +func unpackImage(img v1.Image, dest string) error { if _, statErr := os.Lstat(dest); statErr == nil { // An explicit pre-existing --rootfs directory: merge in place and // never remove it, failed or not; it is not ours to delete. @@ -54,7 +60,7 @@ func unpackImage(s *store, ref, dest string) error { if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { return err } - tmp, err := os.MkdirTemp(filepath.Dir(dest), filepath.Base(dest)+".tmp-") + tmp, err := os.MkdirTemp(filepath.Dir(dest), filepath.Base(dest)+rootfsStagingSuffix) if err != nil { return err } @@ -99,6 +105,35 @@ func unpackInto(img v1.Image, dest string) error { return nil } +// layerPaths tracks what the current layer has created, so a late opaque +// whiteout does not wipe the layer's own additions. entries holds the exact tar +// entry paths; subtree additionally holds every ancestor directory of those +// entries, so an opaque clear can preserve an implicit parent directory the +// layer never gave its own tar entry (e.g. a layer with dir/sub/file but no +// dir/sub entry). +type layerPaths struct { + entries map[string]bool + subtree map[string]bool +} + +func newLayerPaths() layerPaths { + return layerPaths{entries: map[string]bool{}, subtree: map[string]bool{}} +} + +// add records name as a current-layer entry and marks name and all its ancestor +// directories as carrying current-layer content. +func (lp layerPaths) add(name string) { + lp.entries[name] = true + for p := name; p != "." && p != string(filepath.Separator); { + lp.subtree[p] = true + parent := filepath.Dir(p) + if parent == p { + break + } + p = parent + } +} + func applyLayer(root *os.Root, layer v1.Layer) error { r, err := layer.Uncompressed() if err != nil { @@ -109,7 +144,7 @@ func applyLayer(root *os.Root, layer v1.Layer) error { // content only, but tar entry order within a layer is not guaranteed: an // opaque marker may arrive after the directory's own same-layer children, // which must survive the clear (Docker's unpacker keeps the same set). - applied := make(map[string]bool) + lp := newLayerPaths() tr := tar.NewReader(r) for { hdr, err := tr.Next() @@ -119,7 +154,7 @@ func applyLayer(root *os.Root, layer v1.Layer) error { if err != nil { return fmt.Errorf("read tar entry: %w", err) } - if err := applyEntry(root, hdr, tr, applied); err != nil { + if err := applyEntry(root, hdr, tr, lp); err != nil { return fmt.Errorf("entry %q: %w", hdr.Name, err) } } @@ -132,10 +167,11 @@ const whiteoutPrefix = ".wh." // existing children are hidden before the layer's own additions apply. const opaqueMarker = whiteoutPrefix + ".wh..opq" -// applyEntry applies one tar header to the rootfs. applied tracks the paths -// the current layer has created so far, so an opaque whiteout arriving after -// its directory's same-layer children does not delete them. -func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string]bool) error { +// applyEntry applies one tar header to the rootfs. lp tracks the paths the +// current layer has created so far, so an opaque whiteout arriving after its +// directory's same-layer children (or their implicit parents) does not delete +// them. +func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, lp layerPaths) error { name := filepath.Clean(hdr.Name) if strings.HasPrefix(name, "../") || name == ".." { return fmt.Errorf("unsafe entry path %q", hdr.Name) @@ -147,7 +183,7 @@ func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string] base := filepath.Base(name) if base == opaqueMarker { - return clearDirectory(root, filepath.Dir(name), applied) + return clearDirectory(root, filepath.Dir(name), lp) } if trimmed, ok := strings.CutPrefix(base, whiteoutPrefix); ok { // A bare ".wh." would leave an empty target and Join(dir, "") is the @@ -159,7 +195,7 @@ func applyEntry(root *os.Root, hdr *tar.Header, r io.Reader, applied map[string] target := filepath.Join(filepath.Dir(name), trimmed) return root.RemoveAll(target) } - applied[name] = true + lp.add(name) // hdr.FileInfo().Mode() maps the tar header's unix mode bits to os.FileMode // with the special bits (ModeSetuid/Setgid/Sticky) at os.FileMode's high @@ -383,7 +419,7 @@ func shouldDropSpecial(err error, special os.FileMode) bool { // keeping entries the current layer itself created: opaque markers hide // lower-layer content, and a marker ordered after its directory's same-layer // additions must not wipe them. -func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { +func clearDirectory(root *os.Root, dir string, lp layerPaths) error { fi, err := root.Lstat(dir) if err != nil { if os.IsNotExist(err) { @@ -398,7 +434,7 @@ func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { // under this name anyway, mirroring the non-dir replacement mkdirAll and // the regular-file path perform. if !fi.IsDir() { - if applied[dir] { + if lp.entries[dir] { // This layer created the non-dir itself; there is no lower // content beneath it for the marker to hide. return nil @@ -422,7 +458,11 @@ func clearDirectory(root *os.Root, dir string, applied map[string]bool) error { } for _, e := range entries { child := filepath.Join(dir, e.Name()) - if applied[child] { + // Keep a child the current layer created OR that has current-layer + // content beneath it: an implicit parent directory (dir/sub with no + // own tar entry, created for dir/sub/file) has no entries[child] but is + // in subtree, and deleting it would take the layer's own file with it. + if lp.subtree[child] { continue } if err := root.RemoveAll(child); err != nil { diff --git a/cmd/elfuse-oci/unpack_image_test.go b/cmd/elfuse-oci/unpack_image_test.go index a6cfdde7..c424f4e4 100644 --- a/cmd/elfuse-oci/unpack_image_test.go +++ b/cmd/elfuse-oci/unpack_image_test.go @@ -25,6 +25,20 @@ type tarEntry struct { body string } +// unpackRef resolves ref to its pinned image and unpacks it into dest, the +// two-step the production callers do (resolve under the store lock, then +// unpack the resolved image). It fails the test if the ref cannot be +// resolved; the returned error is unpackImage's, so callers testing unpack +// failures still see them. +func unpackRef(t *testing.T, s *store, ref, dest string) error { + t.Helper() + img, err := s.image(ref) + if err != nil { + t.Fatalf("resolve %s: %v", ref, err) + } + return unpackImage(img, dest) +} + func testTarLayer(t *testing.T, entries ...tarEntry) v1.Layer { t.Helper() var buf bytes.Buffer @@ -105,7 +119,7 @@ func TestUnpackImageAppliesLayersInOrder(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:layered", dest); err != nil { + if err := unpackRef(t, s, "local:layered", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -151,7 +165,7 @@ func TestUnpackImageCleansUpPartialRootfs(t *testing.T) { parent := t.TempDir() dest := filepath.Join(parent, "rootfs") - if err := unpackImage(s, "local:partial", dest); err == nil { + if err := unpackRef(t, s, "local:partial", dest); err == nil { t.Fatal("unpackImage succeeded, want failure on fifo entry") } if _, err := os.Lstat(dest); !os.IsNotExist(err) { @@ -170,7 +184,7 @@ func TestUnpackImageCleansUpPartialRootfs(t *testing.T) { if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { t.Fatal(err) } - if err := unpackImage(s, "local:partial", pre); err == nil { + if err := unpackRef(t, s, "local:partial", pre); err == nil { t.Fatal("unpackImage succeeded, want failure on fifo entry") } if _, err := os.Stat(sentinel); err != nil { @@ -194,7 +208,7 @@ func TestUnpackImagePreexistingDestUnpacksInPlace(t *testing.T) { if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { t.Fatal(err) } - if err := unpackImage(s, "local:merge", dest); err != nil { + if err := unpackRef(t, s, "local:merge", dest); err != nil { t.Fatalf("unpackImage: %v", err) } if b, err := os.ReadFile(sentinel); err != nil || string(b) != "keep" { @@ -221,7 +235,7 @@ func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { root, dir := newRoot(t) for _, name := range []string{".", "/"} { h := regHeader(name, 0o644, 0) - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("applyEntry root no-op %q: %v", name, err) } } @@ -234,7 +248,7 @@ func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { } hardlink := linkHeader("escape-link", "../outside") - if err := applyEntry(root, &hardlink, strings.NewReader(""), map[string]bool{}); err == nil { + if err := applyEntry(root, &hardlink, strings.NewReader(""), newLayerPaths()); err == nil { t.Fatal("applyEntry accepted hardlink target escaping root") } @@ -246,7 +260,7 @@ func TestApplyEntryRootNoOpsHardlinkEscapeAndSymlinkReplacement(t *testing.T) { t.Fatal(err) } file := regHeader("replace-me", 0o644, int64(len("inside"))) - if err := applyEntry(root, &file, strings.NewReader("inside"), map[string]bool{}); err != nil { + if err := applyEntry(root, &file, strings.NewReader("inside"), newLayerPaths()); err != nil { t.Fatalf("applyEntry replacing symlink: %v", err) } if b, err := os.ReadFile(outside); err != nil || string(b) != "outside" { @@ -264,7 +278,7 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("unsupported tar type", func(t *testing.T) { root, _ := newRoot(t) h := tar.Header{Name: "weird", Typeflag: 'x'} - err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}) + err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()) if err == nil || !strings.Contains(err.Error(), "unsupported tar type") || !strings.Contains(err.Error(), tarTypeName('x')) { t.Fatalf("unsupported type err = %v, want tar type error", err) } @@ -273,7 +287,7 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("absolute symlink to root", func(t *testing.T) { root, dir := newRoot(t) h := symHeader("usr/root-link", "/") - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("applyEntry symlink to root: %v", err) } target, err := os.Readlink(filepath.Join(dir, "usr", "root-link")) @@ -288,14 +302,14 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("parent path is regular file", func(t *testing.T) { root, dir := newRoot(t) parent := regHeader("parent", 0o644, 0) - if err := applyEntry(root, &parent, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &parent, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatal(err) } // A non-directory on the parent path is replaced with a real // directory (containerd/Docker behavior), not an error: layers may // legitimately turn a lower layer's file into a directory. child := regHeader("parent/child", 0o644, 0) - if err := applyEntry(root, &child, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &child, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("applyEntry child under regular-file parent: %v, want file replaced by directory", err) } fi, err := os.Lstat(filepath.Join(dir, "parent")) @@ -310,7 +324,7 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("opaque missing directory is no-op", func(t *testing.T) { root, _ := newRoot(t) h := tar.Header{Name: "missing/.wh..wh..opq", Typeflag: tar.TypeReg} - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("opaque missing dir: %v", err) } }) @@ -318,14 +332,14 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("opaque marker under lower-layer regular file replaces it", func(t *testing.T) { root, dir := newRoot(t) file := regHeader("notdir", 0o644, 0) - if err := applyEntry(root, &file, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &file, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatal(err) } // The opaque marker hides all lower content under the name, so a // lower layer's non-directory there is replaced with an empty real // directory rather than cleared through or rejected. h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} - if err := applyEntry(root, &h, strings.NewReader(""), map[string]bool{}); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), newLayerPaths()); err != nil { t.Fatalf("opaque marker under regular file: %v, want file replaced by empty directory", err) } fi, err := os.Lstat(filepath.Join(dir, "notdir")) @@ -337,12 +351,12 @@ func TestApplyEntryAdditionalErrorBranches(t *testing.T) { t.Run("opaque marker under same-layer regular file is a no-op", func(t *testing.T) { root, dir := newRoot(t) file := regHeader("notdir", 0o644, 0) - applied := map[string]bool{} - if err := applyEntry(root, &file, strings.NewReader(""), applied); err != nil { + lp := newLayerPaths() + if err := applyEntry(root, &file, strings.NewReader(""), lp); err != nil { t.Fatal(err) } h := tar.Header{Name: "notdir/.wh..wh..opq", Typeflag: tar.TypeReg} - if err := applyEntry(root, &h, strings.NewReader(""), applied); err != nil { + if err := applyEntry(root, &h, strings.NewReader(""), lp); err != nil { t.Fatalf("opaque marker under same-layer file: %v, want no-op", err) } fi, err := os.Lstat(filepath.Join(dir, "notdir")) @@ -403,7 +417,7 @@ func TestUnpackOpaqueAfterSameLayerChildren(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:opq-late", dest); err != nil { + if err := unpackRef(t, s, "local:opq-late", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -415,6 +429,40 @@ func TestUnpackOpaqueAfterSameLayerChildren(t *testing.T) { } } +// TestUnpackOpaqueAfterImplicitParentChildren pins #1: a late opaque marker +// must preserve the current layer's descendants even when their immediate +// parent directory has no tar entry of its own (an implicit parent, created +// only because a deeper file needed it). Here the upper layer writes +// dir/sub/file with NO dir/sub entry, then an opaque marker on dir: dir/sub and +// its file must survive while the lower layer's dir/old is cleared. +func TestUnpackOpaqueAfterImplicitParentChildren(t *testing.T) { + lower := testTarLayer(t, + tarEntry{header: dirHeader("dir", 0o755)}, + tarEntry{header: regHeader("dir/old", 0o644, 0), body: "hidden"}, + ) + upper := testTarLayer(t, + // No dir/sub entry: the parent is implicit, created for dir/sub/file. + tarEntry{header: regHeader("dir/sub/file", 0o644, 0), body: "kept"}, + tarEntry{header: tar.Header{Name: "dir/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}}, + ) + + s := openTestStore(t) + if _, err := s.addImage("local:opq-implicit", testImageWithLayers(t, lower, upper)); err != nil { + t.Fatal(err) + } + dest := t.TempDir() + if err := unpackRef(t, s, "local:opq-implicit", dest); err != nil { + t.Fatalf("unpackImage: %v", err) + } + + if _, err := os.Stat(filepath.Join(dest, "dir", "old")); !os.IsNotExist(err) { + t.Fatalf("opaque-hidden dir/old = %v, want IsNotExist", err) + } + if b, err := os.ReadFile(filepath.Join(dest, "dir", "sub", "file")); err != nil || string(b) != "kept" { + t.Fatalf("dir/sub/file = %q, err=%v; want the implicit-parent child to survive the late marker", b, err) + } +} + // TestUnpackRejectsBareWhiteout pins that a malformed ".wh." entry with no // target suffix fails extraction instead of resolving to Join(dir, "") and // deleting the containing directory. @@ -433,7 +481,7 @@ func TestUnpackRejectsBareWhiteout(t *testing.T) { defer root.Close() hdr := tar.Header{Name: "opt/.wh.", Typeflag: tar.TypeReg, Mode: 0o644} - if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + if err := applyEntry(root, &hdr, strings.NewReader(""), newLayerPaths()); err == nil { t.Fatal("bare .wh. entry applied, want invalid-whiteout error") } if _, err := os.Stat(filepath.Join(dest, "opt", "keep")); err != nil { @@ -460,7 +508,7 @@ func TestUnpackDirReplacesLowerSymlink(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:dir-over-link", dest); err != nil { + if err := unpackRef(t, s, "local:dir-over-link", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -498,7 +546,7 @@ func TestUnpackParentSymlinkReplaced(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:parent-link", dest); err != nil { + if err := unpackRef(t, s, "local:parent-link", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -534,7 +582,7 @@ func TestUnpackDirEntryParentSymlinkReplaced(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:parent-link-dir", dest); err != nil { + if err := unpackRef(t, s, "local:parent-link-dir", dest); err != nil { t.Fatalf("unpackImage: %v", err) } @@ -573,7 +621,7 @@ func TestUnpackOpaqueThroughSymlinkKeepsTarget(t *testing.T) { t.Fatal(err) } dest := t.TempDir() - if err := unpackImage(s, "local:opaque-link", dest); err != nil { + if err := unpackRef(t, s, "local:opaque-link", dest); err != nil { t.Fatalf("unpackImage: %v", err) } diff --git a/cmd/elfuse-oci/unpack_test.go b/cmd/elfuse-oci/unpack_test.go index 5830eaee..c626e744 100644 --- a/cmd/elfuse-oci/unpack_test.go +++ b/cmd/elfuse-oci/unpack_test.go @@ -33,7 +33,7 @@ func applyEntries(t *testing.T, root *os.Root, entries []tar.Header) { content = []byte(strings.Repeat("x", int(h.Size))) } hdr := h - if err := applyEntry(root, &hdr, bytes.NewReader(content), map[string]bool{}); err != nil { + if err := applyEntry(root, &hdr, bytes.NewReader(content), newLayerPaths()); err != nil { t.Fatalf("applyEntry %q: %v", h.Name, err) } } @@ -43,7 +43,7 @@ func applyEntryWithContent(t *testing.T, root *os.Root, h tar.Header, content st t.Helper() hdr := h hdr.Size = int64(len(content)) - if err := applyEntry(root, &hdr, strings.NewReader(content), map[string]bool{}); err != nil { + if err := applyEntry(root, &hdr, strings.NewReader(content), newLayerPaths()); err != nil { t.Fatalf("applyEntry %q: %v", h.Name, err) } } @@ -345,7 +345,7 @@ func TestUnpackSpecialFilesRejected(t *testing.T) { t.Run(tc.want, func(t *testing.T) { root, dir := newRoot(t) hdr := tar.Header{Name: tc.name, Mode: 0o644, Typeflag: tc.typeflag} - err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}) + err := applyEntry(root, &hdr, strings.NewReader(""), newLayerPaths()) if err == nil || !strings.Contains(err.Error(), "unsupported special file type "+tc.want) { t.Fatalf("applyEntry special %s err = %v, want unsupported error", tc.want, err) } @@ -359,7 +359,7 @@ func TestUnpackSpecialFilesRejected(t *testing.T) { func TestUnpackPathEscapeRejected(t *testing.T) { root, _ := newRoot(t) hdr := regHeader("../escape", 0o644, 0) - if err := applyEntry(root, &hdr, strings.NewReader(""), map[string]bool{}); err == nil { + if err := applyEntry(root, &hdr, strings.NewReader(""), newLayerPaths()); err == nil { t.Fatalf("applyEntry accepted ../escape path") } } @@ -385,7 +385,7 @@ func TestUnpackReadsExactSize(t *testing.T) { content := "abc123" r := &countingReader{b: []byte(content + "TRAILING-JUNK")} hdr := regHeader("f", 0o644, int64(len(content))) - if err := applyEntry(root, &hdr, r, map[string]bool{}); err != nil { + if err := applyEntry(root, &hdr, r, newLayerPaths()); err != nil { t.Fatalf("applyEntry: %v", err) } if r.n != len(content) { @@ -397,7 +397,7 @@ func TestUnpackReadsExactSize(t *testing.T) { short := &countingReader{b: []byte("abc")} hdr = regHeader("g", 0o644, int64(len(content))) - if err := applyEntry(root, &hdr, short, map[string]bool{}); err == nil { + if err := applyEntry(root, &hdr, short, newLayerPaths()); err == nil { t.Fatal("applyEntry accepted a short body, want size-mismatch error") } } From 0c900078e94665ffbe4353680b5b76de1c6e9c03 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:38:02 +0200 Subject: [PATCH 08/10] Add OCI conformance checks and lifecycle CI The on-disk store is the contract: pulls must produce a valid OCI image-layout that other tools can read and that agrees with registry truth on the manifest digest. Conformance tests pin the layout shape, and scripts/oci-interop.sh cross-checks the store with crane, skopeo, and umoci. CI splits by runner capability: a Linux job runs the pure-Go store paths (pull, unpack, inspect, lifecycle) without Hypervisor.framework, a hosted macOS job builds and tests the darwin sparsebundle code and drives a run-less pull/inspect/list/rmi/prune lifecycle smoke, and the self-hosted release leg boots guests end to end under HVF: the alpine:3 default-entrypoint smoke plus a full pull -> inspect -> list -> run -> rmi -> prune lifecycle that runs python:3.12-slim with an --entrypoint override and asserts the teardown half of the lifecycle. The lifecycle teardown covers all three reclamation guardrails: a plain rmi reclaims the cold cache with the image, run --keep output refuses rmi without --force, and a live --plain-rootfs guest parked in sleep pins its cache; prune --cache --all must skip it and rmi must refuse until the guest exits, the only end-to-end exercise of the run-lock descriptor riding through the exec into elfuse. --- .github/workflows/main.yml | 190 ++++++++++- Makefile | 34 ++ cmd/elfuse-oci/store_conformance_test.go | 382 +++++++++++++++++++++++ scripts/ci/oci-cli-smoke.sh | 108 +++++++ scripts/ci/oci-lib.sh | 142 +++++++++ scripts/ci/oci-lifecycle.sh | 154 +++++++++ scripts/ci/oci-python-workload.py | 117 +++++++ scripts/ci/oci-run-smoke.sh | 54 ++++ scripts/oci-interop.sh | 212 +++++++++++++ 9 files changed, 1389 insertions(+), 4 deletions(-) create mode 100644 cmd/elfuse-oci/store_conformance_test.go create mode 100755 scripts/ci/oci-cli-smoke.sh create mode 100755 scripts/ci/oci-lib.sh create mode 100755 scripts/ci/oci-lifecycle.sh create mode 100644 scripts/ci/oci-python-workload.py create mode 100755 scripts/ci/oci-run-smoke.sh create mode 100755 scripts/oci-interop.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6276c986..3d12ff9a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,12 @@ # scan-macos : LLVM scan-build via `make analyze` # infer-macos : Facebook Infer capture + analyze over the full build # runtime-macos : HVF runtime tests on self-hosted Apple Silicon, -# including release, ASAN, UBSAN, and TSAN variants +# including release, ASAN, UBSAN, and TSAN variants, +# plus the end-to-end OCI run and image-lifecycle checks +# oci-conformance : OCI image-layout conformance + cross-tool interop +# (crane/skopeo/umoci) on Linux +# oci-macos : elfuse-oci darwin build, unit tests, sparsebundle +# round-trip, and a run-less image-lifecycle smoke # # Runtime and sanitizer tests require Hypervisor.framework, which # GitHub-hosted macOS runners do not expose. Those tests run on self-hosted @@ -94,12 +99,13 @@ jobs: run: .ci/check-security.sh - name: shellcheck - # Scoped to .ci/ -- tests/ has pre-existing warnings that the - # repository's own check-format target already surfaces. + # Scoped to .ci/ and scripts/; tests/ has pre-existing warnings that + # the repository's own check-format target already surfaces. if: ${{ !cancelled() }} run: | set -euo pipefail - mapfile -d '' files < <(git ls-files -z -- '.ci/*.sh') + # git pathspec globs cross '/', so 'scripts/*.sh' covers scripts/ci/ too. + mapfile -d '' files < <(git ls-files -z -- '.ci/*.sh' 'scripts/*.sh') shellcheck --severity=warning "${files[@]}" - name: cppcheck @@ -551,6 +557,15 @@ jobs: ls -l "$ROSETTA" + - name: Set up Go + # Only the release leg runs the OCI run smoke below, which needs the Go + # toolchain to build build/elfuse-oci. + if: ${{ matrix.run_matrix }} + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + - name: Build elfuse # make does not track EXTRA_CFLAGS changes, so an object built for one # sanitizer must not be reused for another. Checkout already wipes @@ -580,6 +595,44 @@ jobs: run: | make EXTRA_CFLAGS="$EXTRA_CFLAGS" ${{ matrix.check_target }} + - name: OCI run smoke (pull -> sparsebundle -> COW clone -> HVF boot) + # The only leg that exercises the full default `run` path end to end: + # pull an image, provision the case-sensitive sparsebundle, COW-clone it, + # boot the guest under HVF, and propagate its exit status. Release leg + # only (sanitizer legs skip the fixture/qemu-heavy paths). + if: ${{ matrix.run_matrix }} + run: | + set -euo pipefail + # Same persistent-disk convention as the fixture cache above: + # checkout wipes the workspace but this self-hosted runner's disk + # survives. pull is idempotent per digest, so a warm store skips + # the image blob downloads (only the manifest HEAD/GET goes + # out) and the warm sparsebundle cache skips the unpack. env: + # values don't expand $HOME, so export here instead. + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-store" + make build/elfuse-oci + scripts/ci/oci-run-smoke.sh + + - name: OCI image lifecycle (pull -> inspect -> list -> run -> rmi -> prune) + # Walks the whole user-facing image lifecycle on the only leg with + # HVF. python:3.12-slim adds what the alpine smoke above does not: + # an --entrypoint override, a glibc dynamically-linked guest, and + # the teardown half of the lifecycle (see the phase functions in + # scripts/ci/oci-lifecycle.sh). Separate store from the smoke step + # so the empty-store assertions are meaningful. Release leg only. + if: ${{ matrix.run_matrix }} + env: + ELFUSE_OCI_STORE: ${{ runner.temp }}/oci-lifecycle-store + IMG: python:3.12-slim + run: | + set -euo pipefail + # Built by the smoke step above; the make target is idempotent. + make build/elfuse-oci + # The seed store lives on the runner's persistent disk (env: does + # not expand $HOME, so export here instead). + export ELFUSE_OCI_SEED_STORE="$HOME/.cache/elfuse-ci/oci-seed-store" + scripts/ci/oci-lifecycle.sh + - name: Test matrix if: ${{ matrix.run_matrix }} run: | @@ -611,3 +664,132 @@ jobs: else echo "No externals/test-fixtures to save" fi + + # OCI image-layout conformance + cross-tool interop on Linux. elfuse-oci + # is pure Go (no Hypervisor.framework), so pull/inspect/unpack and the + # conformance tests run in hosted CI; only `run` needs HVF and is excluded. + # The on-disk store is the contract: it must be a valid OCI image-layout that + # crane/skopeo/umoci can read and that agrees with registry truth. + oci-conformance: + name: OCI conformance + interop (Linux) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + # Pinned to elfuse-oci's go-containerregistry version so the crane + # CLI reads layouts with the same schema handling it writes with. + GGCR_VERSION: v0.21.7 + # umoci release tag for the interop gate; built from a checkout below. + UMOCI_VERSION: v0.6.0 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Install jq + skopeo + # skopeo reads our layout via the oci: transport. CI treats it as part + # of the conformance gate; local runs may omit it and get a skipped + # interop section from scripts/oci-interop.sh. + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y jq skopeo + + - name: Install crane + umoci from source + # crane (registry-truth comparison) and umoci (layout parse) are Go + # tools; install crane at elfuse-oci's ggcr version where applicable. + run: | + set -euo pipefail + go install github.com/google/go-containerregistry/cmd/crane@${GGCR_VERSION} + # `go install pkg@version` refuses umoci: its go.mod carries replace + # directives. Build from a pinned checkout instead, where replace + # directives apply; a read-only `umoci list --layout` conformance + # check needs nothing newer. + git clone --quiet --depth 1 --branch "$UMOCI_VERSION" \ + https://github.com/opencontainers/umoci.git "$RUNNER_TEMP/umoci" + (cd "$RUNNER_TEMP/umoci" && \ + go build -o "$(go env GOPATH)/bin/umoci" ./cmd/umoci) + echo "$(go env GOPATH)/bin" >>"$GITHUB_PATH" + + - name: Build elfuse-oci + # Pure Go target; does not require the C toolchain or HVF. + run: make build/elfuse-oci + + - name: Go fmt + vet (Linux and darwin cross-check) + # The Makefile gate, so CI and local runs cannot drift. oci-lint vets + # native, darwin/arm64, and linux; the darwin pass compile-checks the + # sparsebundle files (csrun.go, sparsebundle.go, cache_darwin.go) + # that never build on this Linux runner. + run: make oci-lint + + - name: CLI lifecycle smoke (pull/list/inspect/unpack/rmi/prune) + # Exercises the built binary through the same user-facing flow that the + # Go unit tests model in-process. `run` itself remains covered by Go + # orchestration tests here and by macOS/HVF runtime jobs. --unpack adds + # the unpack + cache-reclaiming rmi phase to the shared smoke. + run: scripts/ci/oci-cli-smoke.sh --unpack + + - name: Go unit + conformance tests (with network pull round-trip) + # ELFUSE_OCI_NETTEST enables the pull round-trip that re-opens the store + # with crane's independent layout reader and asserts digest agreement. + env: + ELFUSE_OCI_NETTEST: "1" + run: go test -race ./cmd/elfuse-oci/ + + - name: Cross-tool interop (crane + skopeo + umoci) + # Pulls fixtures, then asserts the on-disk layout is spec-shaped and + # that available tools read it and agree with registry truth. + run: scripts/oci-interop.sh + + # Darwin elfuse-oci build + tests on a hosted macOS runner. The default `run` + # path (csrun.go, sparsebundle.go, cache_darwin.go) only compiles on darwin, so + # the Linux job above can only cross-vet it; this job actually builds and runs + # it, and drives the run-less image lifecycle (pull/inspect/list/rmi/prune) + # through the darwin binary. Hosted runners provide hdiutil + case-sensitive + # APFS (so the real sparsebundle round-trip runs) even though they lack + # Hypervisor.framework; the HVF-backed guest boot is covered by the + # self-hosted runtime-macos job. + oci-macos: + name: OCI image CLI (macOS Apple Silicon) + runs-on: macos-15 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Build elfuse-oci + run: make build/elfuse-oci + + - name: Go unit tests (darwin native) + # Runs the whole suite on darwin, exercising the sparsebundle/clone seams + # in csrun/sparsebundle/cache_darwin that the Linux job cannot compile. + # -race: the darwin concurrency-sensitive code (sweep/provision/clone, + # cache-removal lock discipline) compiles only here, so this is the only + # place the race detector ever sees it. + run: go test -race ./cmd/elfuse-oci/ + + - name: Sparsebundle round-trip (hdiutil + case-sensitive APFS) + # ELFUSE_OCI_DARWIN_CS un-skips the real hdiutil create/attach/detach + + # case-sensitive APFS sweep; hosted runners have hdiutil and APFS. + env: + ELFUSE_OCI_DARWIN_CS: "1" + run: go test -run TestDarwinCSSweep ./cmd/elfuse-oci/ + + - name: CLI lifecycle smoke (pull/inspect/list/rmi/prune, no HVF) + # The same shared smoke the Linux job drives, but through the darwin + # binary: everything short of `run` (which needs HVF) works end to + # end on a hosted runner. Without --unpack every rmi takes the + # cache-free path (nothing was ever unpacked, no --force involved) + # that the Linux job's cache-reclaiming flow does not cover. jq + # ships on the macos-15 image. + run: scripts/ci/oci-cli-smoke.sh diff --git a/Makefile b/Makefile index 7323716a..733100c6 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,40 @@ OCI_SRCS := $(shell find cmd/elfuse-oci -type f -name '*.go' 2>/dev/null) .PHONY: elfuse-oci elfuse-oci: $(OCI_BIN) +# OCI image-layout conformance + cross-tool interop. Pulls fixtures into a +# throwaway store and asserts the on-disk layout is spec-shaped and readable by +# crane/skopeo/umoci (whichever are installed locally; all are required in CI). +# Pure Go + jq; no HVF, runs on Linux. Requires network to pull fixtures. +.PHONY: oci-interop +oci-interop: $(OCI_BIN) + $(Q)scripts/oci-interop.sh + +# Go unit tests for the OCI image CLI (offline). Set ELFUSE_OCI_NETTEST=1 +# to also exercise the network pull round-trip conformance test. +.PHONY: oci-test +oci-test: + $(Q)$(GO) test ./cmd/elfuse-oci/ + +# gofmt + go vet gate for the OCI image CLI. `go vet` is run for both GOOS +# values so the darwin-only sparsebundle files are checked from Linux CI and the +# non-darwin stubs are checked from a macOS host. The darwin pass pins +# GOARCH=arm64 (the only supported darwin target, and what CI checks) so an +# amd64 Linux host does not silently vet darwin/amd64 instead. oci-lint +# bundles both so a local run matches the CI gate. +.PHONY: oci-vet oci-fmt-check oci-lint +oci-vet: + $(Q)$(GO) vet ./cmd/elfuse-oci/ + $(Q)GOOS=darwin GOARCH=arm64 $(GO) vet ./cmd/elfuse-oci/ + $(Q)GOOS=linux $(GO) vet ./cmd/elfuse-oci/ + +oci-fmt-check: + $(Q)out="$$(gofmt -l cmd/elfuse-oci)"; \ + if [ -n "$$out" ]; then \ + echo "gofmt needs to run on:"; echo "$$out"; exit 1; \ + fi + +oci-lint: oci-fmt-check oci-vet + # rm -f first: `go build -o` follows an existing symlink at the output path, # so a stale build/elfuse-oci symlink would clobber build/elfuse. $(OCI_BIN): go.mod $(OCI_SRCS) | $(BUILD_DIR) diff --git a/cmd/elfuse-oci/store_conformance_test.go b/cmd/elfuse-oci/store_conformance_test.go new file mode 100644 index 00000000..a0be129b --- /dev/null +++ b/cmd/elfuse-oci/store_conformance_test.go @@ -0,0 +1,382 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" +) + +// TestStoreLayoutFiles asserts openStore creates the OCI image-layout +// scaffolding exactly: an oci-layout file with imageLayoutVersion 1.0.0, an +// index.json with an empty manifests array, and a blobs/sha256/ tree. This is +// the part of the OCI image-layout spec every conforming reader expects. +func TestStoreLayoutFiles(t *testing.T) { + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + _ = s + + b, err := os.ReadFile(filepath.Join(root, "oci-layout")) + if err != nil { + t.Fatal(err) + } + var lay struct { + Version string `json:"imageLayoutVersion"` + } + if err := json.Unmarshal(b, &lay); err != nil { + t.Fatalf("oci-layout is not JSON: %v (raw %q)", err, b) + } + if lay.Version != "1.0.0" { + t.Errorf("imageLayoutVersion: got %q, want 1.0.0", lay.Version) + } + + b, err = os.ReadFile(filepath.Join(root, "index.json")) + if err != nil { + t.Fatal(err) + } + var idx struct { + Schema int `json:"schemaVersion"` + Manifests []json.RawMessage `json:"manifests"` + } + if err := json.Unmarshal(b, &idx); err != nil { + t.Fatalf("index.json is not JSON: %v", err) + } + if idx.Schema != 2 { + t.Errorf("index schemaVersion: got %d, want 2", idx.Schema) + } + if len(idx.Manifests) != 0 { + t.Errorf("fresh index has %d manifests, want 0", len(idx.Manifests)) + } + + if fi, err := os.Stat(filepath.Join(root, "blobs", "sha256")); err != nil || !fi.IsDir() { + t.Errorf("blobs/sha256/ missing or not a directory: %v", err) + } +} + +// TestStoreLayoutRoundTrip stores a tiny image, then re-opens the layout with +// crane's own layout.FromPath reader (independent of our store.go write path) +// and asserts the manifest, config, and layer digests all round-trip. This +// is the offline OCI image-layout conformance signal: the on-disk bytes are +// parseable by the canonical go-containerregistry reader. +func TestStoreLayoutRoundTrip(t *testing.T) { + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + img := tinyImage(t) + wantManifest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + wantConfig, err := img.ConfigName() + if err != nil { + t.Fatal(err) + } + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + wantLayerDigests := make([]v1.Hash, len(layers)) + for i, l := range layers { + d, err := l.Digest() + if err != nil { + t.Fatal(err) + } + wantLayerDigests[i] = d + } + + digest, err := s.addImage("local:tiny", img) + if err != nil { + t.Fatal(err) + } + if digest != wantManifest.String() { + t.Errorf("addImage digest: got %s, want %s", digest, wantManifest) + } + + // Re-open the layout from disk with crane's reader, not our handle. + p, err := layout.FromPath(root) + if err != nil { + t.Fatalf("layout.FromPath: %v (store is not a readable OCI layout)", err) + } + got, err := p.Image(wantManifest) + if err != nil { + t.Fatalf("re-opened layout cannot find manifest %s: %v", wantManifest, err) + } + gotManifest, err := got.Digest() + if err != nil { + t.Fatal(err) + } + if gotManifest != wantManifest { + t.Errorf("manifest digest: got %s, want %s", gotManifest, wantManifest) + } + gotConfig, err := got.ConfigName() + if err != nil { + t.Fatal(err) + } + if gotConfig != wantConfig { + t.Errorf("config digest: got %s, want %s", gotConfig, wantConfig) + } + gotLayers, err := got.Layers() + if err != nil { + t.Fatal(err) + } + if len(gotLayers) != len(wantLayerDigests) { + t.Fatalf("layer count: got %d, want %d", len(gotLayers), len(wantLayerDigests)) + } + for i, l := range gotLayers { + d, err := l.Digest() + if err != nil { + t.Fatal(err) + } + if d != wantLayerDigests[i] { + t.Errorf("layer %d digest: got %s, want %s", i, d, wantLayerDigests[i]) + } + } + + for _, h := range append([]v1.Hash{wantManifest, wantConfig}, wantLayerDigests...) { + p := filepath.Join(root, "blobs", h.Algorithm, h.Hex) + if _, err := os.Stat(p); err != nil { + t.Errorf("blob %s missing on disk: %v", h, err) + } + } + + gotDigest, err := s.digestFor("local:tiny") + if err != nil { + t.Fatal(err) + } + if gotDigest != wantManifest.String() { + t.Errorf("pin: got %s, want %s", gotDigest, wantManifest) + } +} + +// TestStoreInteropPullRoundTrip pulls a real image and asserts the store +// round-trips through crane's independent reader, exactly like the offline +// round-trip but with a registry-fetched image. Gated behind ELFUSE_OCI_NETTEST +// so `go test` stays green offline; CI enables it on the Linux conformance job. +func TestStoreInteropPullRoundTrip(t *testing.T) { + if os.Getenv("ELFUSE_OCI_NETTEST") != "1" { + t.Skip("set ELFUSE_OCI_NETTEST=1 to pull real images") + } + ref := "alpine:3" + cf := commonFlags{platform: defaultPlatform} + + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + if err := pullImage(cf, s, ref); err != nil { + t.Fatalf("pull %s failed with ELFUSE_OCI_NETTEST=1: %v", ref, err) + } + + digestStr, err := s.digestFor(ref) + if err != nil { + t.Fatal(err) + } + wantManifest, err := v1.NewHash(digestStr) + if err != nil { + t.Fatal(err) + } + + p, err := layout.FromPath(root) + if err != nil { + t.Fatalf("layout.FromPath: %v", err) + } + got, err := p.Image(wantManifest) + if err != nil { + t.Fatalf("crane reader cannot find manifest %s: %v", wantManifest, err) + } + gotManifest, err := got.Digest() + if err != nil { + t.Fatal(err) + } + if gotManifest != wantManifest { + t.Errorf("manifest digest: got %s, want %s", gotManifest, wantManifest) + } + if _, err := got.ConfigFile(); err != nil { + t.Errorf("ConfigFile: %v", err) + } + layers, err := got.Layers() + if err != nil || len(layers) == 0 { + t.Fatalf("layers: %v (got %d)", err, len(layers)) + } + for _, l := range layers { + if _, err := l.Digest(); err != nil { + t.Errorf("layer digest: %v", err) + } + } +} + +// TestStoreDedupOnRePull asserts that adding the same image twice does not +// accumulate a second manifest descriptor in the layout index (addImage dedups +// by digest), while the ref pin still resolves to that digest. +func TestStoreDedupOnRePull(t *testing.T) { + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + img := tinyImage(t) + want, err := img.Digest() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tiny", img); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:tiny", img); err != nil { + t.Fatal(err) + } + + b, err := os.ReadFile(filepath.Join(root, "index.json")) + if err != nil { + t.Fatal(err) + } + var idx struct { + Manifests []struct { + Digest string `json:"digest"` + } `json:"manifests"` + } + if err := json.Unmarshal(b, &idx); err != nil { + t.Fatalf("index.json unmarshal: %v", err) + } + count := 0 + for _, m := range idx.Manifests { + if m.Digest == want.String() { + count++ + } + } + if count != 1 { + t.Errorf("manifest descriptor for %s appears %d times, want 1 (dedup)", want, count) + } + + got, err := s.digestFor("local:tiny") + if err != nil { + t.Fatalf("digestFor: %v", err) + } + if got != want.String() { + t.Errorf("pin: got %s, want %s", got, want) + } +} + +// TestStoreLayoutRoundTripAfterDescriptorRemoval asserts the index.json our +// removeManifestDescriptor hand-marshals stays parseable by the canonical +// go-containerregistry reader: after rmi drops one of two images, the reader +// finds the survivor and no longer finds the removed manifest. This guards the +// switch from the layout package's RemoveDescriptors to our own durable +// atomic write. +func TestStoreLayoutRoundTripAfterDescriptorRemoval(t *testing.T) { + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + imgA := buildImage(t, []string{"/a"}) + imgB := buildImage(t, []string{"/b"}) + digestA, err := imgA.Digest() + if err != nil { + t.Fatal(err) + } + digestB, err := imgB.Digest() + if err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:a", imgA); err != nil { + t.Fatal(err) + } + if _, err := s.addImage("local:b", imgB); err != nil { + t.Fatal(err) + } + + if _, err := s.rmi("local:a", false); err != nil { + t.Fatalf("rmi local:a: %v", err) + } + + // Re-open with crane's reader, independent of our write path. + p, err := layout.FromPath(root) + if err != nil { + t.Fatalf("layout.FromPath after descriptor removal: %v (index.json not canonical)", err) + } + ii, err := p.ImageIndex() + if err != nil { + t.Fatal(err) + } + im, err := ii.IndexManifest() + if err != nil { + t.Fatal(err) + } + for _, d := range im.Manifests { + if d.Digest == digestA { + t.Fatalf("removed manifest %s still present in index.json", digestA) + } + } + if _, err := p.Image(digestB); err != nil { + t.Fatalf("surviving manifest %s not readable after removal: %v", digestB, err) + } +} + +// TestWriteFileDurableAtomicAndLeavesPriorOnFailure pins writeFileDurable's two +// guarantees: a successful write replaces the file with exactly the new bytes, +// and a failed write (a read-only directory blocks the temp create) leaves any +// prior file untouched rather than truncating it. +func TestWriteFileDurableAtomicAndLeavesPriorOnFailure(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "f") + if err := writeFileDurable(path, []byte("first"), 0o644); err != nil { + t.Fatalf("writeFileDurable first: %v", err) + } + if b, err := os.ReadFile(path); err != nil || string(b) != "first" { + t.Fatalf("after first write = %q, err=%v; want first", b, err) + } + if err := writeFileDurable(path, []byte("second"), 0o644); err != nil { + t.Fatalf("writeFileDurable second: %v", err) + } + if b, err := os.ReadFile(path); err != nil || string(b) != "second" { + t.Fatalf("after second write = %q, err=%v; want second", b, err) + } + + if os.Getuid() == 0 { + t.Skip("running as root: a read-only dir does not block the temp create") + } + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + if err := writeFileDurable(path, []byte("third"), 0o644); err == nil { + t.Fatal("writeFileDurable into read-only dir succeeded, want failure") + } + if err := os.Chmod(dir, 0o755); err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(path); err != nil || string(b) != "second" { + t.Fatalf("after failed write = %q, err=%v; want prior content second", b, err) + } +} + +// TestDigestForUnpulledErrors asserts digestFor errors (not panics) for a ref +// that has no pin in the store. +func TestDigestForUnpulledErrors(t *testing.T) { + root := t.TempDir() + s, err := openStore(root) + if err != nil { + t.Fatal(err) + } + _, err = s.digestFor("local:never-pulled") + if err == nil { + t.Fatal("digestFor: want error for unpulled ref, got nil") + } + if !strings.Contains(err.Error(), "not pulled") { + t.Errorf("digestFor error %q does not mention %q", err, "not pulled") + } +} diff --git a/scripts/ci/oci-cli-smoke.sh b/scripts/ci/oci-cli-smoke.sh new file mode 100755 index 00000000..fb49a43b --- /dev/null +++ b/scripts/ci/oci-cli-smoke.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Store-level CLI lifecycle smoke: everything short of `run` (which needs +# HVF), driving the built binary through the same user-facing flow the Go +# unit tests model in-process: pull, inspect, list, rmi by ref and by +# unique digest prefix, stale-temp-blob GC, and orphan-blob prune. +# +# With --unpack it also unpacks a rootfs and checks that a plain rmi +# reclaims the cold cache (the Linux CI job). Without it, every rmi runs +# the deliberate cache-free path where nothing was ever unpacked and no +# --force is involved (the hosted-macOS CI job, whose runners lack HVF but +# exercise the darwin binary). Needs jq and network. +# +# Usage: scripts/ci/oci-cli-smoke.sh [--unpack] +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 2; } + +UNPACK=0 +case "${1:-}" in +--unpack) UNPACK=1 ;; +'') ;; +*) + echo "usage: $0 [--unpack]" >&2 + exit 2 + ;; +esac + +STORE="$(mktemp -d)" +# Clean up on every exit path; a failing phase must not leak a populated +# blob store into the runner's temp dir. +trap 'rm -rf "$STORE"' EXIT +REF=alpine:3 + +phase_pull_inspect_list() { + "$BIN" version + "$BIN" pull --store "$STORE" "$REF" + "$BIN" inspect --store "$STORE" --json "$REF" \ + | jq -e '(.os == "linux") and (.architecture == "arm64")' >/dev/null + "$BIN" list --store "$STORE" | expect_grep "$REF" +} + +# A cold unpacked cache is derived state: a plain `rmi` reclaims it as part +# of removing the image, no --force needed. --force is only for a +# `run --keep` cache (retained output) or a live run's volume, neither of +# which a bare `unpack` produces. See TestRmiDropsColdCacheWithoutForce in +# cmd/elfuse-oci/lifecycle_test.go. +phase_unpack_rmi() { + local digest cache refs + "$BIN" unpack --store "$STORE" "$REF" + digest="$("$BIN" images --store "$STORE" --json | jq -er '.[0].digest')" + cache="$STORE/rootfs/sha256/${digest#sha256:}" + test -e "$cache/bin/sh" || fail "unpacked rootfs has no /bin/sh" + + must_report 'dropped unpacked cache' 'plain rmi of an unpacked cache' \ + "$BIN" rmi --store "$STORE" "$REF" + test ! -e "$cache" || fail "unpacked cache survived a plain rmi" + refs="$("$BIN" list --store "$STORE")" + [ -z "$refs" ] || fail "list not empty after rmi" + + # Restore the pulled image for the digest-rmi phase below. + "$BIN" pull --store "$STORE" "$REF" +} + +# rmi resolves a unique digest prefix from the list table, and its GC also +# sweeps an aborted download's temp blob (digest name plus random suffix), +# which is unreachable by digest. +phase_digest_rmi_stale_blob() { + local digest layer_hex stale table short refs + digest="$("$BIN" images --store "$STORE" --json | jq -er '.[0].digest')" + layer_hex="$(jq -er '.layers[0].digest' \ + "$STORE/blobs/sha256/${digest#sha256:}" | sed 's/^sha256://')" + stale="$STORE/blobs/sha256/${layer_hex}1072211852" + printf 'stale temp blob' >"$stale" + + table="$("$BIN" list --store "$STORE")" + printf '%s\n' "$table" + short="$(printf '%s\n' "$table" | awk 'NR == 2 {print $2}')" + test -n "$short" || fail "list table has no digest column to resolve" + "$BIN" rmi --store "$STORE" "$short" + test ! -e "$stale" || fail "stale temp blob survived the rmi GC" + refs="$("$BIN" list --store "$STORE")" + [ -z "$refs" ] || fail "list not empty after digest rmi" +} + +# prune's reachability GC reclaims orphan blobs whether or not their name +# parses as a digest. +phase_orphan_prune() { + local valid malformed + valid="$(printf 'ci-prune-orphan' | sha256_hex)" + malformed="$STORE/blobs/sha256/${valid}9999" + printf 'orphan blob' >"$STORE/blobs/sha256/$valid" + printf 'malformed orphan blob' >"$malformed" + "$BIN" prune --store "$STORE" + test ! -e "$STORE/blobs/sha256/$valid" || fail "orphan blob survived prune" + test ! -e "$malformed" || fail "malformed orphan blob survived prune" + "$BIN" prune --store "$STORE" --cache +} + +phase_pull_inspect_list +if [ "$UNPACK" = 1 ]; then + phase_unpack_rmi +fi +phase_digest_rmi_stale_blob +phase_orphan_prune +assert_store_empty "$STORE" + +echo "cli smoke OK (unpack=$UNPACK)" diff --git a/scripts/ci/oci-lib.sh b/scripts/ci/oci-lib.sh new file mode 100755 index 00000000..5f175917 --- /dev/null +++ b/scripts/ci/oci-lib.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Shared helpers for the OCI CI test scripts (oci-run-smoke.sh, +# oci-lifecycle.sh, oci-cli-smoke.sh). Source this first; it enables strict +# mode and an ERR trap so any unguarded failure reports its file, line, and +# command. Without the trap, a bare `test`/`grep -q` failing under plain +# `set -e` kills the script with no output at all, which once cost a full +# CI log forensics session to diagnose. +# +# Bash 3.2 compatible: macOS ships /bin/bash 3.2, so no mapfile, wait -n, +# or ${var,,} here or in the scripts that source this. + +# -E so the ERR trap fires inside functions too. +set -Eeuo pipefail +on_err() { + local s=$? where cmd=$BASH_COMMAND + where="${BASH_SOURCE[1]:-$0}:${BASH_LINENO[0]:-?}" + # ::error:: mirrors what the pre-extraction inline steps emitted, so + # failures still surface as annotations on the PR checks page. + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::error::$where: $cmd (exit $s)" + fi + echo "FAIL $where: $cmd (exit $s)" >&2 +} +trap on_err ERR + +OCI_CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$OCI_CI_DIR/../.." && pwd)" + +fail() { + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::error::$*" + fi + echo "FAIL: $*" >&2 + exit 1 +} + +# require_bin resolves the elfuse-oci binary into BIN, the same +# resolution scripts/oci-interop.sh uses. +require_bin() { + BIN="${ELFUSE_OCI_BIN:-$ROOT/build/elfuse-oci}" + if [ ! -x "$BIN" ]; then + echo "elfuse-oci not found at $BIN (set ELFUSE_OCI_BIN or run 'make build/elfuse-oci')" >&2 + exit 2 + fi +} + +# wait_for TIMEOUT_SEC DESC CMD... polls CMD twice a second until it +# succeeds, and fails loudly with DESC on timeout. Poll loops must live +# here: an inline loop under set -e dies silently on the first transient +# failure of a command substitution. +wait_for() { + local timeout="$1" desc="$2" tries i=0 + shift 2 + tries=$((timeout * 2)) + while [ "$i" -lt "$tries" ]; do + if "$@" >/dev/null 2>&1; then + return 0 + fi + sleep 0.5 + i=$((i + 1)) + done + fail "timed out after ${timeout}s waiting for: $desc" +} + +# expect_grep PATTERN asserts stdin contains the fixed string PATTERN. +# No -q: grep must drain the pipe, or the producer dies of SIGPIPE +# (exit 141) under pipefail when grep exits at the first match. +expect_grep() { + grep -F -- "$1" >/dev/null +} + +# must_report PATTERN DESC CMD... runs a command that must SUCCEED and +# report the fixed string PATTERN on stderr. The stderr is echoed through +# either way so the actual report is visible in the log. +must_report() { + local pattern="$1" desc="$2" errfile + shift 2 + errfile="$(mktemp)" + if ! "$@" 2>"$errfile"; then + cat "$errfile" >&2 + rm -f "$errfile" + fail "$desc: command failed" + fi + cat "$errfile" >&2 + if ! grep -F -- "$pattern" "$errfile" >/dev/null; then + rm -f "$errfile" + fail "$desc: stderr does not mention '$pattern'" + fi + rm -f "$errfile" +} + +# must_refuse PATTERN DESC CMD... runs a command that must FAIL with a +# stderr containing the fixed string PATTERN. The stderr is echoed either +# way so a wrong refusal message is visible in the log. +must_refuse() { + local pattern="$1" desc="$2" errfile + shift 2 + errfile="$(mktemp)" + if "$@" 2>"$errfile"; then + cat "$errfile" >&2 + rm -f "$errfile" + fail "$desc: command succeeded, expected a refusal" + fi + cat "$errfile" >&2 + if ! grep -F -- "$pattern" "$errfile" >/dev/null; then + rm -f "$errfile" + fail "$desc: refusal does not mention '$pattern'" + fi + rm -f "$errfile" +} + +# assert_store_empty STORE asserts a store retains no image state: no +# pinned refs, no blobs, no cs/ sparsebundle bundles, no plain rootfs +# caches, and no volume still mounted beneath it. On Linux the cs/ and +# mount checks pass vacuously. +assert_store_empty() { + local store="$1" refs + # Capture first: a failing `list` inside `[ -z "$(...)" ]` would be + # swallowed (the [ builtin's status is all set -e sees), letting a + # broken list pass the gate; a plain assignment propagates the status. + refs="$("$BIN" list --store "$store")" + [ -z "$refs" ] || fail "store not empty: list still shows pinned refs" + [ -z "$(ls "$store/blobs/sha256" 2>/dev/null || true)" ] \ + || fail "store not empty: blobs remain" + [ -z "$(find "$store/cs" -mindepth 2 -maxdepth 2 -type d 2>/dev/null || true)" ] \ + || fail "store not empty: cs/ bundle dirs remain" + [ -z "$(find "$store/rootfs/sha256" -mindepth 1 -maxdepth 1 2>/dev/null || true)" ] \ + || fail "store not empty: plain rootfs caches remain" + if mount | grep -F "$store" >/dev/null; then + fail "store not empty: a volume is still mounted under $store" + fi +} + +# sha256_hex prints the hex digest of stdin. Hosted macOS runners ship +# shasum but not coreutils sha256sum. +sha256_hex() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' + fi +} diff --git a/scripts/ci/oci-lifecycle.sh b/scripts/ci/oci-lifecycle.sh new file mode 100755 index 00000000..29319819 --- /dev/null +++ b/scripts/ci/oci-lifecycle.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Whole user-facing image lifecycle against a real HVF-booted guest, then +# the teardown guardrails, one function per phase: +# run_workloads pull/inspect/list, an --entrypoint override, +# and a glibc dynamically-linked python workload +# teardown_plain_rmi a plain rmi reclaims the cold unpacked cache +# with the image +# teardown_keep_guardrails rmi refuses to discard run --keep output +# without --force; --force detaches the still- +# attached volume and drops the bundle +# teardown_live_run_lock a live --plain-rootfs guest pins its cache +# against prune --cache --all and rmi until the +# guest exits; only this exercises the lock +# descriptor's ride through the exec into elfuse +# Needs macOS with Hypervisor.framework and network for the seed pull. +# +# Usage: ELFUSE_OCI_STORE= \ +# ELFUSE_OCI_SEED_STORE= \ +# [IMG=] scripts/ci/oci-lifecycle.sh +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +STORE="${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to an ephemeral store directory}" +SEED="${ELFUSE_OCI_SEED_STORE:?set ELFUSE_OCI_SEED_STORE to the warm seed store directory}" +IMG="${IMG:-python:3.12-slim}" +export ELFUSE_OCI_STORE + +guest="" +on_exit() { + rc=$? + # A failed phase must not leak the backgrounded guest: it would keep + # holding the per-digest flock (and its sleep) and poison the next + # prune/rmi on a persistent self-hosted runner. + if [ -n "$guest" ] && kill -0 "$guest" 2>/dev/null; then + kill "$guest" 2>/dev/null || true + wait "$guest" 2>/dev/null || true + fi + exit "$rc" +} +trap on_exit EXIT + +# The teardowns leave the ephemeral store EMPTY (asserted), so the +# lifecycle store itself cannot persist between CI runs. Keep a warm seed +# store on the persistent disk instead and clone it in per phase (cp -Rc, +# APFS clonefile, the same trick as the fixture-cache restore): the pull +# in run_workloads then dedups every blob by digest and only the manifest +# HEAD/GET goes out, while the empty-store assertions stay meaningful. +seed_warm_store() { + ELFUSE_OCI_STORE="$SEED" "$BIN" pull "$IMG" + # GC blobs stranded in the seed when the tag moves to a new digest. + ELFUSE_OCI_STORE="$SEED" "$BIN" prune >/dev/null +} + +reseed() { + rm -rf "$STORE" + cp -Rc "$SEED" "$STORE" +} + +run_workloads() { + "$BIN" version + "$BIN" pull "$IMG" + "$BIN" inspect "$IMG" | expect_grep 'linux/arm64' + "$BIN" list | expect_grep "$IMG" + + local out + out="$("$BIN" run --entrypoint /usr/local/bin/python3 "$IMG" \ + -c 'import json,math; print(json.dumps({"pi":round(math.pi,5),"ok":True}))')" + printf 'guest said: %s\n' "$out" + [ "$out" = '{"pi": 3.14159, "ok": true}' ] || fail "python one-liner said '$out'" + + # A non-trivial application: concurrent SQLite writers (fcntl locking, + # fsync, WAL where the guest FS supports it) plus a 64-file + # write/read/checksum fan-out. Exercises far more of the dynamically- + # linked glibc guest than the one-liner above; prints a single sentinel + # token only on full success. + out="$("$BIN" run --entrypoint /usr/local/bin/python3 "$IMG" \ + -c "$(cat "$OCI_CI_DIR/oci-python-workload.py")")" + printf 'python workload said: %s\n' "$out" + [ "$out" = elfuse-oci-python-workload-ok ] || fail "python workload said '$out'" +} + +# The runs above left the cache warm and then detached on exit, so a plain +# rmi (no --force, no separate prune) must reclaim it with the image. +teardown_plain_rmi() { + must_report 'dropped unpacked cache' 'plain rmi of a cold cache' \ + "$BIN" rmi "$IMG" + assert_store_empty "$STORE" +} + +# run --keep retains the per-run clone with the volume still attached: rmi +# must refuse without --force, and rmi --force must detach the volume, +# drop the bundle, and GC the blobs. +teardown_keep_guardrails() { + reseed + "$BIN" run --keep --entrypoint /usr/local/bin/python3 "$IMG" -c 'pass' + must_refuse 'retained run --keep output; pass --force' \ + 'rmi of run --keep output without --force' \ + "$BIN" rmi "$IMG" + "$BIN" rmi --force "$IMG" + assert_store_empty "$STORE" +} + +# The published cache is rootfs/sha256/, renamed into place when the +# unpack completes. Never match the unpacker's .tmp- staging +# sibling: it is mid-write, and prune skips it via the digest lock. +published_plain_cache() { + # sed, not `head -n1`: head exits at the first line, and under pipefail + # a find killed by the resulting SIGPIPE would fail the whole pipeline + # even though a cache was found. sed -n 1p drains its input. + find "$STORE/rootfs/sha256" -mindepth 1 -maxdepth 1 -type d \ + ! -name '*.tmp-*' 2>/dev/null | sed -n 1p | grep . +} + +# A live --plain-rootfs guest must pin its cache against prune --cache +# --all and rmi, and its exit alone (no cleanup code, SIGKILL included) +# must free it: the kernel drops the flock with the process. +teardown_live_run_lock() { + reseed + "$BIN" run --plain-rootfs --entrypoint /bin/sleep "$IMG" 60 & + guest=$! + # The run takes its per-digest lock before unpacking, so once the + # cache dir has been published the guest provably holds the lock. + wait_for 120 'published plain rootfs cache' published_plain_cache + local plain_cache + plain_cache="$(published_plain_cache)" + + "$BIN" prune --cache --all + test -d "$plain_cache" || fail 'prune --cache --all reclaimed a live plain rootfs' + must_refuse 'in use by a live run' \ + 'rmi of an image whose plain rootfs hosts a live run' \ + "$BIN" rmi "$IMG" + + kill "$guest" 2>/dev/null || true + wait "$guest" || true + guest="" + # Guest gone means the kernel released the flock; prune reclaims the + # cache (lock file included) and the image can finally be removed. + "$BIN" prune --cache --all + test ! -e "$plain_cache" || fail 'plain rootfs cache survived prune after guest exit' + "$BIN" rmi "$IMG" + assert_store_empty "$STORE" + + # Nothing left to reclaim. + "$BIN" prune --cache +} + +seed_warm_store +reseed +run_workloads +teardown_plain_rmi +teardown_keep_guardrails +teardown_live_run_lock + +echo "lifecycle OK" diff --git a/scripts/ci/oci-python-workload.py b/scripts/ci/oci-python-workload.py new file mode 100644 index 00000000..6c1c044d --- /dev/null +++ b/scripts/ci/oci-python-workload.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Non-trivial guest workload for the elfuse-oci OCI lifecycle CI. + +Exercises more of a real dynamically-linked glibc guest than a one-line print: +concurrent SQLite writers (fcntl locking, fsync, and, when the guest FS +supports it, WAL mmap), plus a filesystem fan-out whose written content is +read back and checksummed. On full success it prints a single sentinel token +the workflow asserts on; any failure exits non-zero with a diagnostic. + +Self-contained (stdlib only) so it runs under the image's bundled Python with +no network or extra packages, and is passed to the guest via `python3 -c`. +""" + +import hashlib +import os +import sqlite3 +import sys +import threading + +DB = "/tmp/elfuse-workload.db" +TREE = "/tmp/elfuse-workload-tree" +THREADS = 8 +PER_THREAD = 1000 +FANOUT = 8 + + +def setup_db(): + con = sqlite3.connect(DB) + try: + # WAL exercises the guest FS's shared-memory index (mmap) and is the + # more demanding path; if the FS cannot back it, SQLite reports a + # different mode and the concurrent-writer count check below still + # validates fcntl locking and durable commits under rollback journal. + con.execute("PRAGMA journal_mode=WAL") + con.execute( + "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "tid INTEGER NOT NULL, n INTEGER NOT NULL)" + ) + con.commit() + finally: + con.close() + + +def worker(tid): + con = sqlite3.connect(DB, timeout=60) + try: + con.execute("PRAGMA busy_timeout=60000") + for n in range(PER_THREAD): + con.execute("INSERT INTO t (tid, n) VALUES (?, ?)", (tid, n)) + con.commit() + finally: + con.close() + + +def db_count(): + con = sqlite3.connect(DB, timeout=60) + try: + (count,) = con.execute("SELECT COUNT(*) FROM t").fetchone() + return count + finally: + con.close() + + +def content(i, j): + return "elfuse-workload-%d-%d\n" % (i, j) + + +def fs_fanout_ok(): + for i in range(FANOUT): + for j in range(FANOUT): + d = os.path.join(TREE, str(i), str(j)) + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, "f"), "w") as fh: + fh.write(content(i, j)) + fh.flush() + os.fsync(fh.fileno()) + read_back = [] + for i in range(FANOUT): + for j in range(FANOUT): + with open(os.path.join(TREE, str(i), str(j), "f")) as fh: + read_back.append(fh.read()) + expected = [content(i, j) for i in range(FANOUT) for j in range(FANOUT)] + got = hashlib.sha256() + for p in sorted(read_back): + got.update(p.encode()) + want = hashlib.sha256() + for p in sorted(expected): + want.update(p.encode()) + return got.hexdigest() == want.hexdigest() + + +def main(): + setup_db() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + count = db_count() + if count != THREADS * PER_THREAD: + print( + "sqlite row count %d != %d (concurrent writers lost rows)" + % (count, THREADS * PER_THREAD), + file=sys.stderr, + ) + sys.exit(1) + + if not fs_fanout_ok(): + print("filesystem fan-out checksum mismatch", file=sys.stderr) + sys.exit(1) + + print("elfuse-oci-python-workload-ok") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/oci-run-smoke.sh b/scripts/ci/oci-run-smoke.sh new file mode 100755 index 00000000..2173dd60 --- /dev/null +++ b/scripts/ci/oci-run-smoke.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# End-to-end smoke for the default `run` path: pull an image, provision the +# case-sensitive sparsebundle, COW-clone it, boot the guest under HVF, and +# check output, exit status, and per-run isolation. Needs macOS with +# Hypervisor.framework; network only when the store is cold (pull is +# idempotent per digest, so a warm store skips the blob downloads). +# +# Usage: ELFUSE_OCI_STORE= scripts/ci/oci-run-smoke.sh +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +: "${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to the store directory to use}" + +out="$("$BIN" run alpine:3 /bin/echo elfuse-oci-ci-ok)" +printf 'guest said: %s\n' "$out" +printf '%s\n' "$out" | expect_grep elfuse-oci-ci-ok + +# The guest's exit status must propagate through the runner untouched; +# cleanup errors must never win over it. +code=0 +"$BIN" run alpine:3 /bin/sh -c 'exit 7' || code=$? +[ "$code" -eq 7 ] || fail "guest exit status: got $code, want 7" + +# Non-trivial multi-stage shell pipeline: generate a 200k-line file, +# gzip it, decompress and byte-compare the round-trip, then checksum +# the original against a constant precomputed from the exact same +# deterministic input. Exercises fork/exec pipelines, pipes, and +# coreutils gzip/sha256sum in the guest. debian:stable-slim rather +# than alpine because the bare-name PATH search must resolve every +# candidate inside the rootfs: the sysroot resolver falls back to +# the host for absent absolute paths, and alpine ships gzip only at +# /bin/gzip while its PATH tries /usr/bin first, which on a macOS +# host holds an incompatible Mach-O gzip. Debian is usr-merged, so +# each searched binary exists at /usr/bin inside the image. +want=5af7b95208fdcff454bab3f5eddf567a688a3796c703d4fef91072e38645c062 +got="$("$BIN" run debian:stable-slim /bin/sh -c 'set -e + seq 1 200000 > /tmp/data.txt + gzip -c /tmp/data.txt > /tmp/data.gz + gunzip -c /tmp/data.gz | cmp - /tmp/data.txt + sha256sum /tmp/data.txt | cut -d" " -f1')" +printf 'debian pipeline sha256: %s\n' "$got" +[ "$got" = "$want" ] || fail "debian pipeline sha256: got $got, want $want" + +# Per-run COW clone isolation: the previous run's /tmp writes must not be +# visible to a fresh run of the same digest. Exact match, not a substring: +# a diagnostic quoting the failed command would also contain the token. +out="$("$BIN" run debian:stable-slim /bin/sh -c 'test ! -e /tmp/data.txt && echo isolated-ok')" +[ "$out" = isolated-ok ] || fail "isolation check said '$out'" + +# When a pinned tag moves, the re-pin strands the old digest's blobs; GC +# them so a persistent store stays bounded. +"$BIN" prune >/dev/null + +echo "run smoke OK" diff --git a/scripts/oci-interop.sh b/scripts/oci-interop.sh new file mode 100755 index 00000000..37f4e89a --- /dev/null +++ b/scripts/oci-interop.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# OCI image-layout conformance + cross-tool interop for the elfuse-oci store. +# +# Treats the on-disk store as the contract: after `elfuse-oci pull`, the store +# must be a valid OCI image-layout that other tools can read and that agrees +# with registry truth on the manifest digest. +# +# Hard assertions (always available, gate the script): +# - oci-layout has imageLayoutVersion 1.0.0; index.json is schemaVersion 2 +# with a manifest descriptor matching the pinned digest. +# - the manifest blob parses and references a config blob + >=1 layer blob, +# all present under blobs/sha256/. +# - if `crane` is installed, the store's pinned manifest digest matches +# registry truth for the selected platform. +# +# Best-effort third-party reads (run when present; fatal on failure so CI can +# promote them once the invocation is confirmed): +# - skopeo inspect --raw oci::@ reads our layout +# - umoci list --layout parses our layout +# +# Usage: scripts/oci-interop.sh [STORE_DIR] +# Env: ELFUSE_OCI_BIN (path to elfuse-oci; default build/elfuse-oci) +# FIXTURES (space-separated refs; default "alpine:3 busybox") +# PLAT_OS / PLAT_ARCH (platform to pull and compare; default linux/arm64) +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +BIN="${ELFUSE_OCI_BIN:-$ROOT/build/elfuse-oci}" + +# A store passed as $1 belongs to the caller; an auto-created one is ours to +# reclaim, including on a mid-run failure. +if [ -n "${1:-}" ]; then + STORE="$1" + STORE_OWNED=0 +else + STORE="$(mktemp -d -t elfuse-interop.XXXXXX)" + STORE_OWNED=1 +fi +# Absolutize the store path: the skopeo view symlinks blobs -> "$STORE/blobs", +# a target resolved relative to the (separate) view dir, so a relative store arg +# would dangle. skopeo/umoci oci: transports also want an absolute layout path. +case "$STORE" in + /*) ;; + *) STORE="$(pwd)/$STORE" ;; +esac + +# Temp files/dirs created per fixture. An EXIT trap reclaims them plus the +# auto-created store, so a failure (set -e) or fail() no longer orphans the +# store's pulled blobs or the per-iteration scratch. Bash 3.2 (macOS) errors on +# "${arr[@]}" for an empty array under set -u, so guard on length. +TMPFILES=() +cleanup() { + if [ "${#TMPFILES[@]}" -gt 0 ]; then + rm -rf "${TMPFILES[@]}" + fi + if [ "$STORE_OWNED" = 1 ]; then + rm -rf "$STORE" + fi + return 0 +} +trap cleanup EXIT + +FIXTURES="${FIXTURES:-alpine:3 busybox}" +# One platform drives BOTH the pull and the registry-truth comparison; setting +# it only on the crane side would fail the comparison against a correctly +# pulled default-platform image. +PLAT_OS="${PLAT_OS:-linux}" +PLAT_ARCH="${PLAT_ARCH:-arm64}" + +have() { command -v "$1" >/dev/null 2>&1; } + +if [ ! -x "$BIN" ]; then + echo "elfuse-oci not found at $BIN (set ELFUSE_OCI_BIN or run 'make build/elfuse-oci')" >&2 + exit 2 +fi +have jq || { echo "jq is required" >&2; exit 2; } + +echo "store: $STORE" +echo "bin: $BIN" +mkdir -p "$STORE" + +# Hard failures exit directly via `fail`. +fail() { echo "FAIL: $*" >&2; exit 1; } + +for ref in $FIXTURES; do + echo + echo "=== $ref ===" + + # Pull into the store. + "$BIN" pull --store "$STORE" --platform "$PLAT_OS/$PLAT_ARCH" "$ref" >/dev/null + digest="$(jq -er --arg ref "$ref" '.[$ref]' "$STORE/refs.json" \ + || fail "refs.json has no pin for $ref")" + echo "pinned manifest digest: $digest" + + # --- Conformance: oci-layout --- + [ "$(jq -r .imageLayoutVersion "$STORE/oci-layout")" = "1.0.0" ] \ + || fail "oci-layout imageLayoutVersion != 1.0.0" + + # --- Conformance: index.json has our manifest descriptor --- + [ "$(jq -r .schemaVersion "$STORE/index.json")" = "2" ] \ + || fail "index.json schemaVersion != 2" + jq -e --arg d "$digest" 'any(.manifests[]; .digest == $d)' \ + "$STORE/index.json" >/dev/null \ + || fail "index.json has no manifest descriptor with digest $digest" + + # --- Conformance: manifest blob parses and references config + layers --- + hex="${digest#sha256:}" + manifest_path="$STORE/blobs/sha256/$hex" + [ -f "$manifest_path" ] || fail "manifest blob missing at $manifest_path" + config_digest="$(jq -er .config.digest "$manifest_path" \ + || fail "manifest blob is not a valid image manifest (no .config.digest)")" + config_hex="${config_digest#sha256:}" + [ -f "$STORE/blobs/sha256/$config_hex" ] \ + || fail "config blob missing for $config_digest" + layer_count="$(jq '.layers | length' "$manifest_path")" + [ "$layer_count" -ge 1 ] || fail "manifest has no layers" + # Every layer blob must exist on disk. + while IFS= read -r ld; do + lx="${ld#sha256:}" + [ -f "$STORE/blobs/sha256/$lx" ] || fail "layer blob missing for $ld" + done < <(jq -r '.layers[].digest' "$manifest_path") + echo "ok: layout valid, $layer_count layer(s), config + manifest + layers present" + + # --- Interop: registry truth via crane (if installed) --- + # `elfuse-oci pull` uses crane.Pull(WithPlatform), which resolves a manifest + # list to the per-arch child manifest and pins THAT digest. So for a + # multi-arch ref, `crane digest` (the list digest) legitimately differs; we + # resolve the platform child from `crane manifest` and compare that. + if have crane; then + plat_os="$PLAT_OS"; plat_arch="$PLAT_ARCH" + top="$(crane manifest "$ref")" + if [ "$(printf '%s' "$top" | jq '.manifests // [] | any(.platform != null)')" = "true" ]; then + # first(...) keeps the comparison single-valued if several entries + # match the platform (e.g. multiple variants), and the annotation + # filter drops BuildKit attestation manifests, which are not + # runnable images. crane.Pull resolves the same first match. + reg_digest="$(printf '%s' "$top" | jq -er --arg os "$plat_os" --arg ar "$plat_arch" \ + 'first(.manifests[] + | select(.platform.os==$os and .platform.architecture==$ar + and (.annotations["vnd.docker.reference.type"] != "attestation-manifest")) + | .digest)' \ + || fail "crane manifest list for $ref has no $plat_os/$plat_arch entry")" + else + reg_digest="$(crane digest "$ref")" + fi + [ "$reg_digest" = "$digest" ] \ + || fail "crane ($reg_digest) != store pin ($digest) for $ref [$plat_os/$plat_arch]" + echo "ok: crane agrees on manifest digest ($plat_os/$plat_arch)" + else + echo "info: crane not installed; skipping registry-truth comparison" + fi + + # --- Interop: skopeo reads our layout (if installed) --- + if have skopeo; then + # skopeo's oci: transport addresses an image by ref-name annotation or + # (since skopeo 1.14) by @source-index. Our layout intentionally + # carries no ref-name annotations, and distro skopeo is often older + # than 1.14, so neither form is portable. Present a single-manifest + # view instead (the same oci-layout and blobs, index.json filtered + # to the pinned descriptor), which every skopeo version resolves + # with a bare oci: reference. + skopeo_view="$(mktemp -d -t elfuse-skopeo-view.XXXXXX)" + TMPFILES+=("$skopeo_view") + cp "$STORE/oci-layout" "$skopeo_view/oci-layout" + jq --arg d "$digest" \ + '.manifests = [.manifests[] | select(.digest == $d)]' \ + "$STORE/index.json" >"$skopeo_view/index.json" + ln -s "$STORE/blobs" "$skopeo_view/blobs" + skopeo_ref="oci:$skopeo_view" + skopeo_raw="$(mktemp -t elfuse-skopeo-raw.XXXXXX)" + skopeo_err="$(mktemp -t elfuse-skopeo-err.XXXXXX)" + TMPFILES+=("$skopeo_raw" "$skopeo_err") + if skopeo inspect --raw "$skopeo_ref" >"$skopeo_raw" 2>"$skopeo_err"; then + jq -e '(.schemaVersion == 2) and (.config.digest | type == "string") and + (.layers | type == "array") and (.layers | length >= 1)' \ + "$skopeo_raw" >/dev/null \ + || fail "skopeo read $skopeo_ref but did not return an image manifest" + echo "ok: skopeo inspect --raw $skopeo_ref read the pinned manifest" + else + echo "skopeo stderr: $(cat "$skopeo_err")" >&2 + fail "skopeo could not read $skopeo_ref" + fi + rm -f "$skopeo_raw" "$skopeo_err" + rm -rf "$skopeo_view" + else + echo "info: skopeo not installed; skipping skopeo interop" + fi + + # --- Interop: umoci parses our layout (if installed) --- + # The layout is valid even when no descriptors carry ref-name annotations. + # elfuse keeps refs.json as its own lookup table for full pull references; + # umoci list may therefore show zero tags, but it must still parse. + if have umoci; then + umoci_out="$(mktemp -t elfuse-umoci-list.XXXXXX)" + umoci_err="$(mktemp -t elfuse-umoci-err.XXXXXX)" + TMPFILES+=("$umoci_out" "$umoci_err") + if umoci list --layout "$STORE" >"$umoci_out" 2>"$umoci_err"; then + echo "ok: umoci list --layout parsed our layout (tags: $(wc -l <"$umoci_out" | tr -d ' '))" + else + echo "umoci stderr: $(cat "$umoci_err")" >&2 + fail "umoci could not parse layout $STORE" + fi + rm -f "$umoci_out" "$umoci_err" + else + echo "info: umoci not installed; skipping umoci interop" + fi +done + +echo +echo "ALL OK: store is a valid OCI image-layout and interops with available tools" +# The EXIT trap reclaims the auto-created store and any per-iteration scratch. From a3d0ba963c68af0ec5a7e6b13b113b3d0000bec6 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Jul 2026 23:38:59 +0200 Subject: [PATCH 09/10] Document OCI image support Cover the two-binary model in README and docs: usage.md documents the elfuse-oci commands and flags, testing.md the offline/CI validation split, internals.md the host-literal path fallback semantics, and oci-design.md records the design rationale (the C/Go boundary, the image-layout store with its refs.json pin table, layer application, run paths, and lifecycle GC), plus an explicit scope-and-limitations accounting of which OCI features are and are not implemented. oci-design.md also records the concurrency model: store metadata is lock-serialized, per-digest sparsebundle state is coordinated by the attach.lock/run.lock pair, and the plain digest-keyed rootfs cache is guarded by a sibling per-digest lock a run holds across the exec into elfuse, so prune skips and rmi refuses a cache a live guest still uses. usage.md notes the resolv.conf fallback nameserver and its split-DNS implication. README states the positioning up front: OCI images are consumed as a distribution vehicle for Linux root filesystems, replacing hand-built --sysroot trees, and the non-isolation limitations (host-path fallback, shared network identity, PID space, and clock) are called out explicitly rather than implied. --- README.md | 43 ++++++- docs/internals.md | 11 +- docs/oci-design.md | 298 +++++++++++++++++++++++++++++++++++++++++++++ docs/testing.md | 110 ++++++++++++++++- docs/usage.md | 180 +++++++++++++++++++++++++++ 5 files changed, 637 insertions(+), 5 deletions(-) create mode 100644 docs/oci-design.md diff --git a/README.md b/README.md index 99bc6a2d..b2db5988 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ boot-time overhead those tools impose. - Xcode Command Line Tools, `clang`, `codesign`, and GNU `make` - GNU `objcopy` or `llvm-objcopy` - Hypervisor entitlement: `com.apple.security.hypervisor` +- Go, for building the `elfuse-oci` OCI CLI To build only (`make elfuse`) without running tests, just the Xcode Command Line Tools and `objcopy` (`brew install binutils`) suffice. @@ -101,11 +102,43 @@ state. The build signs `build/elfuse` before use. Override the signing identity with `SIGN_IDENTITY="Developer ID ..."` when needed. +## OCI Images + +OCI images are handled by `elfuse-oci`, a Go companion binary that owns +the whole image lifecycle and invokes `elfuse` purely as the runtime. The point +is distribution, not containment: an image is consumed as a packaged Linux root +filesystem, replacing the hand-built `--sysroot` trees that dynamically-linked +guests previously required. The execution model is unchanged (Linux ELF, elfuse +syscall translation, macOS kernel), with no VM and no Linux kernel in the loop. + +This is not a container runtime. The image rootfs is the guest's root, not an +isolation boundary: an absolute guest path absent from the rootfs falls back to +the literal host path, and the guest shares the host's network identity, PID +space, and clock. There are no namespaces, cgroups, port mapping, or daemon. +The target is CLI tooling, compilers, and scripting userspace; +[docs/oci-design.md](docs/oci-design.md#scope-and-limitations) lists exactly +which OCI features are implemented. + +```sh +make elfuse elfuse-oci + +build/elfuse-oci pull alpine:3 +build/elfuse-oci run alpine:3 /bin/sh -c 'echo hello from elfuse' +``` + +Images are stored under `$ELFUSE_OCI_STORE`, or `~/.local/share/elfuse/oci` +by default. On macOS, `run` uses a case-sensitive APFS sparsebundle and a +per-run copy-on-write rootfs clone so normal APFS case folding does not +corrupt Linux filenames. + +See [docs/usage.md](docs/usage.md#oci-images) for commands and flags, and +[docs/oci-design.md](docs/oci-design.md) for the implementation model. + ## Documentation - [docs/usage.md](docs/usage.md): command-line options, x86_64 via - Rosetta, dynamic linking via `--sysroot`, and attaching `gdb` / - `lldb` to the built-in stub. + Rosetta, dynamic linking via `--sysroot`, OCI images, and attaching + `gdb` / `lldb` to the built-in stub. - [docs/testing.md](docs/testing.md): build prerequisites, the `make check` flow, the QEMU and Rosetta cross-check matrices, and fixture handling. @@ -113,6 +146,9 @@ The build signs `build/elfuse` before use. Override the signing identity with reference -- runtime lifecycle, HVF constraints, EL1 shim and HVC protocol, page-table splitting, syscall translation tables, threads / futex, fork / clone IPC, signals, ptrace, and the GDB stub. +- [docs/oci-design.md](docs/oci-design.md): how elfuse-oci, the image + store, layer unpacker, sparsebundle run path, and lifecycle commands + fit into elfuse. ## Build And Validation @@ -123,6 +159,7 @@ make elfuse # build and codesign build/elfuse make check # quick unit suite + BusyBox applet smoke make test-gdbstub # debugger integration make test-matrix # cross-check elfuse against QEMU on the same corpus +make oci-test # elfuse-oci unit and conformance tests make lint # clang-tidy ``` @@ -141,6 +178,8 @@ do. - Linux kernel features that have no user-space-syscall analog: namespaces, cgroups, kernel modules, eBPF, `io_uring`, KVM, perf events. +- Docker-compatible container runtime features such as port mapping, + detached containers, `docker exec`, image build/push, and daemon APIs. - Intel Macs. Apple Silicon only (M1 and later). - Hosting a VM from inside a guest. The guest cannot use HVF or KVM. - One guest process tree per `elfuse` host process. HVF allows one VM diff --git a/docs/internals.md b/docs/internals.md index 680a4771..9e18aec8 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -792,7 +792,16 @@ How it works: 4. The entry point becomes `interp_entry + load_base`; the dynamic linker takes over from there. 5. `sys_openat()` redirects guest absolute paths through the sysroot: if - `--sysroot` is set, it tries `/` first. + `--sysroot` is set, it tries `/` first, then falls back to + the literal host path when the target is absent under the sysroot (so a + dynamic binary can still reach host resources such as `/tmp`). + +That host-literal fallback applies to image-launched guests as well: the sysroot +is a root, not a boundary, and `elfuse-oci run` inherits it unchanged. +A guest `PATH` search whose candidate is absent under the sysroot can +therefore resolve to a host binary; `run` compensates by guaranteeing the +guest a `PATH` (Docker's conventional default when the image env provides +none), and callers can reorder the search with `--env PATH=...`. The sysroot is inherited by fork children via IPC state transfer. `sys_execve` also loads the interpreter for dynamically linked targets, so diff --git a/docs/oci-design.md b/docs/oci-design.md new file mode 100644 index 00000000..103b5a75 --- /dev/null +++ b/docs/oci-design.md @@ -0,0 +1,298 @@ +# OCI Image Support Design + +This is the reference for how elfuse consumes OCI images without becoming a +container runtime. It is the single source of truth for what is and is not +implemented. For day-to-day commands and flags, see +[usage.md](usage.md#oci-images); for validation targets, see +[testing.md](testing.md). + +## Model + +elfuse uses the OCI image format for distribution and filesystem packaging +only. It does not implement the OCI runtime spec. The goal is narrow: pull an +image, unpack its layers into a Linux rootfs, resolve the image runtime +configuration, and launch the configured program through the existing +`elfuse --sysroot` path. The guest is a single elfuse process translating +Linux syscalls to Darwin, not an isolated container. + +## Scope And Limitations + +The OCI ecosystem is three specifications plus a set of conventions. elfuse +implements the consumer side of the image format and nothing else. + +Implemented: + +- the on-disk OCI image-layout store (`oci-layout`, `index.json`, + content-addressed blobs) plus an elfuse-specific `refs.json` pin file; +- pulling with `go-containerregistry` through the ambient default keychain, + with `--platform` selection resolved against manifest lists; +- layer application: whiteouts and opaque directories, hardlinks and symlinks + (absolute targets rewritten rootfs-relative), setuid/setgid/sticky bits, and + gzip or zstd compression, all applied under `os.OpenRoot`; +- image-config resolution of `Entrypoint`, `Cmd`, `Env`, `User`, and + `WorkingDir` with Docker-style precedence; +- local lifecycle management (`list`, `rmi`, `prune`) with reachability-based + garbage collection and macOS sparsebundle cache handling; +- runtime injection of `/etc/resolv.conf`, `/etc/hosts`, and `/etc/hostname` + into the run rootfs. + +Out of scope: + +- **OCI runtime spec.** No runtime bundle or `config.json`, and no namespaces, + cgroups, seccomp, capabilities, hooks, or mounts/volumes. The image rootfs is + the guest's root but not a boundary: an absolute guest path absent from the + rootfs falls back to the host filesystem (see [Run Paths](#run-paths)), and + the guest shares the host network identity, PID space, and clock. +- **Distribution write side.** Pull only: no `push`, no image building, and no + `login`. Credentials come from the ambient default keychain (for example an + existing Docker credential store); elfuse adds none of its own. +- **Network and port isolation.** The guest shares the host network. There is + no port mapping, and `ExposedPorts` has no effect. +- **Ignored image-config fields.** `Volumes`, `ExposedPorts`, `Healthcheck`, + `StopSignal`, and `Labels` are accepted but have no runtime effect. +- **Daemon conveniences.** No daemon, no `exec` or `attach`, no detached + containers; each `run` is one foreground guest process. +- **Non-Linux images.** Only `linux` images, on the platforms the runtime + executes: `arm64` natively, `amd64` via Rosetta. +- **Device nodes in layers.** Character, block, and FIFO entries are rejected + rather than materialized; the C runtime synthesizes the supported `/dev` and + `/proc` entries at run time. +- **Supply-chain verification.** Content is verified against manifest digests, + but there is no signature or attestation checking (cosign, notation) and no + policy engine. + +A workload that needs any of the above needs a container runtime, not an ELF +runner that consumes OCI images. + +## Library Boundary + +`elfuse-oci` imports `go-containerregistry` for registry transport and +image-layout blob access, and owns everything above that layer: durable store +writes, layer application, runtime-configuration resolution, and lifecycle +management. The obvious alternative (shelling out to `skopeo` for pull and +delete plus `umoci` for unpack; both install cleanly on macOS) was evaluated +empirically: a prototype wrapper over the two tools was driven through this +repository's entire OCI CI suite unmodified. + +The wrapper passed the suite, but only by reimplementing on top of the tools +every macOS and lifecycle behavior the CI asserts: + +- `umoci gc` aborts on a stale temporary blob whose name is not a valid + digest (the exact debris prune must sweep), so the reachability sweep had + to be rebuilt; +- umoci stores absolute symlink targets verbatim, so the rootfs-relative + rewrite of [Layer Application](#layer-application) had to be rebuilt; +- no copy tool tolerates the setgid `chmod` `EPERM` that group inheritance + produces on macOS volumes, so the special-bit degrade had to be rebuilt; +- no tool models unpacked caches, sparsebundles, or per-run clones, so the + whole [Lifecycle](#lifecycle) layer had to be rebuilt. + +What the tools cover well (registry transport and whiteout-correct layer +application) is the part this design already delegates to +`go-containerregistry` or that changes rarely. Shelling out would also give +up crash-durable store writes (both tools write `index.json` with plain +writes), the flock-based store and live-run locking, the device-node +rejection policy (umoci materializes empty placeholder files instead), and +the single-binary install. The boundary therefore stays put: import +`go-containerregistry` as a library, own every behavior the CI asserts, and +keep skopeo and umoci as independent readers that cross-check the store (see +[Validation](#validation)). + +## Boundary Between C And Go + +There are two binaries with a one-way dependency: `elfuse-oci` calls +`elfuse`, never the reverse, so the runtime stays useful on its own as a plain +ELF runner. + +`build/elfuse` (C) is purely the Linux syscall-to-Darwin runtime, with no OCI +awareness. It provides the positional ELF launcher, the `--user`/`--workdir`/ +`--env`/`--clear-env` launch flags, and the synthetic `/proc` and `/dev` +entries served to every guest. + +`build/elfuse-oci` (Go) is the only OCI entry point. It pulls images, +maintains the image-layout store, inspects and unpacks stored images, resolves +the runtime configuration, prepares the runtime `/etc` files, and invokes +`elfuse` with the resolved launch flags. It locates `elfuse` as a sibling of +its own executable; `$ELFUSE_BIN` overrides the location for tests and wrapper +scripts. + +## Store + +The store is an OCI image-layout directory plus one pin file: + +```text +/ + oci-layout + index.json + blobs/sha256/ + refs.json +``` + +`oci-layout`, `index.json`, and `blobs/` are the standard layout. `refs.json` +maps each original image reference to the manifest digest elfuse pinned at pull +time. It is elfuse-specific lookup metadata; OCI readers parse the layout +through `index.json` and the content-addressed blobs without it. Keeping it +separate preserves the exact pull reference (`docker.io/library/alpine:3`, +`name@sha256:...`). + +The default store is `$ELFUSE_OCI_STORE` when set, otherwise +`~/.local/share/elfuse/oci`. + +## Pull And Platform Selection + +`pull` defaults to `linux/arm64`, matching the native Apple Silicon guest path. +`--platform os/arch[/variant]` selects another image, such as `linux/amd64` for +a Rosetta-backed guest. When a reference is a manifest list, `pull` fetches and +pins the selected platform's child manifest, so the pinned digest can differ +from the top-level manifest-list digest that registry tools report. + +## Layer Application + +Extraction runs under `os.OpenRoot`, so every layer path resolves relative to +the target rootfs and cannot escape via `..` or a symlink. The unpacker +implements the layer behavior common images need: + +- regular files, directories, symlinks, and hardlinks; +- Docker/OCI whiteouts and opaque directory markers; +- permission bits plus setuid, setgid, and sticky, finalized with an explicit + chmod so layer modes survive a restrictive host umask. Where an unprivileged + host rejects the setuid/setgid chmod, the unpacker drops the special bits + (with a warning) and continues rather than aborting: the rootfs is owned by + the invoking user, so those bits could not be honored there anyway. This + fires on macOS when the unpacked file's inherited group is one the invoking + user is not in (a new file takes its parent directory's group, e.g. wheel + under `/tmp`), which is what lets Debian-family images and their shadow suite + (`chage`, `passwd`, ...) unpack at all. Linux keeps the bits (an owned-file + chmod succeeds), so the C runtime's setuid-exec support still applies where + they survive; +- absolute symlink targets rewritten to rootfs-relative links; +- rejection of special files elfuse does not materialize from layers. + +Runtime `/dev` and `/proc` entries are not unpacked from layers; the C runtime +synthesizes the supported ones when the guest opens them. + +## Run Paths + +On macOS the default `run` path uses a case-sensitive APFS sparsebundle per +manifest digest, with the unpacked base rootfs inside it. Each run makes an +APFS copy-on-write clone of the base tree, launches elfuse against the clone, +removes the clone, and detaches the sparsebundle once the last concurrent run +of that digest exits. This protects case-sensitive Linux filenames on normal +(case-insensitive) macOS volumes and keeps repeated runs isolated from +image-layer mutations. + +The plain-rootfs path stays available with `--plain-rootfs` or an explicit +`--rootfs`. It uses a regular directory and execs `elfuse` directly, which is +useful for debugging and for non-Darwin operation. The default sparsebundle +sits on a volume the invoking user created, so unpacked files inherit that +user's own group and the setuid/setgid chmod above succeeds; the special-bit +degrade is therefore mostly a plain-rootfs concern, where `--rootfs` can point +at a directory whose group the user is not in. + +Before launch, `run` writes runtime `/etc/resolv.conf`, `/etc/hosts`, and +`/etc/hostname` into the run rootfs so DNS, localhost, and hostname lookups +work without network namespacing. The writes go through `os.OpenRoot` and +replace any existing entry, so an image that ships one of these names as a +symlink (including a symlinked `/etc`) cannot redirect the write outside the +rootfs. + +`run` launches `elfuse --sysroot ` with the host-literal fallback in +place: an absolute guest path absent under the rootfs resolves to the literal +host path. The rootfs is a root, not a boundary; elfuse favors transparency +over isolation, so an image-launched guest keeps the same filesystem view a +plain positional `elfuse ` run has (a development workflow that +deliberately reaches host resources such as `/tmp`). The practical caveat is +the guest `PATH` search: with an image `Env` `PATH` putting `/usr/bin` before +`/bin`, a bare `gzip` in an image that ships it only at `/bin/gzip` resolves to +the host's incompatible `/usr/bin/gzip` (a macOS Mach-O) first. Prefer +usr-merged images, invoke by absolute path, or reorder the search with +`--env PATH=...`. When the merged environment carries no `PATH`, `run` appends +Docker's conventional default so the guest always has a search path. + +## Runtime Configuration + +`elfuse-oci` resolves the image configuration before calling `elfuse`. + +Command resolution follows Docker-style rules: + +- `--entrypoint` replaces the image Entrypoint and drops the image Cmd; +- without `--entrypoint`, CLI arguments after the reference replace the image + Cmd while preserving the image Entrypoint; +- with neither override, image Entrypoint and Cmd are concatenated; +- an empty final command is an error. + +Environment resolution starts from image `Env`, or from an empty environment +under `--clear-env`. Each `--env KEY=VALUE` sets or replaces a value; a bare +`--env KEY` imports `KEY` from the host when present. + +User resolution accepts numeric `UID[:GID]` and symbolic `name[:group]`. +Symbolic names resolve against the unpacked rootfs `/etc/passwd` and +`/etc/group`, opened through `os.OpenRoot` so a symlinked account file cannot +redirect resolution to host data. Working directories must be guest-absolute. + +## Lifecycle + +The lifecycle commands operate on the local store: + +- `list` reads `refs.json` and the pinned image metadata; +- `rmi` removes a ref (or a unique SHA-256 digest prefix from `list`), + garbage-collects blobs no remaining manifest reaches, and reclaims the + image's unpacked cache when its last ref goes away; +- `prune` garbage-collects unreachable blobs without removing a named ref; + `prune --cache` also removes unpacked rootfs caches. + +Garbage collection is reachability-based: shared manifests, configs, and layers +stay on disk while any remaining ref reaches them. + +On macOS, cache cleanup also handles sparsebundle state, detaching stale mounts +and reaping per-run clone directories left by killed runs, with two safety +rules. A still-pinned digest's bundle is untouched by a plain `prune --cache` +(recovery of a crashed pinned bundle happens on the next `run`, or via `--all`), +and a volume a live run still uses is never force-detached. Liveness is decided +by a per-digest advisory lock (`run.lock`, held shared by every live run for +its whole lifetime), not by inspecting process ids, which avoids the pid-reuse +hazard. A sweep acquires that lock exclusively; while it cannot, the bundle is +left in place, and once it can, every clone is abandoned by construction and is +reaped except those a `run --keep` retained (a `.elfuse-keep` file) and any +leftover unpack staging directory. `rmi` reclaims a cache the same way: a plain +`rmi` drops the removed image's cache (derived state that goes with the image), +but refuses while a live run uses the volume (not even `--force` overrides) and +refuses without `--force` when the cache holds `run --keep` output. + +### Concurrency + +elfuse-oci is a single-user CLI, not a daemon. Store metadata stays +consistent under concurrency: pin and index updates, `rmi`, `prune`'s GC, and +`list`'s snapshot all serialize on an exclusive store file lock, so parallel +pulls and lifecycle commands cannot corrupt `refs.json`, `index.json`, or +reachability accounting. + +Per-digest sparsebundle state uses two advisory locks kept beside the bundle +(outside the mounted volume, so a detach cannot revoke them): an `attach.lock` +serializing lifecycle transitions, and the shared `run.lock` above. Concurrent +runs of the same image share one attach: the first provisions and attaches, +later runs join under a shared `run.lock`, and the volume detaches only when the +last exits. A `prune`/`rmi` sweep takes both locks non-blocking, so it either +wins (no run is live) or skips the busy bundle. A cold run unpacks into a +temporary directory and atomically renames it into place, so a concurrent probe +never sees a half-written rootfs and an interleaved sweep cannot delete one +mid-unpack. + +The plain digest-keyed rootfs cache (`rootfs//`, the default on +non-Darwin and on the `--plain-rootfs` path) follows the same liveness rule +with a single sibling `.lock`, held shared by a run from before the +cache-existence probe until the guest exits. The lock sits beside the directory +because the directory is the guest's `/` and its existence is the +unpack-complete signal. The plain run path execs `elfuse` in place, so the lock +descriptor is made exec-survivable and rides into the elfuse process: the kernel +releases the flock exactly when the guest exits, `SIGKILL` included. An explicit +`--rootfs` directory is user-managed and outside this scheme. + +## Validation + +The on-disk store is the contract: its layout is checked for spec-conformance +and cross-tool readability (crane/skopeo/umoci), and the pipeline is exercised +offline on Linux and end-to-end on macOS/HVF. `elfuse-oci` builds and +unit-tests without Hypervisor.framework; only an actual `run` guest boot needs +it. The concrete targets, env-var gates, and the Linux/macOS CI split live in +[testing.md](testing.md#oci-image-cli). diff --git a/docs/testing.md b/docs/testing.md index c42e1b72..9c0020d2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -15,6 +15,7 @@ Host build requirements: - GNU `make` - GNU `objcopy` or `llvm-objcopy` - GNU coreutils +- Go, for the elfuse-oci OCI CLI - `bash` 3.2+ (the version Apple ships as `/bin/bash`) is sufficient for the test harness; no Homebrew `bash` is required. See `tests/lib/bash-compat.sh` for the cross-version shims (a portable @@ -32,8 +33,35 @@ Guest test builds additionally require: - An AArch64 Linux cross-compiler for C test programs - An AArch64 bare-metal toolchain for the assembly smoke test -The toolchain defaults are defined in `mk/toolchain.mk`. -These variables are intended to be overridden when needed: +OCI interop checks additionally require `jq`. `crane`, `skopeo`, and `umoci` +are used when available by `make oci-interop`; CI installs them so cross-tool +layout parsing and registry-truth checks are hard gates there. + +`elfuse-oci` uses the Go `go-containerregistry` library directly for pulling +images, selecting platforms, and maintaining the OCI image-layout store. The +`crane` CLI is only used here as an independent registry/layout checker, while +`umoci` is used to verify that another OCI implementation can parse the store; +neither tool is on the normal `elfuse-oci run` execution path. + +For a full local `make oci-interop` run on macOS: + +```sh +brew install jq skopeo umoci +go install github.com/google/go-containerregistry/cmd/crane@v0.21.7 +export PATH="$(go env GOPATH)/bin:$PATH" +``` + +For a full local run on Ubuntu: + +```sh +sudo apt-get install -y jq skopeo +go install github.com/google/go-containerregistry/cmd/crane@v0.21.7 +go install github.com/opencontainers/umoci/cmd/umoci@latest +export PATH="$(go env GOPATH)/bin:$PATH" +``` + +The toolchain defaults are defined in `mk/toolchain.mk`, but these variables +are intended to be overridden when needed: - `CROSS_COMPILE` - `BAREMETAL_CROSS` @@ -79,6 +107,10 @@ make check make test-rosetta-all make test-gdbstub make test-matrix +make elfuse-oci +make oci-test +make oci-lint +make oci-interop make lint make clean ``` @@ -118,6 +150,13 @@ What they do: - `make test-gdbstub`: debugger integration checks against the built-in GDB stub - `make test-matrix`: cross-check `elfuse` (aarch64), QEMU (aarch64), and `elfuse` (x86_64-via-Rosetta) on overlapping corpora +- `make elfuse-oci`: build the Go OCI image CLI +- `make oci-test`: run offline elfuse-oci tests +- `make oci-lint`: `gofmt` check plus `go vet` for elfuse-oci (vet runs for both + `GOOS` values, so the darwin sparsebundle files are checked from Linux and the + non-darwin stubs from macOS) +- `make oci-interop`: pull OCI fixtures, check the raw image-layout files with + `jq`, and ask available external tools to read the same store - `make lint`: static analysis through `clang-tidy` ## Quick Iteration @@ -148,6 +187,72 @@ or run all matrix modes back-to-back with `make test-matrix`. green `make check` covers BusyBox validation. Use `make test-busybox` to iterate on a single applet failure without rerunning the unit suite. +### OCI Image CLI + +The OCI image CLI (`build/elfuse-oci`) is pure Go, so most of it builds and tests +without Hypervisor.framework; only an actual `elfuse-oci run` guest boot needs +HVF. For elfuse-oci changes: + +```sh +make elfuse-oci # build +make oci-test # offline unit + conformance tests +make oci-lint # gofmt check + go vet (both GOOS values) +ELFUSE_OCI_NETTEST=1 make oci-test # add the registry pull round-trip +make oci-interop # image-layout + cross-tool interop (needs jq) +``` + +`make oci-interop` always requires `jq`; install `crane`, `skopeo`, and `umoci` +too when you want the local run to match the CI cross-tool gate. The Darwin +sparsebundle round-trip (real `hdiutil` + case-sensitive APFS) is gated behind an +env var and runs only on macOS: + +```sh +ELFUSE_OCI_DARWIN_CS=1 go test -run TestDarwinCSSweep ./cmd/elfuse-oci/ +``` + +CI splits this coverage by what each runner can do: + +- **Linux (hosted)**: build, `gofmt`/`go vet` (including a `GOOS=darwin` + cross-vet of the sparsebundle files), the pull/inspect/unpack/list/rmi/prune + lifecycle, the `ELFUSE_OCI_NETTEST` round-trip, and crane/skopeo/umoci interop. + The unit suite runs with `-race` here. +- **macOS (hosted)**: build plus the full `go test -race` on darwin (exercising + the sparsebundle/clone code the Linux job can only cross-vet). The darwin + concurrency-sensitive code (cache sweep, provision, clone, cache-removal lock + discipline) compiles only on darwin, so this is the only place the race + detector ever sees it; keep `-race` on this step. Plus the real + `ELFUSE_OCI_DARWIN_CS` sparsebundle round-trip, and a run-less + pull/inspect/list/rmi/prune lifecycle smoke through the darwin binary. + No HVF needed. + +### Writing OCI unit-test fixtures + +An image is untrusted input: its layers control file *names and shapes*, not +just contents. When testing unpack, cache, or sweep code, build fixtures that +adversarially exercise those, not only escape-by-content: + +- **Reserved marker names.** An image can ship a path that collides with an + elfuse-managed marker (e.g. a layer with `/.elfuse-keep`). Assert the marker + is ignored when it comes from image content and honored only when + elfuse-oci wrote it out of band (see + `TestListSweepableClonesReapsImageShippedKeepFile`). +- **Implicit directories.** A layer may create `dir/sub/file` with no `dir/sub` + tar entry. Whiteout/opaque logic must treat that implicit parent as + current-layer content (see `TestUnpackOpaqueAfterImplicitParentChildren`). +- **Platform variants.** Set `cfg.Variant` (e.g. `linux/arm/v7`) so `inspect` + and `list` are checked to agree, not just os/arch. +- **Concurrency.** Use the package-level function-var seams (`afterImageResolve`, + `cranePull`, `isMountPointFn`, ...) to stage a racing pull/rmi in the exact + TOCTOU window rather than only single-command paths, and hold the store or + cache flocks directly to assert a busy path refuses. +- **macOS + HVF (self-hosted)**: end-to-end `elfuse-oci run` guest boots: + the alpine:3 default-entrypoint smoke that proves pull → sparsebundle → + COW clone → HVF launch → exit-code propagation, and a full + pull → inspect → list → run → rmi → prune lifecycle that runs + python:3.12-slim with an `--entrypoint` override and asserts both teardowns + (a plain rmi reclaiming the cold cache with the image, then a `run --keep` + cache that rmi refuses without `--force` and `--force` detaches and drops). + ## Test Matrix The matrix driver lives in `tests/test-matrix.sh`. It currently covers three @@ -358,6 +463,7 @@ Suggested minimum validation: | Change area | Recommended validation | |-------------|------------------------| | CLI, logging, docs-only build rules | `make elfuse` | +| OCI lifecycle or store behavior | `make oci-lint && make oci-test && make oci-interop` | | General syscall or runtime logic | `make elfuse && make check && make test-matrix-elfuse-aarch64` | | `/proc`, `/dev`, path, or BusyBox-sensitive behavior | `make elfuse && make check && make test-matrix-elfuse-aarch64` | | Rosetta hosting, x86_64 dispatch, VZ ioctls, AOT cache | `make elfuse && make test-rosetta-all` | diff --git a/docs/usage.md b/docs/usage.md index bef8aca2..51e4eba4 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -212,3 +212,183 @@ That has a few direct implications: work entirely inside the VM. Programs that link against `libfuse` (sshfs, ntfs-3g, AppImage runtimes) run without macFUSE, FUSE-T, or FSKit on the host. + +## OCI Images + +elfuse can run programs from OCI images. All image work is handled by +`build/elfuse-oci` (Go); execution still goes through the normal +`build/elfuse --sysroot` runtime, so `elfuse` itself has no OCI commands. + +```sh +build/elfuse-oci [flags] +``` + +This consumes the OCI image format for distribution; it is not a container +runtime. There are no namespaces, cgroups, port mapping, daemon, `docker exec`, +or image build/push, and the rootfs is the guest's root, not an isolation +boundary. See +[oci-design.md](oci-design.md#scope-and-limitations) for the exact list of what +is and is not implemented, and for the design model. + +### Store And Platform + +`elfuse-oci` stores images in an OCI image-layout directory. The default +store is `$ELFUSE_OCI_STORE` when set, otherwise `~/.local/share/elfuse/oci`. +Use `--store DIR` on any subcommand to override it. + +Pulls default to `linux/arm64`. Use `--platform os/arch[/variant]` to select +another platform, such as `linux/amd64` for a Rosetta-backed guest: + +```sh +build/elfuse-oci pull --platform linux/amd64 alpine:3 +``` + +### Commands + +```sh +build/elfuse-oci pull [--store DIR] [--platform os/arch[/variant]] +``` + +Pull `` into the local store and pin it by its original reference. + +```sh +build/elfuse-oci inspect [--store DIR] [--json] +``` + +Print the stored image's manifest and config summary. `--json` prints the raw +config JSON. + +```sh +build/elfuse-oci unpack [--store DIR] [--rootfs DIR] +``` + +Unpack the stored image's layers into a rootfs directory. Without `--rootfs`, +the store's digest-keyed rootfs cache is used. + +```sh +build/elfuse-oci run [flags] [args...] +``` + +Run an image. If `` is not present, `run` pulls it first; if the rootfs +cache is missing, it unpacks it first. + +```sh +build/elfuse-oci list [--store DIR] [--json] # alias: images +``` + +List pinned refs, their manifest digests, platform, compressed layer size, and +layer count. + +```sh +build/elfuse-oci rmi [--store DIR] [--force] +``` + +Remove a ref, or a unique SHA-256 digest prefix from `list`, and +garbage-collect blobs no remaining ref reaches. Removing the last ref for an +image also reclaims its unpacked cache. `rmi` refuses while a live run still +uses the cache (never overridable), and refuses without `--force` when the +cache holds `run --keep` output. + +```sh +build/elfuse-oci prune [--store DIR] [--cache] [--all] [--dry-run] +``` + +Garbage-collect unreachable blobs. `--cache` also removes unpacked rootfs +caches; with `--cache`, `--all` removes them even for still-pulled refs. +`--dry-run` reports what would be removed. + +### Running Images + +```sh +make elfuse elfuse-oci + +build/elfuse-oci run alpine:3 /bin/sh -c 'echo hello' +``` + +`run` accepts the common flags plus these run-specific flags: + +| Flag | Meaning | +|------|---------| +| `--entrypoint PATH` | Replace the image Entrypoint. The image Cmd is dropped. | +| `--env KEY=VALUE` | Set or replace a guest environment variable. Repeatable. | +| `--env KEY` | Import `KEY` from the host environment when present. | +| `--clear-env` | Start from an empty environment instead of image `Env`. | +| `--user UID[:GID]` | Run as a numeric user and optional group. | +| `--user name[:group]` | Resolve names through rootfs `/etc/passwd` and `/etc/group`. | +| `--workdir DIR` | Set the initial guest working directory. Must be absolute. | +| `--rootfs DIR` | Use an explicit rootfs directory. | +| `--plain-rootfs` | Use a plain directory cache instead of the macOS sparsebundle. | +| `--sparse-size SIZE` | Set the sparsebundle virtual size. Default is `16g`. | +| `--no-clone` | Run against the cached base rootfs directly. Mutations persist. | +| `--keep` | Keep the per-run clone and sparsebundle mount for inspection. | + +The command vector follows Docker-style rules: with no CLI args, the image +Entrypoint plus Cmd is used; CLI args after `` replace image Cmd but keep +image Entrypoint; `--entrypoint` replaces image Entrypoint and discards Cmd; an +empty final command is an error. + +Environment resolution starts from image `Env` (or an empty environment under +`--clear-env`); each `--env KEY=VALUE` sets or appends, and a bare `--env KEY` +imports the host value when set. `--user` defaults to image `User`, then root; +symbolic users and groups are resolved after the rootfs exists, and a name that +fails to resolve is an error. + +### macOS Rootfs Behavior + +By default, `run` uses a case-sensitive APFS sparsebundle per image digest so +the guest's case-sensitive filenames do not collide on a case-insensitive host +volume. The unpacked image lives as a cached base tree inside the sparsebundle, +and each run gets an APFS copy-on-write clone of it, so guest writes do not +mutate the cached rootfs. The clone is removed and the sparsebundle detached +when the guest exits; `--keep` leaves both for inspection, and `prune --cache` +reaps stale caches (including clones left by killed runs) later. + +Use `--plain-rootfs` or `--rootfs DIR` when you explicitly want the regular +directory path. + +### Runtime Files + +Before launching the guest, `run` writes these files into the run rootfs: + +| File | Source | +|------|--------| +| `/etc/resolv.conf` | Host resolver config, with a fallback nameserver if needed. | +| `/etc/hosts` | localhost plus the host name. | +| `/etc/hostname` | Host name. | + +The `resolv.conf` fallback (used only when the host's own config is unreadable +or empty) is Google's public `8.8.8.8`; on hosts using private or split-horizon +DNS, fix the host `/etc/resolv.conf` if guest lookups must stay on the local +resolver. The C runtime also serves synthetic `/proc` and selected `/dev` +entries to every guest, image-launched or not. + +### Host Fallback And PATH + +The image rootfs is a root, not a boundary: an absolute guest path absent from +the rootfs falls back to the literal host path, the same mechanism that lets a +plain positional `elfuse ` run reach host resources such as `/tmp`. + +The visible consequence is the guest `PATH` search. With an image `PATH` +searching `/usr/bin` before `/bin`, a bare `gzip` in an image that ships it only +at `/bin/gzip` resolves to the host's incompatible `/usr/bin/gzip` (a macOS +Mach-O) first. Prefer usr-merged images, invoke by absolute path, or reorder the +search with `--env PATH=...`. When neither the image config nor `--env` provides +a `PATH`, `run` appends Docker's conventional default +(`/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`). See +[oci-design.md](oci-design.md#run-paths) for the rationale. + +### Lifecycle + +The store keeps image blobs separately from unpacked rootfs caches, and garbage +collection is reachability-based: removing a ref never deletes blobs another ref +still reaches, and removing one of several refs to the same digest keeps the +shared cache. A normal cleanup sequence: + +```sh +build/elfuse-oci list +build/elfuse-oci rmi alpine:3 +build/elfuse-oci prune --cache --dry-run +build/elfuse-oci prune --cache +``` + +See [oci-design.md](oci-design.md#lifecycle) for the GC and cache-safety model. From 3dde138cfef9ce99bb3635efa8e38ba3d4bab7d4 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Sat, 18 Jul 2026 20:55:16 +0200 Subject: [PATCH 10/10] Add per-image real-workload CI for elfuse-oci Issue #224 profiled five real images (python:3.12-slim, node:22-alpine, golang:1.23-alpine, eclipse-temurin:21, gcc:14). Add one CI job per image that boots the image under HVF via `elfuse-oci run` and drives that image's operations, so a change that breaks any of them is caught on the PR rather than by hand. A shared driver, scripts/ci/oci-workload.sh , maps each key to its image and guest workload under scripts/ci/workloads/ and asserts a per-image sentinel token: - python: a single-threaded SQLite insert plus an aggregate query, a small file write/read/checksum fan-out, and a JSON round-trip. - node: in-guest compute (fs/crypto/zlib/JSON) plus an HTTP server the job reaches over the host loopback; elfuse maps guest sockets to host sockets and does no network-namespace isolation, so a guest bound to 127.0.0.1 is reachable host-side. - go: go version and a gofmt of a canonical file, with GOMAXPROCS pinned low. - jvm: javac + java exercising collections, file I/O, SHA-256, an 8-thread pool, and a subprocess. - c: a small multi-file make project plus a heavier single translation unit compiled with gcc -O1. The go and python lanes are kept deliberately light so they stay within the runtime's current limits; heavier variants that stress those limits are submitted separately. The jobs run only on self-hosted Apple Silicon because `run` needs Hypervisor.framework. They share a composite action that fetches the elfuse binary from build-macos and builds elfuse-oci, and each keeps a warm per-key store on the runner's persistent disk so only the first run pulls over the network. gcc:14 and eclipse-temurin:21 ship the shadow suite, so those jobs also exercise the unpack setuid/setgid degrade end to end. The existing python workload moves under scripts/ci/workloads/; oci-lifecycle.sh follows the new path. --- .github/actions/hvf-elfuse-setup/action.yml | 66 +++++++++ .github/workflows/main.yml | 143 ++++++++++++++++++ docs/testing.md | 29 ++++ scripts/ci/oci-lifecycle.sh | 2 +- scripts/ci/oci-python-workload.py | 117 --------------- scripts/ci/oci-workload.sh | 154 ++++++++++++++++++++ scripts/ci/workloads/c-workload.sh | 70 +++++++++ scripts/ci/workloads/go-workload.sh | 32 ++++ scripts/ci/workloads/jvm-workload.sh | 83 +++++++++++ scripts/ci/workloads/node-compute.js | 50 +++++++ scripts/ci/workloads/node-server.js | 34 +++++ scripts/ci/workloads/python-workload.py | 78 ++++++++++ 12 files changed, 740 insertions(+), 118 deletions(-) create mode 100644 .github/actions/hvf-elfuse-setup/action.yml delete mode 100644 scripts/ci/oci-python-workload.py create mode 100755 scripts/ci/oci-workload.sh create mode 100644 scripts/ci/workloads/c-workload.sh create mode 100644 scripts/ci/workloads/go-workload.sh create mode 100644 scripts/ci/workloads/jvm-workload.sh create mode 100644 scripts/ci/workloads/node-compute.js create mode 100644 scripts/ci/workloads/node-server.js create mode 100644 scripts/ci/workloads/python-workload.py diff --git a/.github/actions/hvf-elfuse-setup/action.yml b/.github/actions/hvf-elfuse-setup/action.yml new file mode 100644 index 00000000..f2e8065d --- /dev/null +++ b/.github/actions/hvf-elfuse-setup/action.yml @@ -0,0 +1,66 @@ +name: HVF elfuse setup +description: > + Shared setup for the self-hosted HVF workload jobs: fail fast if the run is + superseded by a newer PR commit, then fetch the prebuilt elfuse binary from + the build-macos job and build the pure-Go elfuse-oci CLI. Assumes the repo is + already checked out. + +runs: + using: composite + steps: + # Fail fast if this run targets a commit that is no longer the PR's HEAD. + # cancel-in-progress covers a newer push, but not a manual "Re-run jobs" on + # an old run, which would burn the self-hosted runner re-testing stale code. + # Mirrors the runtime-macos guard; fails (not cancels) because repo policy + # caps the token at actions: read. The lookup fails open. + - name: Fail fast if superseded by a newer PR commit + if: github.event_name == 'pull_request' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -uo pipefail + latest=$(curl -fsSL \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$REPO/pulls/$PR_NUMBER" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["head"]["sha"])') \ + || latest="" + echo "Run targets : $RUN_SHA" + echo "PR HEAD now : ${latest:-}" + if [ -n "$latest" ] && [ "$latest" != "$RUN_SHA" ]; then + echo "::error::This run targets $RUN_SHA, but PR #$PR_NUMBER HEAD is now $latest -- the commit is no longer the latest. Failing instead of re-testing stale code on the self-hosted runner; re-run CI on the current commit." + exit 1 + fi + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + # Reuse the arm64 elfuse binary built + entitlement-checked by build-macos + # instead of rebuilding the C project on the self-hosted runner five times. + # Mach-O code signatures (and their embedded HVF entitlement) travel inside + # the binary, so they survive the artifact zip round-trip; only the execute + # bit is lost and restored here. + - name: Download prebuilt elfuse binary + uses: actions/download-artifact@v7 + with: + name: elfuse-${{ runner.os }}-${{ runner.arch }} + path: build + + - name: Restore execute bit + verify HVF entitlement + shell: bash + run: | + set -euo pipefail + chmod +x build/elfuse + codesign -d --entitlements - build/elfuse 2>&1 \ + | grep -q 'com\.apple\.security\.hypervisor' + + - name: Build elfuse-oci + shell: bash + run: make build/elfuse-oci diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3d12ff9a..f529bcdd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,6 +13,9 @@ # (crane/skopeo/umoci) on Linux # oci-macos : elfuse-oci darwin build, unit tests, sparsebundle # round-trip, and a run-less image-lifecycle smoke +# workload-* : per-image real-workload smokes (python/node/go/jvm/c) that +# boot each image under HVF on self-hosted Apple Silicon and +# drive its characteristic operations (issue #224) # # Runtime and sanitizer tests require Hypervisor.framework, which # GitHub-hosted macOS runners do not expose. Those tests run on self-hosted @@ -793,3 +796,143 @@ jobs: # that the Linux job's cache-reclaiming flow does not cover. jq # ships on the macos-15 image. run: scripts/ci/oci-cli-smoke.sh + + # Per-image real-workload jobs (issue #224). Each boots a real image under HVF + # via `elfuse-oci run` and drives that image's characteristic operations so a + # code change that breaks any of them is caught on the PR. One job per image; + # all share .github/actions/hvf-elfuse-setup, which fetches the elfuse binary + # from build-macos and builds elfuse-oci. `run` needs Hypervisor.framework, so + # these are self-hosted only. Each keeps a warm per-key store on the runner's + # persistent disk so only the first run pulls over the network. gcc:14 and + # eclipse-temurin:21 ship the shadow suite, so they also exercise the unpack + # setuid/setgid degrade end to end. + workload-python: + name: Workload (Python) + needs: build-macos + if: > + github.repository == 'sysprog21/elfuse' && + (github.event_name == 'push' || github.event_name == 'pull_request') + runs-on: [self-hosted, macOS, arm64] + timeout-minutes: 30 + permissions: + contents: read + pull-requests: read + concurrency: + group: workload-python-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: HVF elfuse setup + uses: ./.github/actions/hvf-elfuse-setup + - name: Run python workload + run: | + set -euo pipefail + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-workload-python" + scripts/ci/oci-workload.sh python + + workload-node: + name: Workload (Node) + needs: build-macos + if: > + github.repository == 'sysprog21/elfuse' && + (github.event_name == 'push' || github.event_name == 'pull_request') + runs-on: [self-hosted, macOS, arm64] + timeout-minutes: 30 + permissions: + contents: read + pull-requests: read + concurrency: + group: workload-node-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: HVF elfuse setup + uses: ./.github/actions/hvf-elfuse-setup + - name: Run node workload + # Includes the HTTP-server phase: the guest binds 127.0.0.1 (elfuse maps + # sockets to host sockets, no netns), and this step curls it host-side. + run: | + set -euo pipefail + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-workload-node" + scripts/ci/oci-workload.sh node + + workload-go: + name: Workload (Go) + needs: build-macos + if: > + github.repository == 'sysprog21/elfuse' && + (github.event_name == 'push' || github.event_name == 'pull_request') + runs-on: [self-hosted, macOS, arm64] + timeout-minutes: 30 + permissions: + contents: read + pull-requests: read + concurrency: + group: workload-go-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: HVF elfuse setup + uses: ./.github/actions/hvf-elfuse-setup + - name: Run go workload + run: | + set -euo pipefail + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-workload-go" + scripts/ci/oci-workload.sh go + + workload-jvm: + name: Workload (JVM) + needs: build-macos + if: > + github.repository == 'sysprog21/elfuse' && + (github.event_name == 'push' || github.event_name == 'pull_request') + runs-on: [self-hosted, macOS, arm64] + # javac + java startup and the compile step run slower than the lighter + # images, and eclipse-temurin is a large first pull. + timeout-minutes: 45 + permissions: + contents: read + pull-requests: read + concurrency: + group: workload-jvm-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: HVF elfuse setup + uses: ./.github/actions/hvf-elfuse-setup + - name: Run jvm workload + run: | + set -euo pipefail + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-workload-jvm" + scripts/ci/oci-workload.sh jvm + + workload-c: + name: Workload (C) + needs: build-macos + if: > + github.repository == 'sysprog21/elfuse' && + (github.event_name == 'push' || github.event_name == 'pull_request') + runs-on: [self-hosted, macOS, arm64] + # gcc:14 is the largest first pull and the compile bursts (make + a heavier + # single TU) dominate the wall time. + timeout-minutes: 45 + permissions: + contents: read + pull-requests: read + concurrency: + group: workload-c-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: HVF elfuse setup + uses: ./.github/actions/hvf-elfuse-setup + - name: Run c workload + run: | + set -euo pipefail + export ELFUSE_OCI_STORE="$HOME/.cache/elfuse-ci/oci-workload-c" + scripts/ci/oci-workload.sh c diff --git a/docs/testing.md b/docs/testing.md index 9c0020d2..9363938a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -224,6 +224,35 @@ CI splits this coverage by what each runner can do: `ELFUSE_OCI_DARWIN_CS` sparsebundle round-trip, and a run-less pull/inspect/list/rmi/prune lifecycle smoke through the darwin binary. No HVF needed. +- **macOS (self-hosted, HVF)**: the only place `elfuse-oci run` actually boots a + guest. Besides the run smoke and image lifecycle, a per-image workload suite + (`workload-python|node|go|jvm|c`) drives each image from issue #224 through its + characteristic operations, one job per image. The shared driver is + `scripts/ci/oci-workload.sh `; each image's guest workload lives in + `scripts/ci/workloads/`: + + - **python** (`python:3.12-slim`): concurrent SQLite writers, a 400-file + write/read/checksum fan-out, JSON/regex churn, an `os.walk` of the stdlib, + and interpreter subprocesses. + - **node** (`node:22-alpine`): in-guest compute (fs, crypto, zlib, JSON) plus + an HTTP server the job reaches over the host loopback (elfuse maps guest + sockets to host sockets and does no netns isolation), curling it repeatedly + before `/quit`. + - **go** (`golang:1.23-alpine`): `go version`, `gofmt` over stdlib trees, and + a build+run of a small module. + - **jvm** (`eclipse-temurin:21`): `javac` + `java` exercising collections, + file I/O, SHA-256, an 8-thread pool, and a subprocess. + - **c** (`gcc:14`): a small multi-file `make` project plus a heavier single + translation unit compiled with `gcc -O1`. + + `gcc:14` and `eclipse-temurin:21` are Debian/Ubuntu-based and ship the shadow + suite, so these jobs also exercise the unpack setuid/setgid degrade end to + end. Each job keeps a warm per-key store on the runner's persistent disk, so + only the first run pulls over the network. Run one locally with: + + ```sh + ELFUSE_OCI_STORE=/tmp/elfuse-oci-store scripts/ci/oci-workload.sh c + ``` ### Writing OCI unit-test fixtures diff --git a/scripts/ci/oci-lifecycle.sh b/scripts/ci/oci-lifecycle.sh index 29319819..eee11f29 100755 --- a/scripts/ci/oci-lifecycle.sh +++ b/scripts/ci/oci-lifecycle.sh @@ -74,7 +74,7 @@ run_workloads() { # linked glibc guest than the one-liner above; prints a single sentinel # token only on full success. out="$("$BIN" run --entrypoint /usr/local/bin/python3 "$IMG" \ - -c "$(cat "$OCI_CI_DIR/oci-python-workload.py")")" + -c "$(cat "$OCI_CI_DIR/workloads/python-workload.py")")" printf 'python workload said: %s\n' "$out" [ "$out" = elfuse-oci-python-workload-ok ] || fail "python workload said '$out'" } diff --git a/scripts/ci/oci-python-workload.py b/scripts/ci/oci-python-workload.py deleted file mode 100644 index 6c1c044d..00000000 --- a/scripts/ci/oci-python-workload.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -"""Non-trivial guest workload for the elfuse-oci OCI lifecycle CI. - -Exercises more of a real dynamically-linked glibc guest than a one-line print: -concurrent SQLite writers (fcntl locking, fsync, and, when the guest FS -supports it, WAL mmap), plus a filesystem fan-out whose written content is -read back and checksummed. On full success it prints a single sentinel token -the workflow asserts on; any failure exits non-zero with a diagnostic. - -Self-contained (stdlib only) so it runs under the image's bundled Python with -no network or extra packages, and is passed to the guest via `python3 -c`. -""" - -import hashlib -import os -import sqlite3 -import sys -import threading - -DB = "/tmp/elfuse-workload.db" -TREE = "/tmp/elfuse-workload-tree" -THREADS = 8 -PER_THREAD = 1000 -FANOUT = 8 - - -def setup_db(): - con = sqlite3.connect(DB) - try: - # WAL exercises the guest FS's shared-memory index (mmap) and is the - # more demanding path; if the FS cannot back it, SQLite reports a - # different mode and the concurrent-writer count check below still - # validates fcntl locking and durable commits under rollback journal. - con.execute("PRAGMA journal_mode=WAL") - con.execute( - "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, " - "tid INTEGER NOT NULL, n INTEGER NOT NULL)" - ) - con.commit() - finally: - con.close() - - -def worker(tid): - con = sqlite3.connect(DB, timeout=60) - try: - con.execute("PRAGMA busy_timeout=60000") - for n in range(PER_THREAD): - con.execute("INSERT INTO t (tid, n) VALUES (?, ?)", (tid, n)) - con.commit() - finally: - con.close() - - -def db_count(): - con = sqlite3.connect(DB, timeout=60) - try: - (count,) = con.execute("SELECT COUNT(*) FROM t").fetchone() - return count - finally: - con.close() - - -def content(i, j): - return "elfuse-workload-%d-%d\n" % (i, j) - - -def fs_fanout_ok(): - for i in range(FANOUT): - for j in range(FANOUT): - d = os.path.join(TREE, str(i), str(j)) - os.makedirs(d, exist_ok=True) - with open(os.path.join(d, "f"), "w") as fh: - fh.write(content(i, j)) - fh.flush() - os.fsync(fh.fileno()) - read_back = [] - for i in range(FANOUT): - for j in range(FANOUT): - with open(os.path.join(TREE, str(i), str(j), "f")) as fh: - read_back.append(fh.read()) - expected = [content(i, j) for i in range(FANOUT) for j in range(FANOUT)] - got = hashlib.sha256() - for p in sorted(read_back): - got.update(p.encode()) - want = hashlib.sha256() - for p in sorted(expected): - want.update(p.encode()) - return got.hexdigest() == want.hexdigest() - - -def main(): - setup_db() - threads = [threading.Thread(target=worker, args=(i,)) for i in range(THREADS)] - for t in threads: - t.start() - for t in threads: - t.join() - - count = db_count() - if count != THREADS * PER_THREAD: - print( - "sqlite row count %d != %d (concurrent writers lost rows)" - % (count, THREADS * PER_THREAD), - file=sys.stderr, - ) - sys.exit(1) - - if not fs_fanout_ok(): - print("filesystem fan-out checksum mismatch", file=sys.stderr) - sys.exit(1) - - print("elfuse-oci-python-workload-ok") - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/oci-workload.sh b/scripts/ci/oci-workload.sh new file mode 100755 index 00000000..c17e5fda --- /dev/null +++ b/scripts/ci/oci-workload.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Per-image real-workload smoke: drive each image's characteristic operations +# (issue #224) through `elfuse-oci run` under HVF and assert a sentinel token. +# One key per image. `run` pulls on demand, so a warm persistent +# ELFUSE_OCI_STORE keeps reruns network-free. +# +# Usage: ELFUSE_OCI_STORE= scripts/ci/oci-workload.sh +# shellcheck source=scripts/ci/oci-lib.sh +. "$(dirname "$0")/oci-lib.sh" +require_bin +: "${ELFUSE_OCI_STORE:?set ELFUSE_OCI_STORE to the store directory to use}" +export ELFUSE_OCI_STORE + +key="${1:?usage: oci-workload.sh }" +WL="$OCI_CI_DIR/workloads" + +# assert_sentinel SENTINEL DESC OUTPUT: OUTPUT must contain the fixed SENTINEL. +assert_sentinel() { + printf '%s\n' "$3" | expect_grep "$1" \ + || fail "$2: output missing sentinel '$1'" +} + +# run_capture SENTINEL DESC RUN-ARGS...: run a single-shot guest workload, +# echo its output for the log, and assert the sentinel. Covers every image +# whose workload is one `run` invocation (all but node's two-phase server). +run_capture() { + local sentinel="$1" desc="$2" + shift 2 + local out + out="$("$BIN" run "$@")" + printf '%s\n' "$out" + assert_sentinel "$sentinel" "$desc" "$out" +} + +# Background guest bookkeeping for the node server phase; a failed assertion +# must not leak the guest (it would keep binding the host loopback port and +# poison the next run on a persistent self-hosted runner). +guest="" +node_outfile="" +on_exit() { + rc=$? + if [ -n "$guest" ] && kill -0 "$guest" 2>/dev/null; then + kill "$guest" 2>/dev/null || true + wait "$guest" 2>/dev/null || true + fi + [ -n "$node_outfile" ] && rm -f "$node_outfile" + exit "$rc" +} +trap on_exit EXIT + +run_node() { + # Phase A: in-guest compute. + run_capture elfuse-oci-node-compute-ok node-compute \ + --entrypoint /usr/local/bin/node node:22-alpine \ + -e "$(cat "$WL/node-compute.js")" + + # Phase B: HTTP server reached over the host loopback. elfuse forwards + # socket syscalls to host sockets and does no netns isolation, so a guest + # bound to 127.0.0.1 is reachable from the host. The server binds an + # ephemeral port and prints "PORT="; read it back rather than fixing a + # port that could collide with a leaked or concurrent guest. + local reqs="${WL_NODE_REQUESTS:-100}" + node_outfile="$(mktemp)" + "$BIN" run --entrypoint /usr/local/bin/node node:22-alpine \ + -e "$(cat "$WL/node-server.js")" >"$node_outfile" 2>&1 & + guest=$! + + # Wait for the server to announce its ephemeral port. Poll rather than + # wait_for so a guest that dies (a bind failure or a runtime crash) surfaces + # its own captured output instead of an opaque timeout. + local waited=0 port="" + while [ "$waited" -lt 120 ]; do + port="$(awk -F= '/^PORT=/{print $2; exit}' "$node_outfile")" + [ -n "$port" ] && break + if ! kill -0 "$guest" 2>/dev/null; then + cat "$node_outfile" >&2 + guest="" + fail "node server exited before announcing a port" + fi + sleep 0.5 + waited=$((waited + 1)) + done + if [ -z "$port" ]; then + cat "$node_outfile" >&2 + fail "node server did not announce a port within 60s" + fi + # The PORT= line is the guest flushing stdout, not proof the socket accepts + # connections yet; probe the port directly before hammering it. + wait_for 30 "node server on 127.0.0.1:$port" \ + curl -fsS -o /dev/null "http://127.0.0.1:$port/" + printf 'node server on 127.0.0.1:%s\n' "$port" + + local i=0 body + while [ "$i" -lt "$reqs" ]; do + # Guard the substitution: a bare body=$(curl ...) would trip the ERR + # trap on any transient failure instead of the specific diagnostic. + if ! body="$(curl -fsS "http://127.0.0.1:$port/")"; then + fail "node server request $i failed (curl)" + fi + [ "$body" = elfuse-node-server-ok ] \ + || fail "node server request $i returned '$body'" + i=$((i + 1)) + done + printf 'node server answered %d requests\n' "$reqs" + + # Clean shutdown: /quit makes the guest exit 0. The connection may reset as + # the guest exits, so tolerate the curl status. If /quit never reaches the + # server the guest would run forever, so bound the wait and kill on timeout + # rather than blocking to the job's timeout-minutes. + curl -fsS -o /dev/null "http://127.0.0.1:$port/quit" || true + local waited=0 + while kill -0 "$guest" 2>/dev/null && [ "$waited" -lt 20 ]; do + sleep 0.5 + waited=$((waited + 1)) + done + if kill -0 "$guest" 2>/dev/null; then + kill "$guest" 2>/dev/null || true + wait "$guest" 2>/dev/null || true + guest="" + fail "node server did not exit within 10s of /quit" + fi + if ! wait "$guest"; then + guest="" + fail "node server exited non-zero after /quit" + fi + guest="" +} + +case "$key" in + python) + run_capture elfuse-oci-python-workload-ok python \ + --entrypoint /usr/local/bin/python3 python:3.12-slim \ + -c "$(cat "$WL/python-workload.py")" + ;; + node) run_node ;; + go) + run_capture elfuse-oci-go-workload-ok go \ + golang:1.23-alpine /bin/sh -c "$(cat "$WL/go-workload.sh")" + ;; + jvm) + run_capture elfuse-oci-jvm-workload-ok jvm \ + eclipse-temurin:21 /bin/sh -c "$(cat "$WL/jvm-workload.sh")" + ;; + c) + run_capture elfuse-oci-c-workload-ok c \ + gcc:14 /bin/sh -c "$(cat "$WL/c-workload.sh")" + ;; + *) fail "unknown workload key: $key (want python|node|go|jvm|c)" ;; +esac + +# Keep a persistent store bounded: GC blobs stranded when a pinned tag moves. +"$BIN" prune >/dev/null + +echo "workload $key OK" diff --git a/scripts/ci/workloads/c-workload.sh b/scripts/ci/workloads/c-workload.sh new file mode 100644 index 00000000..ae02cbe0 --- /dev/null +++ b/scripts/ci/workloads/c-workload.sh @@ -0,0 +1,70 @@ +# shellcheck shell=sh +# C-compile image workload (issue #224 c-compile profile), run in the guest via +# `/bin/sh -c`: a small multi-file project built with make, then a larger single +# translation unit compiled with gcc -O1 (the readlinkat/execve-of-cc1/as/ld +# signature). Prints one sentinel token on success. POSIX sh (dash). gcc:14 is +# Debian-based, so this also proves the setuid/setgid unpack degrade on a +# shadow-suite image. +# +# The Makefile uses .RECIPEPREFIX so its recipes are prefixed with '>' rather +# than a literal tab, which keeps this heredoc robust. Scaled down from #224's +# 30-file project + 8 MB TU to keep CI minutes sane; the syscall signature +# (per-TU path resolution, compiler/assembler/linker exec) is preserved. +set -e + +d=/tmp/cwork +rm -rf "$d" +mkdir -p "$d" +cd "$d" + +cat > mathx.h <<'EOF' +#ifndef MATHX_H +#define MATHX_H +int tri(int n); +#endif +EOF +cat > mathx.c <<'EOF' +#include "mathx.h" +int tri(int n) { + int s = 0; + for (int i = 1; i <= n; i++) s += i; + return s; +} +EOF +cat > main.c <<'EOF' +#include +#include "mathx.h" +int main(void) { + printf("%d\n", tri(100)); + return 0; +} +EOF +cat > Makefile <<'EOF' +.RECIPEPREFIX = > +CC ?= gcc +app: main.o mathx.o +> $(CC) -O1 -o app main.o mathx.o +%.o: %.c mathx.h +> $(CC) -O1 -c -o $@ $< +EOF + +make -j1 +r=$(./app) +if [ "$r" != "5050" ]; then + echo "make project produced: $r" >&2 + exit 1 +fi + +# A larger single TU: 1000 functions dispatched through a table, summed and +# checked. Exercises a heavier gcc -O1 compile+link than the tiny project above. +awk 'BEGIN { + for (i = 0; i < 1000; i++) print "int f" i "(void){return " i ";}"; + printf "typedef int(*fn)(void);\nstatic fn t[]={"; + for (i = 0; i < 1000; i++) printf "f%d,", i; + print "};"; + print "int main(void){long s=0;for(int i=0;i<1000;i++)s+=t[i]();return s==499500?0:1;}"; +}' > big.c +gcc -O1 -o big big.c +./big + +echo elfuse-oci-c-workload-ok diff --git a/scripts/ci/workloads/go-workload.sh b/scripts/ci/workloads/go-workload.sh new file mode 100644 index 00000000..1e9f9161 --- /dev/null +++ b/scripts/ci/workloads/go-workload.sh @@ -0,0 +1,32 @@ +# shellcheck shell=sh +# Go image workload, run in the guest via `/bin/sh -c`. A deliberately light +# check that the go toolchain runs under elfuse: `go version` plus a `gofmt` of +# a tiny file. GOMAXPROCS is pinned low because a full `go build`/`go run` +# spawns one OS thread per blocked syscall and overruns elfuse's fixed thread +# table; the compile-and-run stress variant lives on the oci/workload-stress +# branch. Prints one sentinel token on success. POSIX sh (busybox ash). +set -e + +# Keep the go runtime's OS-thread count well under elfuse's thread table, and +# give it a writable cache/toolchain path (the guest gets no HOME). +export GOMAXPROCS=1 GOCACHE=/tmp/go-build GOTOOLCHAIN=local + +v=$(go version) +case "$v" in + *"go version go1."*) ;; + *) echo "unexpected go version: $v" >&2; exit 1 ;; +esac + +# gofmt (a single-threaded, compile-free tool) over a canonical file must report +# nothing to reformat; any output means gofmt itself misbehaved in the guest. +d=/tmp/gowork +rm -rf "$d" +mkdir -p "$d" +printf 'package main\n\nfunc main() {}\n' > "$d/main.go" +flagged=$(gofmt -l "$d") +if [ -n "$flagged" ]; then + echo "gofmt -l flagged a canonical file: $flagged" >&2 + exit 1 +fi + +echo elfuse-oci-go-workload-ok diff --git a/scripts/ci/workloads/jvm-workload.sh b/scripts/ci/workloads/jvm-workload.sh new file mode 100644 index 00000000..55786139 --- /dev/null +++ b/scripts/ci/workloads/jvm-workload.sh @@ -0,0 +1,83 @@ +# shellcheck shell=sh +# JVM image workload (issue #224 jvm profile), run in the guest via +# `/bin/sh -c`: javac-compile a small program, then run it exercising +# collections, file I/O, a SHA-256 digest, an 8-thread pool, and a subprocess +# (the futex/clock_gettime-heavy signature). The program prints the sentinel +# token itself on success. POSIX sh (dash). eclipse-temurin is Ubuntu-based, so +# this also proves the setuid/setgid unpack degrade on a shadow-suite image. +set -e + +d=/tmp/jvmwork +rm -rf "$d" +mkdir -p "$d" +cd "$d" +cat > Main.java <<'EOF' +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; +import java.util.concurrent.*; + +public class Main { + static String sha256(byte[] b) throws Exception { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] d = md.digest(b); + StringBuilder sb = new StringBuilder(); + for (byte x : d) sb.append(String.format("%02x", x)); + return sb.toString(); + } + + public static void main(String[] args) throws Exception { + // Collections. + Map m = new HashMap<>(); + for (int i = 0; i < 1000; i++) m.put(i, i * i); + long collSum = 0; + for (int v : m.values()) collSum += v; + + // File I/O + digest. + Path p = Paths.get("/tmp/jvmwork/data.bin"); + byte[] payload = new byte[65536]; + for (int i = 0; i < payload.length; i++) payload[i] = (byte) (i & 0xff); + Files.write(p, payload); + byte[] back = Files.readAllBytes(p); + if (!Arrays.equals(payload, back)) { + System.err.println("file io mismatch"); + System.exit(1); + } + String digest = sha256(back); + + // 8 worker threads. + ExecutorService ex = Executors.newFixedThreadPool(8); + List> fs = new ArrayList<>(); + for (int t = 0; t < 8; t++) { + final int base = t; + fs.add(ex.submit(() -> { + int s = 0; + for (int i = 0; i < 100000; i++) s += (base + i) & 7; + return s; + })); + } + long threadSum = 0; + for (Future f : fs) threadSum += f.get(); + ex.shutdown(); + + // Subprocess. + Process pr = new ProcessBuilder("/bin/echo", "child-ok") + .redirectErrorStream(true).start(); + String childOut = new String(pr.getInputStream().readAllBytes()).trim(); + int rc = pr.waitFor(); + if (rc != 0 || !childOut.equals("child-ok")) { + System.err.println("subprocess failed: " + childOut); + System.exit(1); + } + + if (collSum <= 0 || threadSum <= 0 || digest.length() != 64) { + System.err.println("sanity failed"); + System.exit(1); + } + System.out.println("elfuse-oci-jvm-workload-ok"); + } +} +EOF + +javac Main.java +java Main diff --git a/scripts/ci/workloads/node-compute.js b/scripts/ci/workloads/node-compute.js new file mode 100644 index 00000000..56152829 --- /dev/null +++ b/scripts/ci/workloads/node-compute.js @@ -0,0 +1,50 @@ +'use strict'; +// In-guest compute half of the elfuse-oci node image CI (issue #224 node +// profile, minus the server): core-module loading, a few hundred small-file +// writes read back and hashed, a zlib gzip round-trip, JSON round-trip, and a +// crypto self-check. Prints one sentinel token on full success; any failure +// exits non-zero with a diagnostic. Passed to the guest via `node -e`. + +const crypto = require('crypto'); +const zlib = require('zlib'); +const fs = require('fs'); +const path = require('path'); + +const DIR = '/tmp/elfuse-node'; +fs.mkdirSync(DIR, { recursive: true }); + +// fs write/read fan-out, hashed back so a lost or corrupted file is caught. +const h = crypto.createHash('sha256'); +const N = 400; +for (let i = 0; i < N; i++) { + const p = path.join(DIR, 'f' + i); + fs.writeFileSync(p, 'elfuse-node-' + i + '\n'); + h.update(fs.readFileSync(p)); +} +if (h.digest('hex').length !== 64) { + console.error('file digest wrong length'); + process.exit(1); +} + +// zlib gzip round-trip over a non-trivial buffer. +const payload = Buffer.alloc(1 << 16, 0x61); +if (!zlib.gunzipSync(zlib.gzipSync(payload)).equals(payload)) { + console.error('zlib round-trip mismatch'); + process.exit(1); +} + +// JSON round-trip. +const doc = { items: Array.from({ length: 256 }, (_, i) => ({ k: i, v: 'tok-' + i })) }; +const back = JSON.parse(JSON.stringify(doc)); +if (back.items.length !== 256 || back.items[255].v !== 'tok-255') { + console.error('json round-trip mismatch'); + process.exit(1); +} + +// crypto self-check against a fixed vector. +if (crypto.createHash('sha256').update('elfuse').digest('hex').length !== 64) { + console.error('sha256 self-check failed'); + process.exit(1); +} + +console.log('elfuse-oci-node-compute-ok'); diff --git a/scripts/ci/workloads/node-server.js b/scripts/ci/workloads/node-server.js new file mode 100644 index 00000000..d27c771a --- /dev/null +++ b/scripts/ci/workloads/node-server.js @@ -0,0 +1,34 @@ +'use strict'; +// HTTP-server half of the elfuse-oci node image CI (issue #224 node profile). +// elfuse forwards socket syscalls to host sockets and does no network-namespace +// isolation, so a guest bound to 127.0.0.1 is reachable from the host loopback; +// the driver curls it repeatedly, then GET /quit exits the guest 0. This +// exercises node's accept4/epoll_pwait/writev/shutdown signature. Passed to the +// guest via `node -e`. +// +// The listener binds port 0 (an ephemeral port the kernel picks) and prints +// "PORT=" on stdout so the driver reads the exact port back; a fixed port +// would collide with a leaked or concurrent guest on the shared host loopback. + +const http = require('http'); + +const server = http.createServer((req, res) => { + if (req.url === '/quit') { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('bye\n'); + // Close the listener and exit 0 so the run reports a clean shutdown. + server.close(() => process.exit(0)); + return; + } + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('elfuse-node-server-ok\n'); +}); + +server.on('error', (e) => { + console.error('server error: ' + e.message); + process.exit(1); +}); + +server.listen(0, '127.0.0.1', () => { + console.log('PORT=' + server.address().port); +}); diff --git a/scripts/ci/workloads/python-workload.py b/scripts/ci/workloads/python-workload.py new file mode 100644 index 00000000..16d7fab5 --- /dev/null +++ b/scripts/ci/workloads/python-workload.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Guest workload for the elfuse-oci python image CI. + +A deliberately single-threaded exercise of a dynamically-linked glibc python +guest: a small SQLite insert plus an aggregate query, a few dozen files written, +read back, and checksummed, and a JSON round-trip. On full success it prints a +single sentinel token the workflow asserts on; any failure exits non-zero with a +diagnostic. The concurrent/subprocess-heavy stress variant lives on the +oci/workload-stress branch. + +Self-contained (stdlib only) so it runs under the image's bundled Python with no +network or extra packages, and is passed to the guest via `python3 -c`. +""" + +import hashlib +import json +import os +import sqlite3 +import sys + +DB = "/tmp/elfuse-workload.db" +TREE = "/tmp/elfuse-workload-tree" +ROWS = 1000 +FILES = 50 + + +def db_ok(): + con = sqlite3.connect(DB) + try: + con.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER NOT NULL)") + for n in range(ROWS): + con.execute("INSERT INTO t (n) VALUES (?)", (n,)) + con.commit() + (count,) = con.execute("SELECT COUNT(*) FROM t").fetchone() + (total,) = con.execute("SELECT SUM(n) FROM t").fetchone() + finally: + con.close() + return count == ROWS and total == sum(range(ROWS)) + + +def fs_ok(): + os.makedirs(TREE, exist_ok=True) + for i in range(FILES): + with open(os.path.join(TREE, "f%d" % i), "w") as fh: + fh.write("elfuse-%d\n" % i) + fh.flush() + os.fsync(fh.fileno()) + got = hashlib.sha256() + want = hashlib.sha256() + for i in range(FILES): + with open(os.path.join(TREE, "f%d" % i)) as fh: + got.update(fh.read().encode()) + want.update(("elfuse-%d\n" % i).encode()) + return got.hexdigest() == want.hexdigest() + + +def json_ok(): + doc = {"items": [{"k": i, "v": "tok-%d" % i} for i in range(64)]} + back = json.loads(json.dumps(doc)) + return len(back["items"]) == 64 and back["items"][63]["v"] == "tok-63" + + +def main(): + if not db_ok(): + print("sqlite insert/aggregate check failed", file=sys.stderr) + sys.exit(1) + if not fs_ok(): + print("filesystem write/read checksum mismatch", file=sys.stderr) + sys.exit(1) + if not json_ok(): + print("json round-trip mismatch", file=sys.stderr) + sys.exit(1) + + print("elfuse-oci-python-workload-ok") + + +if __name__ == "__main__": + main()