From ecb45e2a7cc05fc4ca2510be4d4c7d1132321902 Mon Sep 17 00:00:00 2001 From: Nishad Date: Wed, 19 Aug 2026 15:15:06 -0700 Subject: [PATCH 1/5] Bound CAOS image cache growth --- README.md | 3 + build-builtins.sh | 2 +- caos-tools/build.sh | 8 +- crates/server/src/compute.rs | 224 +++++++++++++++-- design/image-cleanup.md | 50 ++++ design/one-stack-image.md | 1 + examples/consumer/README.md | 4 +- examples/consumer/flake.nix | 2 +- flake.nix | 14 +- image-cleanup.sh | 460 +++++++++++++++++++++++++++++++++++ tests/std-lint/cli.sh | 46 ++++ 11 files changed, 777 insertions(+), 37 deletions(-) create mode 100644 design/image-cleanup.md create mode 100644 image-cleanup.sh diff --git a/README.md b/README.md index e824d7d9..fd7b898a 100644 --- a/README.md +++ b/README.md @@ -603,6 +603,8 @@ spawning no container. - Build the stack with `nix build` - Run the dev stack with `result/bin/caosd up` +- Stack state is shared across worktrees under + `${XDG_CACHE_HOME:-$HOME/.cache}/caos`; set `CAOS_DATA` to override it. - **Check `caosd version` before believing a bug report.** A devShell that fails to build leaves direnv on the *previous* environment, so the `caosd` on PATH can be far older than the `flake.lock` that names it — and the symptom is an @@ -615,6 +617,7 @@ caosd up # bring the stack up + publish all of std, then return. Updates it caosd logs # follow the running stack's logs (Ctrl-C returns; stack stays up) caosd down # stop it (Redis + registry volumes and the server repo are kept) caosd reset # stop and wipe those volumes + the server repo for a clean slate +caosd image-cleanup # dry-run bounded registry LRU + disposable Docker cleanup caosd version # the caos revision this command was built from ``` diff --git a/build-builtins.sh b/build-builtins.sh index 2a3ee15e..8e15ba1b 100755 --- a/build-builtins.sh +++ b/build-builtins.sh @@ -77,7 +77,7 @@ export CAOS_SERVER_URL=$SERVER_URL # shape a user has. `caos-cli` builds objects here (in-process via gix); `git # push` ships them to the server. Reused across runs (git init is idempotent). # CAOS_CLIENT_REPO relocates it off PROJECT (which is read-only when caosd runs -# us from the store); caosd points it at $CAOS_DATA so it persists per-project. +# us from the store); caosd points it at $CAOS_DATA so it persists across worktrees. CLIENT=${CAOS_CLIENT_REPO:-$PROJECT/.caos-dev/client-repo} git init -q "$CLIENT" # When the caller says this repo dies with the process (caos-tools/build.sh runs diff --git a/caos-tools/build.sh b/caos-tools/build.sh index 164118f1..5a1c2872 100644 --- a/caos-tools/build.sh +++ b/caos-tools/build.sh @@ -86,9 +86,9 @@ reduce) caos put "$R" /cas/reduced # THE BUILD'S OWN INPUTS, and nothing else. `make` compiles the workspace, - # runs build-builtins.sh (which reads std/ and crates/worker-common), and - # installs stack/serve and test-stack/worker into the image. It reads - # nothing else in the tree. + # runs build-builtins.sh (which reads std/ and crates/worker-common), embeds + # image-cleanup.sh in caosd, and installs stack/serve and test-stack/worker + # into the image. It reads nothing else in the tree. # # It used to get the WHOLE tree, so a one-line edit to tests//cli.sh — # or to a design doc, or this comment — recompiled the workspace, @@ -106,7 +106,7 @@ reduce) S=/tmp/src rm -rf "$S"; mkdir -p "$S" for e in Cargo.toml Cargo.lock rust-toolchain.toml \ - crates std stack test-stack build-builtins.sh; do + crates std stack test-stack build-builtins.sh image-cleanup.sh; do if [ -e "$e" ]; then cp -RL --preserve=mode "$e" "$S/$e"; fi done caos put "$S" /cas/src diff --git a/crates/server/src/compute.rs b/crates/server/src/compute.rs index cb02bb14..22494d34 100644 --- a/crates/server/src/compute.rs +++ b/crates/server/src/compute.rs @@ -32,7 +32,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{mpsc, Mutex, OnceLock}; -use std::time::Duration; +use std::time::{Duration, Instant}; use sha2::{Digest, Sha256}; @@ -43,6 +43,16 @@ use crate::{Config, HttpError}; /// digest, so the name is arbitrary and fixed. const REGISTRY_REPO: &str = "caos"; +/// A stable tag whose link mtime is the manifest's durable last-used clock. +/// Retagging the same immutable manifest changes no image bytes or digest, and +/// works from nested test servers that share the host registry but not its +/// filesystem or Redis. +const IMAGE_USED_TAG_PREFIX: &str = "caos-used-"; + +/// Avoid a registry write for every warm job while keeping a much tighter +/// bound than image-cleanup's default multi-day retention window. +const IMAGE_USED_REFRESH: Duration = Duration::from_secs(60 * 60); + /// Prefix marking the `image` parameter as an ordinary docker reference rather /// than one of our git images (the default). const DOCKER_SCHEME: &str = "docker://"; @@ -1379,32 +1389,37 @@ fn validate_docker_reference(reference: &str, allow_seeded: bool) -> Result<(), /// git images (the default): convert it to a real image, push it to the registry, /// and return a digest reference into the registry. fn resolve_image(config: &Config, image: &str) -> Result { - if let Some(reference) = image.strip_prefix(DOCKER_SCHEME) { + let reference = if let Some(reference) = image.strip_prefix(DOCKER_SCHEME) { validate_docker_reference(reference, true).map_err(|e| HttpError::new(400, e))?; - return Ok(reference.to_string()); - } - if !image.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(HttpError::new( - 400, - format!("git image must be a hex hash (or use {DOCKER_SCHEME}): {image:?}"), - )); - } - // The nested test stack (tests/lib/run-test.sh) runs on images the - // outer suite already built and pushed, so its server passes git images - // through unconverted: no OCI convert, no registry round-trip. The - // default keeps converting. - if std::env::var("CAOS_IMAGE_RESOLVE").as_deref() == Ok("none") { - return Ok(image.to_string()); - } - // A git image tree, converted. NOT a flake: do not add a branch here that - // notices `flake.nix` + `flake.lock` and builds it. Doing so needs a builder - // resolved BY NAME out of an ambient library, which is the one thing the - // server must not have — it holds an arg tree, not a project tree, so it - // cannot resolve a dependency by descent the way a client can. A flake - // directory says `run --base:@=DEEP-DEPS/flake-builder --in:@=.` and the CLIENT - // evaluates it, so what arrives here is already an image (design/caos-expr.md). - convert_git_image(config, image) - .map_err(|e| HttpError::new(500, format!("converting git image {image}: {e}"))) + reference.to_string() + } else { + if !image.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(HttpError::new( + 400, + format!("git image must be a hex hash (or use {DOCKER_SCHEME}): {image:?}"), + )); + } + // The nested test stack (tests/lib/run-test.sh) runs on images the + // outer suite already built and pushed, so its server passes git images + // through unconverted: no OCI convert, no registry round-trip. The + // default keeps converting. + if std::env::var("CAOS_IMAGE_RESOLVE").as_deref() == Ok("none") { + return Ok(image.to_string()); + } + // A git image tree, converted. NOT a flake: do not add a branch here that + // notices `flake.nix` + `flake.lock` and builds it. Doing so needs a builder + // resolved BY NAME out of an ambient library, which is the one thing the + // server must not have — it holds an arg tree, not a project tree, so it + // cannot resolve a dependency by descent the way a client can. A flake + // directory says `run --base:@=DEEP-DEPS/flake-builder --in:@=.` and the CLIENT + // evaluates it, so what arrives here is already an image (design/caos-expr.md). + convert_git_image(config, image) + .map_err(|e| HttpError::new(500, format!("converting git image {image}: {e}")))? + }; + + mark_local_image_used(config, &reference) + .map_err(|e| HttpError::new(500, format!("recording image use for {reference}: {e}")))?; + Ok(reference) } /// The lock for one cache key, minted on first use. A redis cache read followed @@ -1531,6 +1546,109 @@ fn image_ref(config: &Config, manifest_digest: &str) -> String { ) } +/// Return the digest when `reference` names this server's own registry repo. +/// The push and pull names differ in the host placement (`caos-registry` from +/// the server, `localhost` from Docker), so both spellings are local. +fn local_registry_digest<'a>( + push_url: &str, + pull_host: &str, + reference: &'a str, +) -> Option<&'a str> { + let (name, digest) = reference.rsplit_once('@')?; + let push = push_url.trim_end_matches('/'); + let push_host = push + .strip_prefix("http://") + .or_else(|| push.strip_prefix("https://")) + .unwrap_or(push); + let push_name = format!("{push_host}/{REGISTRY_REPO}"); + let pull_name = format!("{}/{REGISTRY_REPO}", pull_host.trim_end_matches('/')); + if name == push_name || name == pull_name { + Some(digest) + } else { + None + } +} + +fn image_used_tag(digest: &str) -> Result { + let encoded = digest + .strip_prefix("sha256:") + .ok_or_else(|| format!("unsupported manifest digest {digest}"))?; + if encoded.len() != 64 + || !encoded + .bytes() + .all(|b| b.is_ascii_digit() || matches!(b, b'a'..=b'f')) + { + return Err(format!("invalid manifest digest {digest}")); + } + Ok(format!("{IMAGE_USED_TAG_PREFIX}{encoded}")) +} + +/// Retag a local immutable manifest under its stable usage tag. Distribution +/// updates the tag link atomically, and its mtime is the cleanup clock. The +/// memo is only a write throttle; the durable record remains in the registry. +fn mark_local_image_used(config: &Config, reference: &str) -> Result<(), String> { + let Some(digest) = local_registry_digest( + &config.registry_push_url, + &config.registry_pull_host, + reference, + ) else { + return Ok(()); + }; + mark_manifest_used(config, digest) +} + +fn mark_manifest_used(config: &Config, digest: &str) -> Result<(), String> { + static RECENT: OnceLock>> = OnceLock::new(); + let memo_key = format!("{} {digest}", config.registry_push_url); + let recent = RECENT.get_or_init(|| Mutex::new(HashMap::new())); + if recent + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(&memo_key) + .is_some_and(|at| at.elapsed() < IMAGE_USED_REFRESH) + { + return Ok(()); + } + + let tag = image_used_tag(digest)?; + let base = config.registry_push_url.trim_end_matches('/'); + let source = format!("{base}/v2/{REGISTRY_REPO}/manifests/{digest}"); + let accept = "application/vnd.oci.image.manifest.v1+json, \ + application/vnd.docker.distribution.manifest.v2+json"; + let response = minreq::get(&source) + .with_header("Accept", accept) + .send() + .map_err(|e| format!("GET {source}: {e}"))?; + if !(200..300).contains(&response.status_code) { + return Err(format!( + "reading manifest {digest}: {} {}", + response.status_code, response.reason_phrase + )); + } + let content_type = response + .headers + .get("content-type") + .cloned() + .ok_or_else(|| format!("manifest {digest} response has no Content-Type"))?; + let target = format!("{base}/v2/{REGISTRY_REPO}/manifests/{tag}"); + let response = minreq::put(&target) + .with_header("Content-Type", content_type) + .with_body(response.as_bytes().to_vec()) + .send() + .map_err(|e| format!("PUT {target}: {e}"))?; + if !(200..300).contains(&response.status_code) { + return Err(format!( + "tagging manifest {digest} as used: {} {}", + response.status_code, response.reason_phrase + )); + } + recent + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(memo_key, Instant::now()); + Ok(()) +} + /// Copy a base image (`base_ref`, a bare docker reference) from its source /// registry into our own repo with skopeo, so its blobs are available for a /// converted git image to reference. Returns the base's manifest layers @@ -1601,6 +1719,15 @@ fn fetch_base(config: &Config, base_ref: &str) -> Result { resp.status_code, resp.reason_phrase )); } + let base_manifest_digest = resp + .headers + .get("docker-content-digest") + .cloned() + .unwrap_or_else(|| format!("sha256:{}", sha256_hex(resp.as_bytes()))); + // A copied base is a registry artifact in its own right. Its functional + // `base-*` tag is a build memo, not a usage clock, so refresh the same + // stable usage tag as runnable images do. + mark_manifest_used(config, &base_manifest_digest)?; let manifest: serde_json::Value = serde_json::from_slice(resp.as_bytes()) .map_err(|e| format!("parsing base manifest: {e}"))?; let layers = manifest["layers"] @@ -2412,6 +2539,51 @@ mod single_flight_tests { } } +#[cfg(test)] +mod image_usage_tests { + use super::*; + + #[test] + fn recognizes_both_names_for_the_local_registry() { + let digest = format!("sha256:{}", "a".repeat(64)); + assert_eq!( + local_registry_digest( + "http://caos-registry:5000", + "localhost:5000", + &format!("caos-registry:5000/caos@{digest}") + ), + Some(digest.as_str()) + ); + assert_eq!( + local_registry_digest( + "http://caos-registry:5000/", + "localhost:5000/", + &format!("localhost:5000/caos@{digest}") + ), + Some(digest.as_str()) + ); + assert_eq!( + local_registry_digest( + "http://caos-registry:5000", + "localhost:5000", + &format!("ghcr.io/example/caos@{digest}") + ), + None + ); + } + + #[test] + fn usage_tag_is_stable_and_digest_specific() { + let digest = format!("sha256:{}", "b".repeat(64)); + assert_eq!( + image_used_tag(&digest).unwrap(), + format!("caos-used-{}", "b".repeat(64)) + ); + assert!(image_used_tag("sha256:short").is_err()); + assert!(image_used_tag(&format!("sha512:{}", "b".repeat(64))).is_err()); + } +} + #[cfg(test)] mod continuation_shape_tests { use super::*; diff --git a/design/image-cleanup.md b/design/image-cleanup.md new file mode 100644 index 00000000..300e59eb --- /dev/null +++ b/design/image-cleanup.md @@ -0,0 +1,50 @@ +# Image cleanup + +CAOS has two image caches with different jobs: + +- Docker is the working set. Once no container uses a CAOS image, its local + copy is disposable because Docker can pull it back from the local registry. +- The registry is the backing cache. It retains recently used content up to a + size ceiling, plus correctness roots. + +`caosd image-cleanup` applies that distinction. It is a dry run unless passed +`--execute`. + +## Last used and the registry budget + +Immediately before dispatch, the server retags a local manifest as +`caos-used-`. The tag points at the same immutable manifest, +so it changes neither the image nor its digest. Replacing the tag updates +Distribution's tag-link mtime, which is the durable last-used clock. Each +server process writes at most once per manifest per hour. + +The record lives in the registry because host and nested test servers share the +registry but not a filesystem or Redis instance. A manifest without a usage +tag is treated as used now and receives a tag on the first executing cleanup. + +The default policy removes anything unused for seven days, then applies a +20 GiB ceiling to what remains. Non-root manifests are considered newest first. +Their manifest, config, and layer blobs are added to a set, so shared layers are +counted once; the least-recently-used closures that do not fit are removed. +`--unused-for=d` and `--max-size=GiB` override the defaults. + +## Roots and execution + +The direct image digests under `refs/caos/seed` are permanent roots for the +current std generation. A seed hit already promises that its digest exists, so +deleting it would turn a valid result into a later Docker pull failure. +Republishing std moves the ref and makes the old generation ordinary LRU data. + +`--execute` refuses while a `caos-worker-*` container is active. Otherwise it: + +1. stops the host stack if it was running and removes idle test stacks; +2. tags retained legacy manifests and deletes selected manifests; +3. runs Distribution's offline garbage collector when a manifest was deleted; +4. clears Redis only after registry deletion, because cached results can carry + deleted digests indirectly; +5. removes every unused local Docker copy of a CAOS registry image and obsolete + `caos-stack-src:*` tags; and +6. restores the host stack if it was running. + +Nix is a machine-wide store rather than a CAOS-owned cache. This command does +not run global Nix garbage collection. diff --git a/design/one-stack-image.md b/design/one-stack-image.md index 005ce9ea..49a32dde 100644 --- a/design/one-stack-image.md +++ b/design/one-stack-image.md @@ -178,6 +178,7 @@ caosd logs tail the group's logs caosd reset stop and wipe $CAOS_DATA caosd std-build publish std to this stack's registry and git, then exit caosd std-check verify what std references still exists; non-zero if not +caosd image-cleanup dry-run bounded registry LRU and local image cleanup ``` **Die as a group.** Any member's death takes the group down, so a half-dead diff --git a/examples/consumer/README.md b/examples/consumer/README.md index c1aa9dcc..c9d254ba 100644 --- a/examples/consumer/README.md +++ b/examples/consumer/README.md @@ -18,8 +18,8 @@ of the relative path used here. nix develop # 1. Bring the stack up (redis + registry + caos server). Foreground; Ctrl-C -# stops it. Server state (the bare git repo) lives in ./.caos-data — override -# with CAOS_DATA. This also publishes the builtin stdlib on startup. +# stops it. Server state lives in $XDG_CACHE_HOME/caos (or ~/.cache/caos) — +# override with CAOS_DATA. This also publishes the builtin stdlib on startup. caosd up # 2. In another shell (also `nix develop`), add the server as the `caos` remote. diff --git a/examples/consumer/flake.nix b/examples/consumer/flake.nix index 3f976fdd..466b3ee5 100644 --- a/examples/consumer/flake.nix +++ b/examples/consumer/flake.nix @@ -25,7 +25,7 @@ # caosd — bring the stack up (foreground; Ctrl-C stops it). It also # (re)publishes the builtin worker library on each startup. # caos-cli — drive workers (run/get/curry/…) - # caosd honors CAOS_DATA (default ./.caos-data) for the server's repo; + # caosd honors CAOS_DATA (default $XDG_CACHE_HOME/caos) for the server's repo; # caos-cli must run inside a git working tree (this one) that has the # server as its `caos` remote — that remote URL *is* the server, so # there's no CAOS_SERVER_URL to set: diff --git a/flake.nix b/flake.nix index 18232b66..0342c5e7 100644 --- a/flake.nix +++ b/flake.nix @@ -691,6 +691,10 @@ # working stack and not homework — so this is the strict # gate for anything that runs against a stack it did not # just bring up (design/one-stack-image.md). + # caosd image-cleanup dry-run bounded registry LRU and disposable + # local-image cleanup; --execute briefly stops and + # restores an idle stack. The current std seed is always + # retained; see design/image-cleanup.md. # caosd version the caos revision THIS command was built from. Ask it # before believing a bug report: a devShell that fails to # build leaves direnv on the previous environment, so the @@ -717,10 +721,10 @@ # and runnerd actually changed (stage_bins). runtimeInputs = [ pkgs.coreutils pkgs.git pkgs.curl pkgs.bash pkgs.skopeo pkgs.gzip - pkgs.util-linux pkgs.diffutils + pkgs.util-linux pkgs.diffutils pkgs.gnugrep pkgs.findutils pkgs.jq ]; text = '' - : "''${CAOS_DATA:=$PWD/.caos-data}" + : "''${CAOS_DATA:=''${XDG_CACHE_HOME:-$HOME/.cache}/caos}" CAOS_DATA="$(readlink -m "$CAOS_DATA")" export CAOS_DATA mkdir -p "$CAOS_DATA" @@ -894,7 +898,7 @@ usage() { echo "caosd ($CAOS_REV)" - echo "usage: caosd [up|down|reset|logs|std-build|std-check|version]" + echo "usage: caosd [up|down|reset|logs|std-build|std-check|image-cleanup|version]" } case "''${1:-up}" in @@ -1011,6 +1015,10 @@ std-check) std_check ;; + image-cleanup) + shift + bash ${./image-cleanup.sh} "$@" + ;; *) echo "caosd: unknown command '$1'" >&2 usage >&2 diff --git a/image-cleanup.sh b/image-cleanup.sh new file mode 100644 index 00000000..86596618 --- /dev/null +++ b/image-cleanup.sh @@ -0,0 +1,460 @@ +#!/usr/bin/env bash +# Bound CAOS's two image caches. Docker is only the local working set, so every +# unused CAOS image there is disposable. The registry is the backing cache: it +# keeps the most recently used manifest closures up to a size ceiling, plus the +# current std seed regardless of age or size. +set -euo pipefail +export LC_ALL=C + +: "${CAOS_DATA:?image-cleanup needs CAOS_DATA}" + +case "$CAOS_DATA" in + /|"") + echo "caosd image-cleanup: refusing unsafe CAOS_DATA '$CAOS_DATA'" >&2 + exit 1 + ;; +esac + +STATE="$CAOS_DATA/stack" +CLIENT="$CAOS_DATA/publish-client-repo" +REPOSITORY=caos +USED_TAG_PREFIX=caos-used- +DEFAULT_UNUSED_FOR=7d +DEFAULT_MAX_SIZE=20GiB +GIB=$((1024 * 1024 * 1024)) + +execute=no +unused_for=$DEFAULT_UNUSED_FOR +max_size=$DEFAULT_MAX_SIZE + +usage() { + echo "usage: caosd image-cleanup [--unused-for=d] [--max-size=GiB] [--execute]" >&2 +} + +fail() { + echo "caosd image-cleanup: $*" >&2 + exit 1 +} + +mtime() { + if stat -c %Y "$1" 2>/dev/null; then + return + fi + stat -f %m "$1" +} + +format_epoch() { + if date -u -d "@$1" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null; then + return + fi + date -u -r "$1" '+%Y-%m-%dT%H:%M:%SZ' +} + +format_bytes() { + local bytes=$1 whole tenth + whole=$((bytes / GIB)) + tenth=$(((bytes % GIB) * 10 / GIB)) + printf '%d.%dGiB' "$whole" "$tenth" +} + +for arg in "$@"; do + case "$arg" in + --execute) + execute=yes + ;; + --unused-for=*) + unused_for=${arg#--unused-for=} + ;; + --max-size=*) + max_size=${arg#--max-size=} + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + fail "unknown argument '$arg'" + ;; + esac +done + +if [[ ! "$unused_for" =~ ^[0-9]+d$ ]]; then + fail "--unused-for must be a whole number of days such as 7d" +fi +unused_days=${unused_for%d} +if [ "$unused_days" -lt 1 ]; then + fail "--unused-for must be at least 1d" +fi +if [[ ! "$max_size" =~ ^[0-9]+GiB$ ]]; then + fail "--max-size must be whole GiB such as 20GiB" +fi +max_gib=${max_size%GiB} +if [ "$max_gib" -lt 1 ]; then + fail "--max-size must be at least 1GiB" +fi +max_bytes=${CAOS_IMAGE_CLEANUP_MAX_BYTES:-$((max_gib * GIB))} +if [[ ! "$max_bytes" =~ ^[0-9]+$ ]] || [ "$max_bytes" -lt 1 ]; then + fail "CAOS_IMAGE_CLEANUP_MAX_BYTES must be positive bytes" +fi +budget=$(format_bytes "$max_bytes") + +[ -d "$STATE/registry" ] || fail "no registry under $STATE (run 'caosd up' first)" +[ -d "$CLIENT" ] || fail "no publish client under $CLIENT (run 'caosd up' first)" + +manifest_root="$STATE/registry/docker/registry/v2/repositories/$REPOSITORY/_manifests" +revision_root="$manifest_root/revisions/sha256" +tag_root="$manifest_root/tags" +blob_root="$STATE/registry/docker/registry/v2/blobs/sha256" +[ -d "$revision_root" ] || fail "registry has no $REPOSITORY manifest revisions" + +scratch=$(mktemp -d "$STATE/image-cleanup.XXXXXX") +inventory="$scratch/inventory" +candidates="$scratch/candidates" +candidate_report="$scratch/candidate-report" +retained="$scratch/retained" +: > "$inventory" +: > "$candidates" +: > "$candidate_report" +: > "$retained" + +registry_container="" +gc_container="" +stack_stopped=no +finish() { + local rc=$? + trap - EXIT + if [ -n "$registry_container" ] && docker container inspect "$registry_container" >/dev/null 2>&1; then + docker container stop "$registry_container" >/dev/null 2>&1 || true + docker container rm "$registry_container" >/dev/null 2>&1 || true + fi + if [ -n "$gc_container" ] && docker container inspect "$gc_container" >/dev/null 2>&1; then + docker container stop "$gc_container" >/dev/null 2>&1 || true + docker container rm "$gc_container" >/dev/null 2>&1 || true + fi + rm -rf "$scratch" + if [ "$stack_stopped" = yes ]; then + if docker container start caos-stack >/dev/null 2>&1; then + ready="" + for _ in $(seq 1 60); do + if curl -s -o /dev/null --max-time 2 http://localhost:9090/; then + ready=yes + break + fi + sleep 1 + done + if [ -n "$ready" ]; then + echo "caosd image-cleanup: restarted caos-stack" + else + echo "caosd image-cleanup: caos-stack did not become ready after restart" >&2 + rc=1 + fi + else + echo "caosd image-cleanup: could not restart caos-stack" >&2 + rc=1 + fi + fi + exit "$rc" +} +trap finish EXIT + +declare -A protected=() +seed=$(git -C "$CLIENT" rev-parse --verify -q refs/caos/seed) \ + || fail "no refs/caos/seed (run 'caosd up' first)" +protected_count=0 +while read -r _mode _kind oid name; do + base=$(git -C "$CLIENT" cat-file -p "$oid:result/base" 2>/dev/null) || continue + digest=${base##*@} + if [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] && [ -z "${protected[$digest]:-}" ]; then + protected[$digest]=$name + protected_count=$((protected_count + 1)) + fi +done < <(git -C "$CLIENT" ls-tree "$seed") +[ "$protected_count" -gt 0 ] || fail "the current std seed names no registry manifests" + +now=${CAOS_IMAGE_CLEANUP_NOW:-$(date +%s)} +if [[ ! "$now" =~ ^[0-9]+$ ]]; then + fail "CAOS_IMAGE_CLEANUP_NOW must be Unix seconds" +fi +cutoff=$((now - unused_days * 86400)) +manifest_count=0 +untracked_count=0 + +while IFS= read -r -d '' revision_link; do + digest_hex=${revision_link%/link} + digest_hex=${digest_hex##*/} + digest="sha256:$digest_hex" + recorded=$(<"$revision_link") + [ "$recorded" = "$digest" ] || fail "manifest revision $revision_link records $recorded" + manifest_count=$((manifest_count + 1)) + + marker="$tag_root/$USED_TAG_PREFIX$digest_hex/current/link" + if [ -n "${protected[$digest]:-}" ]; then + printf 'seed\t%s\t%s\t%s\n' "$now" "$digest" "${protected[$digest]}" >> "$inventory" + elif [ -f "$marker" ]; then + marker_digest=$(<"$marker") + [ "$marker_digest" = "$digest" ] \ + || fail "usage marker $marker records $marker_digest, expected $digest" + printf 'tracked\t%s\t%s\t-\n' "$(mtime "$marker")" "$digest" >> "$inventory" + else + # No creation timestamp is trustworthy for reproducible images. Treat a + # legacy manifest as used now; --execute gives it a durable marker. + printf 'untracked\t%s\t%s\t-\n' "$now" "$digest" >> "$inventory" + untracked_count=$((untracked_count + 1)) + fi +done < <(find "$revision_root" -mindepth 2 -maxdepth 2 -type f -name link -print0) + +blob_path() { + local digest=$1 hex=${1#sha256:} + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid registry blob digest $digest" + printf '%s/%s/%s/data\n' "$blob_root" "${hex:0:2}" "$hex" +} + +declare -A kept_blobs=() +closure_digests=() +closure_increment=0 +plan_closure() { + local digest=$1 manifest_blob blob path size + manifest_blob=$(blob_path "$digest") + [ -f "$manifest_blob" ] || fail "manifest blob $digest is missing" + jq -r '(.config.digest? // empty), (.layers[]?.digest // empty)' \ + "$manifest_blob" > "$scratch/descriptors" \ + || fail "manifest blob $digest is not valid JSON" + { + printf '%s\n' "$digest" + while IFS= read -r blob; do printf '%s\n' "$blob"; done < "$scratch/descriptors" + } | sort -u > "$scratch/closure" \ + || fail "sorting the blob closure for $digest" + closure_digests=() + while IFS= read -r blob; do + [ -n "$blob" ] || continue + closure_digests+=("$blob") + done < "$scratch/closure" + closure_increment=0 + for blob in "${closure_digests[@]}"; do + if [ -n "${kept_blobs[$blob]:-}" ]; then + continue + fi + path=$(blob_path "$blob") + [ -f "$path" ] || fail "blob $blob referenced by $digest is missing" + size=$(stat -c %s "$path" 2>/dev/null || stat -f %z "$path") + closure_increment=$((closure_increment + size)) + done +} + +retain_planned_closure() { + local blob + for blob in "${closure_digests[@]}"; do + kept_blobs[$blob]=1 + done +} + +# Roots enter first and may exceed the ceiling: correctness beats the cache +# budget. Everything else is considered newest-first, yielding a true LRU over +# the union of manifest/config/layer blobs rather than double-counting layers. +retained_bytes=0 +while IFS=$'\t' read -r _kind _used digest detail; do + plan_closure "$digest" + retain_planned_closure + retained_bytes=$((retained_bytes + closure_increment)) + printf '%s\n' "$digest" >> "$retained" + printf ' KEEP %s current seed (%s)\n' "$digest" "$detail" +done < <(grep '^seed' "$inventory" || true) +roots_over_budget=no +if [ "$retained_bytes" -gt "$max_bytes" ]; then + roots_over_budget=yes +fi + +candidate_count=0 +while IFS=$'\t' read -r kind used digest _detail; do + if [ "$used" -lt "$cutoff" ]; then + printf '%s\n' "$digest" >> "$candidates" + printf 'age\t%s\t%s\n' "$used" "$digest" >> "$candidate_report" + candidate_count=$((candidate_count + 1)) + continue + fi + + plan_closure "$digest" + if [ $((retained_bytes + closure_increment)) -gt "$max_bytes" ]; then + printf '%s\n' "$digest" >> "$candidates" + printf 'size\t%s\t%s\n' "$used" "$digest" >> "$candidate_report" + candidate_count=$((candidate_count + 1)) + continue + fi + retain_planned_closure + retained_bytes=$((retained_bytes + closure_increment)) + printf '%s\n' "$digest" >> "$retained" +done < <(grep -v '^seed' "$inventory" | sort -t $'\t' -k2,2nr -k3,3 || true) + +echo "caosd image-cleanup: $manifest_count manifests; $protected_count seed roots; $candidate_count selected; $untracked_count without history" +echo "caosd image-cleanup: planned registry content $(format_bytes "$retained_bytes") / $budget; age limit $unused_for" +if [ "$roots_over_budget" = yes ]; then + echo "caosd image-cleanup: seed roots alone exceed the registry budget" >&2 +fi +while IFS=$'\t' read -r reason used digest; do + stamp=$(format_epoch "$used") + case "$reason" in + age) printf ' DELETE %s last used %s\n' "$digest" "$stamp" ;; + size) printf ' DELETE %s LRU over %s (last used %s)\n' "$digest" "$budget" "$stamp" ;; + esac +done < "$candidate_report" + +if [ "$execute" = no ]; then + echo "caosd image-cleanup: dry run; pass --execute to apply" + exit 0 +fi + +# Stop accepting work before touching either cache. A worker already running is +# left alone; the caller can retry once it finishes. With no worker containers, +# a test stack is idle and safe to remove. +running=$(docker ps --format '{{.Names}}' | grep -E '^caos-worker-' || true) +if [ -n "$running" ]; then + echo "caosd image-cleanup: active workers:" >&2 + while IFS= read -r name; do echo " $name" >&2; done <<< "$running" + fail "wait for active work to finish before --execute" +fi + +if docker container inspect caos-registry-cleanup >/dev/null 2>&1 \ + || docker container inspect caos-registry-gc >/dev/null 2>&1; then + fail "a previous cleanup container remains; inspect and remove it first" +fi + +if [ "$(docker inspect -f '{{.State.Running}}' caos-stack 2>/dev/null || true)" = true ]; then + docker container stop caos-stack >/dev/null + stack_stopped=yes +fi + +running=$(docker ps --format '{{.Names}}' | grep -E '^caos-worker-' || true) +[ -z "$running" ] || fail "a worker started while the stack was stopping; retry cleanup" + +while IFS= read -r container_id; do + [ -n "$container_id" ] || continue + docker container rm -f "$container_id" >/dev/null +done < <(docker ps -aq --filter name=caos-test-stack-) + +if [ "$candidate_count" -gt 0 ] || [ "$untracked_count" -gt 0 ]; then + config="$scratch/registry.yml" + cat > "$config" </dev/null + + ready="" + for _ in $(seq 1 30); do + if curl -fs -o /dev/null http://localhost:5000/v2/; then + ready=yes + break + fi + sleep 1 + done + [ -n "$ready" ] || fail "temporary registry did not become ready" + + accept='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' + while IFS= read -r digest; do + digest_hex=${digest#sha256:} + marker="$tag_root/$USED_TAG_PREFIX$digest_hex/current/link" + if [ -f "$marker" ]; then + continue + fi + curl -fsS -D "$scratch/headers" -o "$scratch/manifest" \ + -H "Accept: $accept" \ + "http://localhost:5000/v2/$REPOSITORY/manifests/$digest" + content_type="" + while IFS= read -r header; do + header=${header%$'\r'} + case "$header" in + Content-Type:*|content-type:*) + content_type=${header#*:} + content_type=${content_type# } + ;; + esac + done < "$scratch/headers" + [ -n "$content_type" ] || fail "manifest $digest response has no Content-Type" + curl -fsS -o /dev/null -X PUT \ + -H "Content-Type: $content_type" \ + --data-binary @"$scratch/manifest" \ + "http://localhost:5000/v2/$REPOSITORY/manifests/$USED_TAG_PREFIX$digest_hex" + done < "$retained" + + while IFS= read -r digest; do + code=$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE \ + "http://localhost:5000/v2/$REPOSITORY/manifests/$digest") + case "$code" in + 202|404) ;; + *) fail "deleting $digest returned HTTP $code" ;; + esac + done < "$candidates" + + docker container stop "$registry_container" >/dev/null + docker container rm "$registry_container" >/dev/null + registry_container="" + + if [ "$candidate_count" -gt 0 ]; then + # A cached result can contain a now-deleted registry digest indirectly, so + # invalidate the cache only when deletion actually happened. + rm -rf "$STATE/redis" + mkdir -p "$STATE/redis" + + gc_container=caos-registry-gc + if ! docker run --name "$gc_container" \ + -v "$STATE:/state" \ + --entrypoint /bin/registry \ + caos-stack:latest garbage-collect --quiet --delete-untagged "$config_in_container" \ + > "$scratch/gc.log" 2>&1; then + tail -n 30 "$scratch/gc.log" >&2 + fail "registry garbage collection failed" + fi + docker container rm "$gc_container" >/dev/null + gc_container="" + fi +fi + +# Docker is a working set in front of the local registry. Once CAOS is idle, +# every pulled CAOS image can go; a later run restores it from the registry. +docker_removed=0 +while IFS= read -r image_id; do + [ -n "$image_id" ] || continue + local_caos="" + while IFS= read -r reference; do + case "$reference" in + localhost:5000/caos@sha256:*|caos-registry:5000/caos@sha256:*) + local_caos=yes + break + ;; + esac + done < <(docker image inspect -f '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id") + if [ -n "$local_caos" ] && docker image rm "$image_id" >/dev/null 2>&1; then + docker_removed=$((docker_removed + 1)) + fi +done < <(docker image ls -q --no-trunc | sort -u) + +# load_once's content tags avoid repeated docker loads, but only the tag for the +# current stack image is useful. +current_stack=$(docker image inspect -f '{{.Id}}' caos-stack:latest 2>/dev/null || true) +while read -r tag image_id; do + case "$tag" in + caos-stack-src:*) + if [ "$image_id" != "$current_stack" ]; then + docker image rm "$tag" >/dev/null 2>&1 || true + fi + ;; + esac +done < <(docker image ls --no-trunc --format '{{.Repository}}:{{.Tag}} {{.ID}}') + +registry_bytes=$(du -sk "$STATE/registry" | while read -r kib _rest; do echo $((kib * 1024)); done) +echo "caosd image-cleanup: deleted $candidate_count registry manifests; removed $docker_removed local CAOS images" +echo "caosd image-cleanup: registry now uses $(format_bytes "$registry_bytes")" diff --git a/tests/std-lint/cli.sh b/tests/std-lint/cli.sh index 3eca722b..fb29701d 100644 --- a/tests/std-lint/cli.sh +++ b/tests/std-lint/cli.sh @@ -32,4 +32,50 @@ echo "== lint-bake-anchor.sh: every std tool's crates.io deps are anchored ==" > bash "$CAOS_PROJECT/lint-bake-anchor.sh" \ || fail "a std tool's crates.io dep is missing from bake-anchor (see above)" +echo "== image-cleanup.sh: seed roots, age, and LRU size bound ==" >&2 +fixture=$(mktemp -d) +repo="$fixture/data/publish-client-repo" +mkdir -p "$repo" +git init -q "$repo" +seed_digest=sha256:$(printf 'a%.0s' {1..64}) +stale_digest=sha256:$(printf 'b%.0s' {1..64}) +recent_digest=sha256:$(printf 'c%.0s' {1..64}) +base_blob=$(printf 'docker://localhost:5000/caos@%s' "$seed_digest" \ + | git -C "$repo" hash-object -w --stdin) +result_tree=$(printf '100644 blob %s\tbase\n' "$base_blob" | git -C "$repo" mktree) +record_tree=$(printf '040000 tree %s\tresult\n' "$result_tree" | git -C "$repo" mktree) +seed_tree=$(printf '040000 tree %s\trunner\n' "$record_tree" | git -C "$repo" mktree) +git -C "$repo" update-ref refs/caos/seed "$seed_tree" +revision_root="$fixture/data/stack/registry/docker/registry/v2/repositories/caos/_manifests/revisions/sha256" +tag_root="$fixture/data/stack/registry/docker/registry/v2/repositories/caos/_manifests/tags" +mkdir -p "$revision_root/${seed_digest#sha256:}" \ + "$revision_root/${stale_digest#sha256:}" \ + "$revision_root/${recent_digest#sha256:}" \ + "$tag_root/caos-used-${stale_digest#sha256:}/current" \ + "$tag_root/caos-used-${recent_digest#sha256:}/current" +printf '%s' "$seed_digest" > "$revision_root/${seed_digest#sha256:}/link" +printf '%s' "$stale_digest" > "$revision_root/${stale_digest#sha256:}/link" +printf '%s' "$recent_digest" > "$revision_root/${recent_digest#sha256:}/link" +printf '%s' "$stale_digest" > "$tag_root/caos-used-${stale_digest#sha256:}/current/link" +printf '%s' "$recent_digest" > "$tag_root/caos-used-${recent_digest#sha256:}/current/link" +touch -t 197001010000.01 "$tag_root/caos-used-${stale_digest#sha256:}/current/link" +touch -t 197001121346.40 "$tag_root/caos-used-${recent_digest#sha256:}/current/link" +blob_root="$fixture/data/stack/registry/docker/registry/v2/blobs/sha256" +for digest in "$seed_digest" "$stale_digest" "$recent_digest"; do + hex=${digest#sha256:} + mkdir -p "$blob_root/${hex:0:2}/$hex" + printf '{"schemaVersion":2,"layers":[]}' > "$blob_root/${hex:0:2}/$hex/data" +done +cleanup_out=$( + CAOS_DATA="$fixture/data" CAOS_IMAGE_CLEANUP_NOW=1000000 \ + CAOS_IMAGE_CLEANUP_MAX_BYTES=40 \ + bash "$CAOS_PROJECT/image-cleanup.sh" --unused-for=7d +) || fail "image cleanup dry run failed" +printf '%s\n' "$cleanup_out" | grep -q "KEEP $seed_digest current seed (runner)" \ + || fail "the current seed was not retained" +printf '%s\n' "$cleanup_out" | grep -q "DELETE $stale_digest" \ + || fail "the stale non-seed manifest was not selected" +printf '%s\n' "$cleanup_out" | grep -q "DELETE $recent_digest LRU over" \ + || fail "the recent manifest beyond the size ceiling was not selected" + echo "std-lint: ALL PASS" >&2 From cb4defe40dd64b8abcec348fe8433081f2d26bc7 Mon Sep 17 00:00:00 2001 From: Nishad Date: Wed, 19 Aug 2026 15:22:51 -0700 Subject: [PATCH 2/5] Keep CAOS data project-local by default --- README.md | 2 -- build-builtins.sh | 2 +- examples/consumer/README.md | 4 ++-- examples/consumer/flake.nix | 2 +- flake.nix | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fd7b898a..c11b59f1 100644 --- a/README.md +++ b/README.md @@ -603,8 +603,6 @@ spawning no container. - Build the stack with `nix build` - Run the dev stack with `result/bin/caosd up` -- Stack state is shared across worktrees under - `${XDG_CACHE_HOME:-$HOME/.cache}/caos`; set `CAOS_DATA` to override it. - **Check `caosd version` before believing a bug report.** A devShell that fails to build leaves direnv on the *previous* environment, so the `caosd` on PATH can be far older than the `flake.lock` that names it — and the symptom is an diff --git a/build-builtins.sh b/build-builtins.sh index 8e15ba1b..2a3ee15e 100755 --- a/build-builtins.sh +++ b/build-builtins.sh @@ -77,7 +77,7 @@ export CAOS_SERVER_URL=$SERVER_URL # shape a user has. `caos-cli` builds objects here (in-process via gix); `git # push` ships them to the server. Reused across runs (git init is idempotent). # CAOS_CLIENT_REPO relocates it off PROJECT (which is read-only when caosd runs -# us from the store); caosd points it at $CAOS_DATA so it persists across worktrees. +# us from the store); caosd points it at $CAOS_DATA so it persists per-project. CLIENT=${CAOS_CLIENT_REPO:-$PROJECT/.caos-dev/client-repo} git init -q "$CLIENT" # When the caller says this repo dies with the process (caos-tools/build.sh runs diff --git a/examples/consumer/README.md b/examples/consumer/README.md index c9d254ba..c1aa9dcc 100644 --- a/examples/consumer/README.md +++ b/examples/consumer/README.md @@ -18,8 +18,8 @@ of the relative path used here. nix develop # 1. Bring the stack up (redis + registry + caos server). Foreground; Ctrl-C -# stops it. Server state lives in $XDG_CACHE_HOME/caos (or ~/.cache/caos) — -# override with CAOS_DATA. This also publishes the builtin stdlib on startup. +# stops it. Server state (the bare git repo) lives in ./.caos-data — override +# with CAOS_DATA. This also publishes the builtin stdlib on startup. caosd up # 2. In another shell (also `nix develop`), add the server as the `caos` remote. diff --git a/examples/consumer/flake.nix b/examples/consumer/flake.nix index 466b3ee5..3f976fdd 100644 --- a/examples/consumer/flake.nix +++ b/examples/consumer/flake.nix @@ -25,7 +25,7 @@ # caosd — bring the stack up (foreground; Ctrl-C stops it). It also # (re)publishes the builtin worker library on each startup. # caos-cli — drive workers (run/get/curry/…) - # caosd honors CAOS_DATA (default $XDG_CACHE_HOME/caos) for the server's repo; + # caosd honors CAOS_DATA (default ./.caos-data) for the server's repo; # caos-cli must run inside a git working tree (this one) that has the # server as its `caos` remote — that remote URL *is* the server, so # there's no CAOS_SERVER_URL to set: diff --git a/flake.nix b/flake.nix index 0342c5e7..d8146ad4 100644 --- a/flake.nix +++ b/flake.nix @@ -724,7 +724,7 @@ pkgs.util-linux pkgs.diffutils pkgs.gnugrep pkgs.findutils pkgs.jq ]; text = '' - : "''${CAOS_DATA:=''${XDG_CACHE_HOME:-$HOME/.cache}/caos}" + : "''${CAOS_DATA:=$PWD/.caos-data}" CAOS_DATA="$(readlink -m "$CAOS_DATA")" export CAOS_DATA mkdir -p "$CAOS_DATA" From e9c76ffd70d02cc96453e94d4bf04d8e7a804c7d Mon Sep 17 00:00:00 2001 From: Nishad Date: Wed, 19 Aug 2026 15:35:17 -0700 Subject: [PATCH 3/5] Simplify image cleanup to a cache reset --- README.md | 2 +- caos-tools/build.sh | 8 +- crates/server/src/compute.rs | 224 ++--------------- design/image-cleanup.md | 50 ---- design/one-stack-image.md | 2 +- flake.nix | 85 ++++++- image-cleanup.sh | 460 ----------------------------------- tests/std-lint/cli.sh | 46 ---- 8 files changed, 111 insertions(+), 766 deletions(-) delete mode 100644 design/image-cleanup.md delete mode 100644 image-cleanup.sh diff --git a/README.md b/README.md index c11b59f1..3d0b5407 100644 --- a/README.md +++ b/README.md @@ -615,7 +615,7 @@ caosd up # bring the stack up + publish all of std, then return. Updates it caosd logs # follow the running stack's logs (Ctrl-C returns; stack stays up) caosd down # stop it (Redis + registry volumes and the server repo are kept) caosd reset # stop and wipe those volumes + the server repo for a clean slate -caosd image-cleanup # dry-run bounded registry LRU + disposable Docker cleanup +caosd image-cleanup # report cache usage; after `down`, add --execute to clear it caosd version # the caos revision this command was built from ``` diff --git a/caos-tools/build.sh b/caos-tools/build.sh index 5a1c2872..164118f1 100644 --- a/caos-tools/build.sh +++ b/caos-tools/build.sh @@ -86,9 +86,9 @@ reduce) caos put "$R" /cas/reduced # THE BUILD'S OWN INPUTS, and nothing else. `make` compiles the workspace, - # runs build-builtins.sh (which reads std/ and crates/worker-common), embeds - # image-cleanup.sh in caosd, and installs stack/serve and test-stack/worker - # into the image. It reads nothing else in the tree. + # runs build-builtins.sh (which reads std/ and crates/worker-common), and + # installs stack/serve and test-stack/worker into the image. It reads + # nothing else in the tree. # # It used to get the WHOLE tree, so a one-line edit to tests//cli.sh — # or to a design doc, or this comment — recompiled the workspace, @@ -106,7 +106,7 @@ reduce) S=/tmp/src rm -rf "$S"; mkdir -p "$S" for e in Cargo.toml Cargo.lock rust-toolchain.toml \ - crates std stack test-stack build-builtins.sh image-cleanup.sh; do + crates std stack test-stack build-builtins.sh; do if [ -e "$e" ]; then cp -RL --preserve=mode "$e" "$S/$e"; fi done caos put "$S" /cas/src diff --git a/crates/server/src/compute.rs b/crates/server/src/compute.rs index 22494d34..cb02bb14 100644 --- a/crates/server/src/compute.rs +++ b/crates/server/src/compute.rs @@ -32,7 +32,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{mpsc, Mutex, OnceLock}; -use std::time::{Duration, Instant}; +use std::time::Duration; use sha2::{Digest, Sha256}; @@ -43,16 +43,6 @@ use crate::{Config, HttpError}; /// digest, so the name is arbitrary and fixed. const REGISTRY_REPO: &str = "caos"; -/// A stable tag whose link mtime is the manifest's durable last-used clock. -/// Retagging the same immutable manifest changes no image bytes or digest, and -/// works from nested test servers that share the host registry but not its -/// filesystem or Redis. -const IMAGE_USED_TAG_PREFIX: &str = "caos-used-"; - -/// Avoid a registry write for every warm job while keeping a much tighter -/// bound than image-cleanup's default multi-day retention window. -const IMAGE_USED_REFRESH: Duration = Duration::from_secs(60 * 60); - /// Prefix marking the `image` parameter as an ordinary docker reference rather /// than one of our git images (the default). const DOCKER_SCHEME: &str = "docker://"; @@ -1389,37 +1379,32 @@ fn validate_docker_reference(reference: &str, allow_seeded: bool) -> Result<(), /// git images (the default): convert it to a real image, push it to the registry, /// and return a digest reference into the registry. fn resolve_image(config: &Config, image: &str) -> Result { - let reference = if let Some(reference) = image.strip_prefix(DOCKER_SCHEME) { + if let Some(reference) = image.strip_prefix(DOCKER_SCHEME) { validate_docker_reference(reference, true).map_err(|e| HttpError::new(400, e))?; - reference.to_string() - } else { - if !image.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(HttpError::new( - 400, - format!("git image must be a hex hash (or use {DOCKER_SCHEME}): {image:?}"), - )); - } - // The nested test stack (tests/lib/run-test.sh) runs on images the - // outer suite already built and pushed, so its server passes git images - // through unconverted: no OCI convert, no registry round-trip. The - // default keeps converting. - if std::env::var("CAOS_IMAGE_RESOLVE").as_deref() == Ok("none") { - return Ok(image.to_string()); - } - // A git image tree, converted. NOT a flake: do not add a branch here that - // notices `flake.nix` + `flake.lock` and builds it. Doing so needs a builder - // resolved BY NAME out of an ambient library, which is the one thing the - // server must not have — it holds an arg tree, not a project tree, so it - // cannot resolve a dependency by descent the way a client can. A flake - // directory says `run --base:@=DEEP-DEPS/flake-builder --in:@=.` and the CLIENT - // evaluates it, so what arrives here is already an image (design/caos-expr.md). - convert_git_image(config, image) - .map_err(|e| HttpError::new(500, format!("converting git image {image}: {e}")))? - }; - - mark_local_image_used(config, &reference) - .map_err(|e| HttpError::new(500, format!("recording image use for {reference}: {e}")))?; - Ok(reference) + return Ok(reference.to_string()); + } + if !image.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(HttpError::new( + 400, + format!("git image must be a hex hash (or use {DOCKER_SCHEME}): {image:?}"), + )); + } + // The nested test stack (tests/lib/run-test.sh) runs on images the + // outer suite already built and pushed, so its server passes git images + // through unconverted: no OCI convert, no registry round-trip. The + // default keeps converting. + if std::env::var("CAOS_IMAGE_RESOLVE").as_deref() == Ok("none") { + return Ok(image.to_string()); + } + // A git image tree, converted. NOT a flake: do not add a branch here that + // notices `flake.nix` + `flake.lock` and builds it. Doing so needs a builder + // resolved BY NAME out of an ambient library, which is the one thing the + // server must not have — it holds an arg tree, not a project tree, so it + // cannot resolve a dependency by descent the way a client can. A flake + // directory says `run --base:@=DEEP-DEPS/flake-builder --in:@=.` and the CLIENT + // evaluates it, so what arrives here is already an image (design/caos-expr.md). + convert_git_image(config, image) + .map_err(|e| HttpError::new(500, format!("converting git image {image}: {e}"))) } /// The lock for one cache key, minted on first use. A redis cache read followed @@ -1546,109 +1531,6 @@ fn image_ref(config: &Config, manifest_digest: &str) -> String { ) } -/// Return the digest when `reference` names this server's own registry repo. -/// The push and pull names differ in the host placement (`caos-registry` from -/// the server, `localhost` from Docker), so both spellings are local. -fn local_registry_digest<'a>( - push_url: &str, - pull_host: &str, - reference: &'a str, -) -> Option<&'a str> { - let (name, digest) = reference.rsplit_once('@')?; - let push = push_url.trim_end_matches('/'); - let push_host = push - .strip_prefix("http://") - .or_else(|| push.strip_prefix("https://")) - .unwrap_or(push); - let push_name = format!("{push_host}/{REGISTRY_REPO}"); - let pull_name = format!("{}/{REGISTRY_REPO}", pull_host.trim_end_matches('/')); - if name == push_name || name == pull_name { - Some(digest) - } else { - None - } -} - -fn image_used_tag(digest: &str) -> Result { - let encoded = digest - .strip_prefix("sha256:") - .ok_or_else(|| format!("unsupported manifest digest {digest}"))?; - if encoded.len() != 64 - || !encoded - .bytes() - .all(|b| b.is_ascii_digit() || matches!(b, b'a'..=b'f')) - { - return Err(format!("invalid manifest digest {digest}")); - } - Ok(format!("{IMAGE_USED_TAG_PREFIX}{encoded}")) -} - -/// Retag a local immutable manifest under its stable usage tag. Distribution -/// updates the tag link atomically, and its mtime is the cleanup clock. The -/// memo is only a write throttle; the durable record remains in the registry. -fn mark_local_image_used(config: &Config, reference: &str) -> Result<(), String> { - let Some(digest) = local_registry_digest( - &config.registry_push_url, - &config.registry_pull_host, - reference, - ) else { - return Ok(()); - }; - mark_manifest_used(config, digest) -} - -fn mark_manifest_used(config: &Config, digest: &str) -> Result<(), String> { - static RECENT: OnceLock>> = OnceLock::new(); - let memo_key = format!("{} {digest}", config.registry_push_url); - let recent = RECENT.get_or_init(|| Mutex::new(HashMap::new())); - if recent - .lock() - .unwrap_or_else(|e| e.into_inner()) - .get(&memo_key) - .is_some_and(|at| at.elapsed() < IMAGE_USED_REFRESH) - { - return Ok(()); - } - - let tag = image_used_tag(digest)?; - let base = config.registry_push_url.trim_end_matches('/'); - let source = format!("{base}/v2/{REGISTRY_REPO}/manifests/{digest}"); - let accept = "application/vnd.oci.image.manifest.v1+json, \ - application/vnd.docker.distribution.manifest.v2+json"; - let response = minreq::get(&source) - .with_header("Accept", accept) - .send() - .map_err(|e| format!("GET {source}: {e}"))?; - if !(200..300).contains(&response.status_code) { - return Err(format!( - "reading manifest {digest}: {} {}", - response.status_code, response.reason_phrase - )); - } - let content_type = response - .headers - .get("content-type") - .cloned() - .ok_or_else(|| format!("manifest {digest} response has no Content-Type"))?; - let target = format!("{base}/v2/{REGISTRY_REPO}/manifests/{tag}"); - let response = minreq::put(&target) - .with_header("Content-Type", content_type) - .with_body(response.as_bytes().to_vec()) - .send() - .map_err(|e| format!("PUT {target}: {e}"))?; - if !(200..300).contains(&response.status_code) { - return Err(format!( - "tagging manifest {digest} as used: {} {}", - response.status_code, response.reason_phrase - )); - } - recent - .lock() - .unwrap_or_else(|e| e.into_inner()) - .insert(memo_key, Instant::now()); - Ok(()) -} - /// Copy a base image (`base_ref`, a bare docker reference) from its source /// registry into our own repo with skopeo, so its blobs are available for a /// converted git image to reference. Returns the base's manifest layers @@ -1719,15 +1601,6 @@ fn fetch_base(config: &Config, base_ref: &str) -> Result { resp.status_code, resp.reason_phrase )); } - let base_manifest_digest = resp - .headers - .get("docker-content-digest") - .cloned() - .unwrap_or_else(|| format!("sha256:{}", sha256_hex(resp.as_bytes()))); - // A copied base is a registry artifact in its own right. Its functional - // `base-*` tag is a build memo, not a usage clock, so refresh the same - // stable usage tag as runnable images do. - mark_manifest_used(config, &base_manifest_digest)?; let manifest: serde_json::Value = serde_json::from_slice(resp.as_bytes()) .map_err(|e| format!("parsing base manifest: {e}"))?; let layers = manifest["layers"] @@ -2539,51 +2412,6 @@ mod single_flight_tests { } } -#[cfg(test)] -mod image_usage_tests { - use super::*; - - #[test] - fn recognizes_both_names_for_the_local_registry() { - let digest = format!("sha256:{}", "a".repeat(64)); - assert_eq!( - local_registry_digest( - "http://caos-registry:5000", - "localhost:5000", - &format!("caos-registry:5000/caos@{digest}") - ), - Some(digest.as_str()) - ); - assert_eq!( - local_registry_digest( - "http://caos-registry:5000/", - "localhost:5000/", - &format!("localhost:5000/caos@{digest}") - ), - Some(digest.as_str()) - ); - assert_eq!( - local_registry_digest( - "http://caos-registry:5000", - "localhost:5000", - &format!("ghcr.io/example/caos@{digest}") - ), - None - ); - } - - #[test] - fn usage_tag_is_stable_and_digest_specific() { - let digest = format!("sha256:{}", "b".repeat(64)); - assert_eq!( - image_used_tag(&digest).unwrap(), - format!("caos-used-{}", "b".repeat(64)) - ); - assert!(image_used_tag("sha256:short").is_err()); - assert!(image_used_tag(&format!("sha512:{}", "b".repeat(64))).is_err()); - } -} - #[cfg(test)] mod continuation_shape_tests { use super::*; diff --git a/design/image-cleanup.md b/design/image-cleanup.md deleted file mode 100644 index 300e59eb..00000000 --- a/design/image-cleanup.md +++ /dev/null @@ -1,50 +0,0 @@ -# Image cleanup - -CAOS has two image caches with different jobs: - -- Docker is the working set. Once no container uses a CAOS image, its local - copy is disposable because Docker can pull it back from the local registry. -- The registry is the backing cache. It retains recently used content up to a - size ceiling, plus correctness roots. - -`caosd image-cleanup` applies that distinction. It is a dry run unless passed -`--execute`. - -## Last used and the registry budget - -Immediately before dispatch, the server retags a local manifest as -`caos-used-`. The tag points at the same immutable manifest, -so it changes neither the image nor its digest. Replacing the tag updates -Distribution's tag-link mtime, which is the durable last-used clock. Each -server process writes at most once per manifest per hour. - -The record lives in the registry because host and nested test servers share the -registry but not a filesystem or Redis instance. A manifest without a usage -tag is treated as used now and receives a tag on the first executing cleanup. - -The default policy removes anything unused for seven days, then applies a -20 GiB ceiling to what remains. Non-root manifests are considered newest first. -Their manifest, config, and layer blobs are added to a set, so shared layers are -counted once; the least-recently-used closures that do not fit are removed. -`--unused-for=d` and `--max-size=GiB` override the defaults. - -## Roots and execution - -The direct image digests under `refs/caos/seed` are permanent roots for the -current std generation. A seed hit already promises that its digest exists, so -deleting it would turn a valid result into a later Docker pull failure. -Republishing std moves the ref and makes the old generation ordinary LRU data. - -`--execute` refuses while a `caos-worker-*` container is active. Otherwise it: - -1. stops the host stack if it was running and removes idle test stacks; -2. tags retained legacy manifests and deletes selected manifests; -3. runs Distribution's offline garbage collector when a manifest was deleted; -4. clears Redis only after registry deletion, because cached results can carry - deleted digests indirectly; -5. removes every unused local Docker copy of a CAOS registry image and obsolete - `caos-stack-src:*` tags; and -6. restores the host stack if it was running. - -Nix is a machine-wide store rather than a CAOS-owned cache. This command does -not run global Nix garbage collection. diff --git a/design/one-stack-image.md b/design/one-stack-image.md index 49a32dde..b980af54 100644 --- a/design/one-stack-image.md +++ b/design/one-stack-image.md @@ -178,7 +178,7 @@ caosd logs tail the group's logs caosd reset stop and wipe $CAOS_DATA caosd std-build publish std to this stack's registry and git, then exit caosd std-check verify what std references still exists; non-zero if not -caosd image-cleanup dry-run bounded registry LRU and local image cleanup +caosd image-cleanup report or clear rebuildable image caches while stopped ``` **Die as a group.** Any member's death takes the group down, so a half-dead diff --git a/flake.nix b/flake.nix index d8146ad4..8dfe3885 100644 --- a/flake.nix +++ b/flake.nix @@ -691,10 +691,9 @@ # working stack and not homework — so this is the strict # gate for anything that runs against a stack it did not # just bring up (design/one-stack-image.md). - # caosd image-cleanup dry-run bounded registry LRU and disposable - # local-image cleanup; --execute briefly stops and - # restores an idle stack. The current std seed is always - # retained; see design/image-cleanup.md. + # caosd image-cleanup report rebuildable image-cache usage; with + # --execute, clear it while the stack is stopped. The + # next `up` republishes std and warms images on demand. # caosd version the caos revision THIS command was built from. Ask it # before believing a bug report: a devShell that fails to # build leaves direnv on the previous environment, so the @@ -721,7 +720,7 @@ # and runnerd actually changed (stage_bins). runtimeInputs = [ pkgs.coreutils pkgs.git pkgs.curl pkgs.bash pkgs.skopeo pkgs.gzip - pkgs.util-linux pkgs.diffutils pkgs.gnugrep pkgs.findutils pkgs.jq + pkgs.util-linux pkgs.diffutils ]; text = '' : "''${CAOS_DATA:=$PWD/.caos-data}" @@ -891,6 +890,80 @@ echo "==> the seeded core is intact ($checked images)" >&2 } + # The registry and Redis are caches: git holds the image inputs and + # `up` republishes the irreducible std images. Clearing both avoids + # retaining a cached result that names a registry digest just removed. + # Require an idle stack instead of hiding stop/restart orchestration + # and recovery inside a cleanup command. + image_cleanup() { + local execute=no arg registry_size=0 image_id current_stack tag running + + for arg in "$@"; do + case "$arg" in + --execute) execute=yes ;; + -h|--help) + echo "usage: caosd image-cleanup [--execute]" + return + ;; + *) + echo "caosd image-cleanup: unknown argument '$arg'" >&2 + return 2 + ;; + esac + done + + if [ -d "$CAOS_DATA/stack/registry" ]; then + registry_size=$(du -sh "$CAOS_DATA/stack/registry" | cut -f1) + fi + echo "caosd image-cleanup: registry $registry_size" + docker image ls --format ' local image {{.ID}} {{.Size}}' \ + localhost:5000/caos | sort -u + docker image ls --format ' stack image {{.Repository}}:{{.Tag}} {{.Size}}' \ + caos-stack-src + if [ "$execute" != yes ]; then + echo "caosd image-cleanup: dry run; run 'caosd down' then pass --execute" + return + fi + + running=$( + while IFS= read -r tag; do + case "$tag" in + caos-stack|caos-worker-*|caos-test-stack-*) echo "$tag" ;; + esac + done < <(docker ps --format '{{.Names}}') + ) + if [ -n "$running" ]; then + echo "caosd image-cleanup: CAOS is still running:" >&2 + while IFS= read -r tag; do echo " $tag" >&2; done <<< "$running" + echo "run 'caosd down' and wait for active workers before cleanup" >&2 + return 1 + fi + + case "$CAOS_DATA" in + /|"") + echo "caosd image-cleanup: refusing unsafe CAOS_DATA '$CAOS_DATA'" >&2 + return 1 + ;; + esac + rm -rf "$CAOS_DATA/stack/registry" "$CAOS_DATA/stack/redis" + + while IFS= read -r image_id; do + [ -n "$image_id" ] || continue + docker image rm "$image_id" >/dev/null 2>&1 || true + done < <(docker image ls -q localhost:5000/caos | sort -u) + + current_stack=$(docker image inspect -f '{{.Id}}' caos-stack:latest 2>/dev/null || true) + while read -r tag image_id; do + if [ "$image_id" != "$current_stack" ]; then + docker image rm "$tag" >/dev/null 2>&1 || true + fi + done < <(docker image ls --no-trunc \ + --format '{{.Repository}}:{{.Tag}} {{.ID}}' caos-stack-src) + + echo "caosd image-cleanup: cleared the registry, Redis, and unused local CAOS images" + echo "caosd image-cleanup: run 'caosd up' to republish std" + } + # The revision this caosd was built from, printed by `version` and # by every usage banner. A stale binary on PATH is otherwise # indistinguishable from a caos bug (see `caosRev`). @@ -1017,7 +1090,7 @@ ;; image-cleanup) shift - bash ${./image-cleanup.sh} "$@" + image_cleanup "$@" ;; *) echo "caosd: unknown command '$1'" >&2 diff --git a/image-cleanup.sh b/image-cleanup.sh deleted file mode 100644 index 86596618..00000000 --- a/image-cleanup.sh +++ /dev/null @@ -1,460 +0,0 @@ -#!/usr/bin/env bash -# Bound CAOS's two image caches. Docker is only the local working set, so every -# unused CAOS image there is disposable. The registry is the backing cache: it -# keeps the most recently used manifest closures up to a size ceiling, plus the -# current std seed regardless of age or size. -set -euo pipefail -export LC_ALL=C - -: "${CAOS_DATA:?image-cleanup needs CAOS_DATA}" - -case "$CAOS_DATA" in - /|"") - echo "caosd image-cleanup: refusing unsafe CAOS_DATA '$CAOS_DATA'" >&2 - exit 1 - ;; -esac - -STATE="$CAOS_DATA/stack" -CLIENT="$CAOS_DATA/publish-client-repo" -REPOSITORY=caos -USED_TAG_PREFIX=caos-used- -DEFAULT_UNUSED_FOR=7d -DEFAULT_MAX_SIZE=20GiB -GIB=$((1024 * 1024 * 1024)) - -execute=no -unused_for=$DEFAULT_UNUSED_FOR -max_size=$DEFAULT_MAX_SIZE - -usage() { - echo "usage: caosd image-cleanup [--unused-for=d] [--max-size=GiB] [--execute]" >&2 -} - -fail() { - echo "caosd image-cleanup: $*" >&2 - exit 1 -} - -mtime() { - if stat -c %Y "$1" 2>/dev/null; then - return - fi - stat -f %m "$1" -} - -format_epoch() { - if date -u -d "@$1" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null; then - return - fi - date -u -r "$1" '+%Y-%m-%dT%H:%M:%SZ' -} - -format_bytes() { - local bytes=$1 whole tenth - whole=$((bytes / GIB)) - tenth=$(((bytes % GIB) * 10 / GIB)) - printf '%d.%dGiB' "$whole" "$tenth" -} - -for arg in "$@"; do - case "$arg" in - --execute) - execute=yes - ;; - --unused-for=*) - unused_for=${arg#--unused-for=} - ;; - --max-size=*) - max_size=${arg#--max-size=} - ;; - -h|--help) - usage - exit 0 - ;; - *) - usage - fail "unknown argument '$arg'" - ;; - esac -done - -if [[ ! "$unused_for" =~ ^[0-9]+d$ ]]; then - fail "--unused-for must be a whole number of days such as 7d" -fi -unused_days=${unused_for%d} -if [ "$unused_days" -lt 1 ]; then - fail "--unused-for must be at least 1d" -fi -if [[ ! "$max_size" =~ ^[0-9]+GiB$ ]]; then - fail "--max-size must be whole GiB such as 20GiB" -fi -max_gib=${max_size%GiB} -if [ "$max_gib" -lt 1 ]; then - fail "--max-size must be at least 1GiB" -fi -max_bytes=${CAOS_IMAGE_CLEANUP_MAX_BYTES:-$((max_gib * GIB))} -if [[ ! "$max_bytes" =~ ^[0-9]+$ ]] || [ "$max_bytes" -lt 1 ]; then - fail "CAOS_IMAGE_CLEANUP_MAX_BYTES must be positive bytes" -fi -budget=$(format_bytes "$max_bytes") - -[ -d "$STATE/registry" ] || fail "no registry under $STATE (run 'caosd up' first)" -[ -d "$CLIENT" ] || fail "no publish client under $CLIENT (run 'caosd up' first)" - -manifest_root="$STATE/registry/docker/registry/v2/repositories/$REPOSITORY/_manifests" -revision_root="$manifest_root/revisions/sha256" -tag_root="$manifest_root/tags" -blob_root="$STATE/registry/docker/registry/v2/blobs/sha256" -[ -d "$revision_root" ] || fail "registry has no $REPOSITORY manifest revisions" - -scratch=$(mktemp -d "$STATE/image-cleanup.XXXXXX") -inventory="$scratch/inventory" -candidates="$scratch/candidates" -candidate_report="$scratch/candidate-report" -retained="$scratch/retained" -: > "$inventory" -: > "$candidates" -: > "$candidate_report" -: > "$retained" - -registry_container="" -gc_container="" -stack_stopped=no -finish() { - local rc=$? - trap - EXIT - if [ -n "$registry_container" ] && docker container inspect "$registry_container" >/dev/null 2>&1; then - docker container stop "$registry_container" >/dev/null 2>&1 || true - docker container rm "$registry_container" >/dev/null 2>&1 || true - fi - if [ -n "$gc_container" ] && docker container inspect "$gc_container" >/dev/null 2>&1; then - docker container stop "$gc_container" >/dev/null 2>&1 || true - docker container rm "$gc_container" >/dev/null 2>&1 || true - fi - rm -rf "$scratch" - if [ "$stack_stopped" = yes ]; then - if docker container start caos-stack >/dev/null 2>&1; then - ready="" - for _ in $(seq 1 60); do - if curl -s -o /dev/null --max-time 2 http://localhost:9090/; then - ready=yes - break - fi - sleep 1 - done - if [ -n "$ready" ]; then - echo "caosd image-cleanup: restarted caos-stack" - else - echo "caosd image-cleanup: caos-stack did not become ready after restart" >&2 - rc=1 - fi - else - echo "caosd image-cleanup: could not restart caos-stack" >&2 - rc=1 - fi - fi - exit "$rc" -} -trap finish EXIT - -declare -A protected=() -seed=$(git -C "$CLIENT" rev-parse --verify -q refs/caos/seed) \ - || fail "no refs/caos/seed (run 'caosd up' first)" -protected_count=0 -while read -r _mode _kind oid name; do - base=$(git -C "$CLIENT" cat-file -p "$oid:result/base" 2>/dev/null) || continue - digest=${base##*@} - if [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] && [ -z "${protected[$digest]:-}" ]; then - protected[$digest]=$name - protected_count=$((protected_count + 1)) - fi -done < <(git -C "$CLIENT" ls-tree "$seed") -[ "$protected_count" -gt 0 ] || fail "the current std seed names no registry manifests" - -now=${CAOS_IMAGE_CLEANUP_NOW:-$(date +%s)} -if [[ ! "$now" =~ ^[0-9]+$ ]]; then - fail "CAOS_IMAGE_CLEANUP_NOW must be Unix seconds" -fi -cutoff=$((now - unused_days * 86400)) -manifest_count=0 -untracked_count=0 - -while IFS= read -r -d '' revision_link; do - digest_hex=${revision_link%/link} - digest_hex=${digest_hex##*/} - digest="sha256:$digest_hex" - recorded=$(<"$revision_link") - [ "$recorded" = "$digest" ] || fail "manifest revision $revision_link records $recorded" - manifest_count=$((manifest_count + 1)) - - marker="$tag_root/$USED_TAG_PREFIX$digest_hex/current/link" - if [ -n "${protected[$digest]:-}" ]; then - printf 'seed\t%s\t%s\t%s\n' "$now" "$digest" "${protected[$digest]}" >> "$inventory" - elif [ -f "$marker" ]; then - marker_digest=$(<"$marker") - [ "$marker_digest" = "$digest" ] \ - || fail "usage marker $marker records $marker_digest, expected $digest" - printf 'tracked\t%s\t%s\t-\n' "$(mtime "$marker")" "$digest" >> "$inventory" - else - # No creation timestamp is trustworthy for reproducible images. Treat a - # legacy manifest as used now; --execute gives it a durable marker. - printf 'untracked\t%s\t%s\t-\n' "$now" "$digest" >> "$inventory" - untracked_count=$((untracked_count + 1)) - fi -done < <(find "$revision_root" -mindepth 2 -maxdepth 2 -type f -name link -print0) - -blob_path() { - local digest=$1 hex=${1#sha256:} - [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid registry blob digest $digest" - printf '%s/%s/%s/data\n' "$blob_root" "${hex:0:2}" "$hex" -} - -declare -A kept_blobs=() -closure_digests=() -closure_increment=0 -plan_closure() { - local digest=$1 manifest_blob blob path size - manifest_blob=$(blob_path "$digest") - [ -f "$manifest_blob" ] || fail "manifest blob $digest is missing" - jq -r '(.config.digest? // empty), (.layers[]?.digest // empty)' \ - "$manifest_blob" > "$scratch/descriptors" \ - || fail "manifest blob $digest is not valid JSON" - { - printf '%s\n' "$digest" - while IFS= read -r blob; do printf '%s\n' "$blob"; done < "$scratch/descriptors" - } | sort -u > "$scratch/closure" \ - || fail "sorting the blob closure for $digest" - closure_digests=() - while IFS= read -r blob; do - [ -n "$blob" ] || continue - closure_digests+=("$blob") - done < "$scratch/closure" - closure_increment=0 - for blob in "${closure_digests[@]}"; do - if [ -n "${kept_blobs[$blob]:-}" ]; then - continue - fi - path=$(blob_path "$blob") - [ -f "$path" ] || fail "blob $blob referenced by $digest is missing" - size=$(stat -c %s "$path" 2>/dev/null || stat -f %z "$path") - closure_increment=$((closure_increment + size)) - done -} - -retain_planned_closure() { - local blob - for blob in "${closure_digests[@]}"; do - kept_blobs[$blob]=1 - done -} - -# Roots enter first and may exceed the ceiling: correctness beats the cache -# budget. Everything else is considered newest-first, yielding a true LRU over -# the union of manifest/config/layer blobs rather than double-counting layers. -retained_bytes=0 -while IFS=$'\t' read -r _kind _used digest detail; do - plan_closure "$digest" - retain_planned_closure - retained_bytes=$((retained_bytes + closure_increment)) - printf '%s\n' "$digest" >> "$retained" - printf ' KEEP %s current seed (%s)\n' "$digest" "$detail" -done < <(grep '^seed' "$inventory" || true) -roots_over_budget=no -if [ "$retained_bytes" -gt "$max_bytes" ]; then - roots_over_budget=yes -fi - -candidate_count=0 -while IFS=$'\t' read -r kind used digest _detail; do - if [ "$used" -lt "$cutoff" ]; then - printf '%s\n' "$digest" >> "$candidates" - printf 'age\t%s\t%s\n' "$used" "$digest" >> "$candidate_report" - candidate_count=$((candidate_count + 1)) - continue - fi - - plan_closure "$digest" - if [ $((retained_bytes + closure_increment)) -gt "$max_bytes" ]; then - printf '%s\n' "$digest" >> "$candidates" - printf 'size\t%s\t%s\n' "$used" "$digest" >> "$candidate_report" - candidate_count=$((candidate_count + 1)) - continue - fi - retain_planned_closure - retained_bytes=$((retained_bytes + closure_increment)) - printf '%s\n' "$digest" >> "$retained" -done < <(grep -v '^seed' "$inventory" | sort -t $'\t' -k2,2nr -k3,3 || true) - -echo "caosd image-cleanup: $manifest_count manifests; $protected_count seed roots; $candidate_count selected; $untracked_count without history" -echo "caosd image-cleanup: planned registry content $(format_bytes "$retained_bytes") / $budget; age limit $unused_for" -if [ "$roots_over_budget" = yes ]; then - echo "caosd image-cleanup: seed roots alone exceed the registry budget" >&2 -fi -while IFS=$'\t' read -r reason used digest; do - stamp=$(format_epoch "$used") - case "$reason" in - age) printf ' DELETE %s last used %s\n' "$digest" "$stamp" ;; - size) printf ' DELETE %s LRU over %s (last used %s)\n' "$digest" "$budget" "$stamp" ;; - esac -done < "$candidate_report" - -if [ "$execute" = no ]; then - echo "caosd image-cleanup: dry run; pass --execute to apply" - exit 0 -fi - -# Stop accepting work before touching either cache. A worker already running is -# left alone; the caller can retry once it finishes. With no worker containers, -# a test stack is idle and safe to remove. -running=$(docker ps --format '{{.Names}}' | grep -E '^caos-worker-' || true) -if [ -n "$running" ]; then - echo "caosd image-cleanup: active workers:" >&2 - while IFS= read -r name; do echo " $name" >&2; done <<< "$running" - fail "wait for active work to finish before --execute" -fi - -if docker container inspect caos-registry-cleanup >/dev/null 2>&1 \ - || docker container inspect caos-registry-gc >/dev/null 2>&1; then - fail "a previous cleanup container remains; inspect and remove it first" -fi - -if [ "$(docker inspect -f '{{.State.Running}}' caos-stack 2>/dev/null || true)" = true ]; then - docker container stop caos-stack >/dev/null - stack_stopped=yes -fi - -running=$(docker ps --format '{{.Names}}' | grep -E '^caos-worker-' || true) -[ -z "$running" ] || fail "a worker started while the stack was stopping; retry cleanup" - -while IFS= read -r container_id; do - [ -n "$container_id" ] || continue - docker container rm -f "$container_id" >/dev/null -done < <(docker ps -aq --filter name=caos-test-stack-) - -if [ "$candidate_count" -gt 0 ] || [ "$untracked_count" -gt 0 ]; then - config="$scratch/registry.yml" - cat > "$config" </dev/null - - ready="" - for _ in $(seq 1 30); do - if curl -fs -o /dev/null http://localhost:5000/v2/; then - ready=yes - break - fi - sleep 1 - done - [ -n "$ready" ] || fail "temporary registry did not become ready" - - accept='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' - while IFS= read -r digest; do - digest_hex=${digest#sha256:} - marker="$tag_root/$USED_TAG_PREFIX$digest_hex/current/link" - if [ -f "$marker" ]; then - continue - fi - curl -fsS -D "$scratch/headers" -o "$scratch/manifest" \ - -H "Accept: $accept" \ - "http://localhost:5000/v2/$REPOSITORY/manifests/$digest" - content_type="" - while IFS= read -r header; do - header=${header%$'\r'} - case "$header" in - Content-Type:*|content-type:*) - content_type=${header#*:} - content_type=${content_type# } - ;; - esac - done < "$scratch/headers" - [ -n "$content_type" ] || fail "manifest $digest response has no Content-Type" - curl -fsS -o /dev/null -X PUT \ - -H "Content-Type: $content_type" \ - --data-binary @"$scratch/manifest" \ - "http://localhost:5000/v2/$REPOSITORY/manifests/$USED_TAG_PREFIX$digest_hex" - done < "$retained" - - while IFS= read -r digest; do - code=$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE \ - "http://localhost:5000/v2/$REPOSITORY/manifests/$digest") - case "$code" in - 202|404) ;; - *) fail "deleting $digest returned HTTP $code" ;; - esac - done < "$candidates" - - docker container stop "$registry_container" >/dev/null - docker container rm "$registry_container" >/dev/null - registry_container="" - - if [ "$candidate_count" -gt 0 ]; then - # A cached result can contain a now-deleted registry digest indirectly, so - # invalidate the cache only when deletion actually happened. - rm -rf "$STATE/redis" - mkdir -p "$STATE/redis" - - gc_container=caos-registry-gc - if ! docker run --name "$gc_container" \ - -v "$STATE:/state" \ - --entrypoint /bin/registry \ - caos-stack:latest garbage-collect --quiet --delete-untagged "$config_in_container" \ - > "$scratch/gc.log" 2>&1; then - tail -n 30 "$scratch/gc.log" >&2 - fail "registry garbage collection failed" - fi - docker container rm "$gc_container" >/dev/null - gc_container="" - fi -fi - -# Docker is a working set in front of the local registry. Once CAOS is idle, -# every pulled CAOS image can go; a later run restores it from the registry. -docker_removed=0 -while IFS= read -r image_id; do - [ -n "$image_id" ] || continue - local_caos="" - while IFS= read -r reference; do - case "$reference" in - localhost:5000/caos@sha256:*|caos-registry:5000/caos@sha256:*) - local_caos=yes - break - ;; - esac - done < <(docker image inspect -f '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id") - if [ -n "$local_caos" ] && docker image rm "$image_id" >/dev/null 2>&1; then - docker_removed=$((docker_removed + 1)) - fi -done < <(docker image ls -q --no-trunc | sort -u) - -# load_once's content tags avoid repeated docker loads, but only the tag for the -# current stack image is useful. -current_stack=$(docker image inspect -f '{{.Id}}' caos-stack:latest 2>/dev/null || true) -while read -r tag image_id; do - case "$tag" in - caos-stack-src:*) - if [ "$image_id" != "$current_stack" ]; then - docker image rm "$tag" >/dev/null 2>&1 || true - fi - ;; - esac -done < <(docker image ls --no-trunc --format '{{.Repository}}:{{.Tag}} {{.ID}}') - -registry_bytes=$(du -sk "$STATE/registry" | while read -r kib _rest; do echo $((kib * 1024)); done) -echo "caosd image-cleanup: deleted $candidate_count registry manifests; removed $docker_removed local CAOS images" -echo "caosd image-cleanup: registry now uses $(format_bytes "$registry_bytes")" diff --git a/tests/std-lint/cli.sh b/tests/std-lint/cli.sh index fb29701d..3eca722b 100644 --- a/tests/std-lint/cli.sh +++ b/tests/std-lint/cli.sh @@ -32,50 +32,4 @@ echo "== lint-bake-anchor.sh: every std tool's crates.io deps are anchored ==" > bash "$CAOS_PROJECT/lint-bake-anchor.sh" \ || fail "a std tool's crates.io dep is missing from bake-anchor (see above)" -echo "== image-cleanup.sh: seed roots, age, and LRU size bound ==" >&2 -fixture=$(mktemp -d) -repo="$fixture/data/publish-client-repo" -mkdir -p "$repo" -git init -q "$repo" -seed_digest=sha256:$(printf 'a%.0s' {1..64}) -stale_digest=sha256:$(printf 'b%.0s' {1..64}) -recent_digest=sha256:$(printf 'c%.0s' {1..64}) -base_blob=$(printf 'docker://localhost:5000/caos@%s' "$seed_digest" \ - | git -C "$repo" hash-object -w --stdin) -result_tree=$(printf '100644 blob %s\tbase\n' "$base_blob" | git -C "$repo" mktree) -record_tree=$(printf '040000 tree %s\tresult\n' "$result_tree" | git -C "$repo" mktree) -seed_tree=$(printf '040000 tree %s\trunner\n' "$record_tree" | git -C "$repo" mktree) -git -C "$repo" update-ref refs/caos/seed "$seed_tree" -revision_root="$fixture/data/stack/registry/docker/registry/v2/repositories/caos/_manifests/revisions/sha256" -tag_root="$fixture/data/stack/registry/docker/registry/v2/repositories/caos/_manifests/tags" -mkdir -p "$revision_root/${seed_digest#sha256:}" \ - "$revision_root/${stale_digest#sha256:}" \ - "$revision_root/${recent_digest#sha256:}" \ - "$tag_root/caos-used-${stale_digest#sha256:}/current" \ - "$tag_root/caos-used-${recent_digest#sha256:}/current" -printf '%s' "$seed_digest" > "$revision_root/${seed_digest#sha256:}/link" -printf '%s' "$stale_digest" > "$revision_root/${stale_digest#sha256:}/link" -printf '%s' "$recent_digest" > "$revision_root/${recent_digest#sha256:}/link" -printf '%s' "$stale_digest" > "$tag_root/caos-used-${stale_digest#sha256:}/current/link" -printf '%s' "$recent_digest" > "$tag_root/caos-used-${recent_digest#sha256:}/current/link" -touch -t 197001010000.01 "$tag_root/caos-used-${stale_digest#sha256:}/current/link" -touch -t 197001121346.40 "$tag_root/caos-used-${recent_digest#sha256:}/current/link" -blob_root="$fixture/data/stack/registry/docker/registry/v2/blobs/sha256" -for digest in "$seed_digest" "$stale_digest" "$recent_digest"; do - hex=${digest#sha256:} - mkdir -p "$blob_root/${hex:0:2}/$hex" - printf '{"schemaVersion":2,"layers":[]}' > "$blob_root/${hex:0:2}/$hex/data" -done -cleanup_out=$( - CAOS_DATA="$fixture/data" CAOS_IMAGE_CLEANUP_NOW=1000000 \ - CAOS_IMAGE_CLEANUP_MAX_BYTES=40 \ - bash "$CAOS_PROJECT/image-cleanup.sh" --unused-for=7d -) || fail "image cleanup dry run failed" -printf '%s\n' "$cleanup_out" | grep -q "KEEP $seed_digest current seed (runner)" \ - || fail "the current seed was not retained" -printf '%s\n' "$cleanup_out" | grep -q "DELETE $stale_digest" \ - || fail "the stale non-seed manifest was not selected" -printf '%s\n' "$cleanup_out" | grep -q "DELETE $recent_digest LRU over" \ - || fail "the recent manifest beyond the size ceiling was not selected" - echo "std-lint: ALL PASS" >&2 From 62539b5b1829d1df14ded6a7a6c60f8daba2e666 Mon Sep 17 00:00:00 2001 From: Nishad Date: Wed, 19 Aug 2026 16:53:28 -0700 Subject: [PATCH 4/5] Use Docker ownership metadata for cleanup --- flake.nix | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/flake.nix b/flake.nix index 8dfe3885..c12f5ec9 100644 --- a/flake.nix +++ b/flake.nix @@ -737,7 +737,7 @@ # multi-second `docker load`; a changed build has a new store path, # hence a new tag, and loads. load_once() { - local name="$1" image="$2" src_tag + local name="$1" image="$2" src_tag old_tag src_tag="$name-src:$(printf '%s' "$image" | sha1sum | cut -c1-12)" if docker image inspect "$src_tag" >/dev/null 2>&1; then echo "==> $name image already loaded — skipping docker load" >&2 @@ -755,6 +755,14 @@ # the core-seeder-runner existed, so every seeded key fell through # to the generic runner and died pulling `seeded:latest`. docker tag "$src_tag" "$name:latest" + + # This function owns the content-addressed source tags. Retire + # superseded ones here instead of rediscovering them at cleanup. + while IFS= read -r old_tag; do + if [ "$old_tag" != "$src_tag" ]; then + docker image rm "$old_tag" >/dev/null 2>&1 || true + fi + done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' "$name-src") } die() { # @@ -896,7 +904,7 @@ # Require an idle stack instead of hiding stop/restart orchestration # and recovery inside a cleanup command. image_cleanup() { - local execute=no arg registry_size=0 image_id current_stack tag running + local execute=no arg registry_size=0 image_id running for arg in "$@"; do case "$arg" in @@ -918,19 +926,17 @@ echo "caosd image-cleanup: registry $registry_size" docker image ls --format ' local image {{.ID}} {{.Size}}' \ localhost:5000/caos | sort -u - docker image ls --format ' stack image {{.Repository}}:{{.Tag}} {{.Size}}' \ - caos-stack-src if [ "$execute" != yes ]; then echo "caosd image-cleanup: dry run; run 'caosd down' then pass --execute" return fi running=$( - while IFS= read -r tag; do - case "$tag" in - caos-stack|caos-worker-*|caos-test-stack-*) echo "$tag" ;; - esac - done < <(docker ps --format '{{.Names}}') + if [ "$(docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null || true)" = true ]; then + echo "$NAME" + fi + docker ps --filter label=caos.runnerd.owner --format '{{.Names}}' + docker ps --filter label=caos.test-stack --format '{{.Names}}' ) if [ -n "$running" ]; then echo "caosd image-cleanup: CAOS is still running:" >&2 @@ -952,14 +958,6 @@ docker image rm "$image_id" >/dev/null 2>&1 || true done < <(docker image ls -q localhost:5000/caos | sort -u) - current_stack=$(docker image inspect -f '{{.Id}}' caos-stack:latest 2>/dev/null || true) - while read -r tag image_id; do - if [ "$image_id" != "$current_stack" ]; then - docker image rm "$tag" >/dev/null 2>&1 || true - fi - done < <(docker image ls --no-trunc \ - --format '{{.Repository}}:{{.Tag}} {{.ID}}' caos-stack-src) - echo "caosd image-cleanup: cleared the registry, Redis, and unused local CAOS images" echo "caosd image-cleanup: run 'caosd up' to republish std" } From 134fee6cfdfb454723dccb503dfe8b0992730099 Mon Sep 17 00:00:00 2001 From: Nishad Date: Wed, 19 Aug 2026 17:11:56 -0700 Subject: [PATCH 5/5] Centralize the local registry endpoint --- flake.nix | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index b6729ced..01b26054 100644 --- a/flake.nix +++ b/flake.nix @@ -730,6 +730,9 @@ NET=caos-net NAME=caos-stack + REGISTRY_PORT=5000 + REGISTRY=localhost:$REGISTRY_PORT + REGISTRY_REPO=$REGISTRY/caos # Load the stack image only when this exact build isn't already in # docker. The tag is content-addressed (sha1 of the image's @@ -826,6 +829,7 @@ std_build() { echo "==> publishing stdlib (build-builtins.sh)" >&2 CAOS_SERVER_URL=http://localhost:9090 \ + CAOS_REGISTRY_HTTP="$REGISTRY" \ CAOS_CLI=${caos-cli}/bin/caos-cli \ CAOS_CLIENT_REPO="$CLIENT" \ CAOS_BUILTIN_IMAGES="${ @@ -848,7 +852,7 @@ # source directory carrying a `.caos-expr`, so it names no digest at # all and a walk over entries would verify nothing. # - # Checked through localhost:5000 — the name the DOCKER DAEMON pulls + # Checked through $REGISTRY — the name the DOCKER DAEMON pulls # with. The seed's refs spell the same registry caos-registry:5000, # which is how the SERVER reaches it; one registry, two names, so the # check says which one it used. @@ -856,7 +860,7 @@ # `checked` must stay: a guard that silently verifies nothing is worse # than no guard, so finding no digests is itself an error. std_check() { - local reg=localhost:5000 tree missing=0 checked=0 oid name base digest code + local reg="$REGISTRY" tree missing=0 checked=0 oid name base digest code [ -d "$CLIENT" ] \ || { echo "caosd: no client repo at $CLIENT — run 'caosd std-build'" >&2; exit 1; } tree=$(git -C "$CLIENT" rev-parse --verify -q refs/caos/seed) \ @@ -925,7 +929,7 @@ fi echo "caosd image-cleanup: registry $registry_size" docker image ls --format ' local image {{.ID}} {{.Size}}' \ - localhost:5000/caos | sort -u + "$REGISTRY_REPO" | sort -u if [ "$execute" != yes ]; then echo "caosd image-cleanup: dry run; run 'caosd down' then pass --execute" return @@ -956,7 +960,7 @@ while IFS= read -r image_id; do [ -n "$image_id" ] || continue docker image rm "$image_id" >/dev/null 2>&1 || true - done < <(docker image ls -q localhost:5000/caos | sort -u) + done < <(docker image ls -q "$REGISTRY_REPO" | sort -u) echo "caosd image-cleanup: cleared the registry, Redis, and unused local CAOS images" echo "caosd image-cleanup: run 'caosd up' to republish std" @@ -1032,7 +1036,7 @@ --network-alias caos-server \ --network-alias caos-registry \ --network-alias caos-redis \ - -p 9090:80 -p 5000:5000 \ + -p 9090:80 -p "$REGISTRY_PORT:5000" \ -v "$CAOS_DATA/stack:/state" \ -v /var/run/docker.sock:/var/run/docker.sock \ -e CAOS_STACK_STATE=/state \ @@ -1045,6 +1049,7 @@ -e CAOS_STACK_SEEDER=yes \ -e CAOS_STACK_RUNNER_SERVER_URL=http://caos-server \ -e CAOS_STACK_RUNNER_REDIS_ADDR=caos-redis:6379 \ + -e CAOS_REGISTRY_PULL_HOST="$REGISTRY" \ -e CAOS_DOCKER_NETWORK="$NET" \ -e CAOS_RUNNER_SOCKET=/var/run/docker.sock \ -e CAOS_PENDING_TIMEOUT_SECS=900 \