#!/usr/bin/env bash # Regression test for prune-cache.sh. Builds a real scratch git repo standing # in for `origin` and a real scratch directory standing in for the cache root, # then runs the ACTUAL script against both — not a simulation of its logic. # # What each scenario demonstrates, and why the controls matter as much as the # fixes (a scenario that always passes proves nothing): # # 1. DEAD BRANCH PRUNED, not gated on disk pressure — a cache whose branch # no longer exists on origin is removed even with plenty of free space. # Waiting for pressure to notice means paying for dead caches until then. # 2. LIVE BRANCH SURVIVES despite being OLDER than the dead one — liveness, # not age, is what decides pass 1. # 3. PROTECTED REFS NEVER EVICTED under forced disk pressure, even when # their caches are the oldest on disk and would rank first for LRU. # 4. LOCKED CACHE PROTECTED even when dead, old, and under pressure — and # the pass's closing summary agrees with the decline it just logged, # rather than reporting that it found nothing. # 5. STALE LOCK NOT HONOURED FOREVER — the same cache with a lock older than # STALE_LOCK_SECONDS is evicted, so a crashed job cannot pin a directory # permanently. # 6. LIVENESS UNAVAILABLE FAILS SAFE — origin unreachable: a genuinely dead # cache is NOT pruned, the log says so plainly, and the pressure fallback # still works independently. "Unavailable" degrades to pressure-only, not # to no eviction at all. # 7. TARGET DIRS EVICTED BEFORE SNAPSHOTS — the ordering that differs from # the obvious one. A snapshot is hardlinked to the caches cloned from it, # so evicting it frees almost nothing while costing every future PR its # warm start. # 8. SELF-CLEAR REPORTS LOUDLY to the job summary, not just a log warning. # 9. OWN CACHE NEVER EVICTED by a sibling pass. # 10. SCOPED TO THE CACHE ROOT — a decoy outside it (standing in for another # project's volume) is never touched. # 11. A LIVE READER MARKER PROTECTS A CACHE the same way a lock file does — a # directory somebody is hardlink-cloning this instant is not a candidate, # however dead and however tight the disk. # 12. AND SO DOES ONE PUBLISHED INSIDE THE CHECK-TO-UNLINK WINDOW, which is # the property a check-then-delete eviction does NOT have. This is the # one that fails against the pre-fix script. # 13. A DEFERRED EVICTION IS RECLAIMED, but not while its reader is live. # Nothing else globs a dotted name, so an unswept one is disk lost for # good on the volume whose whole problem is disk. # 14. AND NOT WHILE ANOTHER PASS MAY STILL BE EVICTING IT. An aside with no # readers is indistinguishable from one a concurrent pass has just # renamed and not yet decided about; reclaiming that one lets `rm -rf` # empty a tree its owner may still restore under a live cache name. Its # fixture is an OLD directory renamed a moment ago — production's shape, # and what lets it tell the two timestamps apart. # 15. A MERGED-BUT-UNDELETED BRANCH IS DEAD TOO. This forge keeps branches # after merge, so `ls-remote` reports them forever and scenario 1's # signal never fires for them — which is how three 40 GB caches sat on a # full volume until somebody removed them by hand (gitdan-actions#20). A # branch whose tip is an ancestor of a protected branch's tip is pruned # like a deleted one; an unmerged branch beside it is not. # 16. AND "CANNOT TELL" IS STILL NOT DEATH, at both granularities: a branch # whose tip is not in this checkout is kept with a warning naming it, # and a shallow checkout — where a missing object is the normal case — # withholds the whole signal rather than reading it as "nothing merged". # The deleted-branch signal keeps working in both. # 17. THE FREE-SPACE REQUIREMENT IS MEASURED OFF THE SOURCE, not taken as a # percentage of the volume: the pass evicts until the clone the seed is # about to make fits, and stops there rather than draining the volume. # When it cannot get there it FAILS, naming the shortfall and every # directory it kept instead — because the seed would otherwise fail # seconds later against a staging path that names nothing. # 18. AND ON THE LAYOUT THAT PRODUCED THE BUG: three equal-sized caches, one # of them a merged-but-undeleted branch's, with disk to spare. Exactly # that one goes. Equal sizes and no pressure are the point — nothing but # the merge state can be what decides. set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$script_dir/cache-lib.sh" prune="$script_dir/prune-cache.sh" # Two scenarios below put a stub of a real tool on PATH for one command. # Captured once, here, rather than read back at each of those sites: a `$PATH` # read after the first of them is indistinguishable, to a static check, from # reading the modification the subshell lost. outer_path="$PATH" scratch=$(mktemp -d) trap 'rm -rf "$scratch"' EXIT pass_count=0 fail() { echo "ASSERTION FAILED: $*" >&2; [ -n "${1:-}" ] && [ -f "$scratch/log" ] && { echo "--- log ---" >&2; cat "$scratch/log" >&2; }; exit 1; } ok() { pass_count=$((pass_count + 1)); echo "PASS: $*"; } assert_gone() { [ -e "$1" ] && fail "expected gone: $1 ($2)"; ok "$2"; } assert_kept() { [ -e "$1" ] || fail "expected kept: $1 ($2)"; ok "$2"; } assert_log() { grep -q -- "$1" "$scratch/log" || fail "expected in log: $1 ($2)"; ok "$2"; } echo "=== building a scratch origin with real branches ===" origin="$scratch/origin.git"; git init -q --bare "$origin" work="$scratch/work"; git init -q "$work" ( cd "$work" git -c user.email=t@t -c user.name=t -c commit.gpgsign=false commit -q --allow-empty -m init git branch -M main git checkout -q -b dev; git -c user.email=t@t -c user.name=t -c commit.gpgsign=false commit -q --allow-empty -m dev git checkout -q -b feat/live; git -c user.email=t@t -c user.name=t -c commit.gpgsign=false commit -q --allow-empty -m live git remote add origin "$origin" git push -q origin main dev feat/live ) cd "$work" MAIN=$(cache_key main); DEV=$(cache_key dev); LIVE=$(cache_key feat/live) DEAD=$(cache_key feat/dead); OWN=$(cache_key feat/own) root="$scratch/cache" mk() { mkdir -p "$root/$1"; head -c 4096 /dev/zero > "$root/$1/blob"; touch -d "$2" "$root/$1/.cache-last-used"; } reset_cache() { rm -rf "$root"; mkdir -p "$root" mk "target-$MAIN" '2020-01-01' mk "snapshot-$MAIN" '2020-01-01' mk "target-$DEV" '2020-01-01' mk "snapshot-$DEV" '2020-01-01' mk "target-$LIVE" '2020-01-02' # older than the dead one, deliberately mk "target-$DEAD" '2030-01-01' # newest on disk, but its branch is gone mk "snapshot-$DEAD" '2030-01-01' mk "target-$OWN" '2025-01-01' } # run_prune [settle-seconds] # # The settle window is only set when a scenario asks for it, so every other # scenario — scenario 14 above all — runs against the script's own default # rather than against a value this file chose. run_prune() { local free="${1:-}" if [ -n "${2:-}" ]; then export EVICTION_ASIDE_SETTLE_SECONDS="$2"; else unset EVICTION_ASIDE_SETTLE_SECONDS; fi CACHE_DF_OVERRIDE="$free" GITHUB_STEP_SUMMARY="$scratch/summary" \ bash "$prune" "$root" "$root/target-$OWN" "dev main" 10 > "$scratch/log" 2>&1 \ || { cat "$scratch/log"; fail "prune-cache.sh exited non-zero"; } } echo echo "=== 1/2: dead pruned unconditionally; older-but-live survives ===" reset_cache run_prune "1000000 900000" # 90% free: no pressure at all assert_gone "$root/target-$DEAD" "dead branch's target dir pruned with no disk pressure" assert_gone "$root/snapshot-$DEAD" "dead branch's snapshot pruned too" assert_kept "$root/target-$LIVE" "live branch survives despite an older marker than the dead one" assert_log "no matching branch on origin" "eviction reason reported" echo echo "=== 3: protected refs never evicted under forced pressure ===" reset_cache run_prune "1000000 1000" # 0.1% free assert_kept "$root/target-$DEV" "dev's target dir survives disk pressure" assert_kept "$root/snapshot-$DEV" "dev's snapshot survives disk pressure" assert_kept "$root/target-$MAIN" "main's target dir survives disk pressure" assert_kept "$root/snapshot-$MAIN" "main's snapshot survives disk pressure" echo echo "=== 9: own cache never evicted by a sibling pass ===" assert_kept "$root/target-$OWN" "this run's own cache survives" echo echo "=== 7: target dirs evicted before snapshots ===" reset_cache # Only the live branch is evictable; give it both a target dir and a snapshot # with identical markers so ordering, not age, decides. mk "snapshot-$LIVE" '2020-01-02' # The df override is a fixed reading, so the pressure loop drains everything # evictable — which is what makes the ORDER the observable property here, not # what survives. Assert the eviction order directly from the log. run_prune "1000000 1000" order=$(grep -o "evicted \(target\|snapshot\)-$LIVE" "$scratch/log" | sed "s/evicted //") [ "$(printf '%s\n' "$order" | head -1)" = "target-$LIVE" ] \ || fail "expected target-$LIVE to be evicted before snapshot-$LIVE, got: $order" ok "target dirs are evicted before snapshots" echo echo "=== 4: a fresh lock protects a dead, old, under-pressure cache ===" reset_cache date +%s > "$root/target-$DEAD/.ci-lock-ci-1" run_prune "1000000 1000" assert_kept "$root/target-$DEAD" "locked cache survives both passes" assert_log "held open by" "lock reported in the log" # With the locked one the only dead cache left, the pass has declined every # dead cache it found — at which point "no dead-branch caches found" is a # false summary of the decline logged two lines above it. rm -rf "$root/snapshot-$DEAD" run_prune "1000000 900000" assert_kept "$root/target-$DEAD" "still not evicted when it is the only dead cache" assert_log "none pruned this pass" "a pass that declined every dead cache reports that" if grep -q "no dead-branch caches found" "$scratch/log"; then fail "the closing summary contradicts the decline logged above it" fi ok "the summary does not claim it found nothing" echo echo "=== 5: a stale lock is not honoured forever ===" reset_cache echo 0 > "$root/target-$DEAD/.ci-lock-ci-1" touch -d '2000-01-01' "$root/target-$DEAD/.ci-lock-ci-1" run_prune "1000000 900000" assert_gone "$root/target-$DEAD" "cache with an abandoned lock is evicted" assert_log "treating as abandoned" "abandoned lock reported in the log" echo echo "=== 6: liveness unavailable fails safe, pressure fallback still works ===" reset_cache ( cd "$work" && git remote set-url origin "$scratch/nonexistent.git" ) run_prune "1000000 900000" # no pressure assert_kept "$root/target-$DEAD" "dead cache NOT pruned when liveness is unavailable" assert_log "treating as UNAVAILABLE" "unavailability reported plainly, not folded into 'no branches'" run_prune "1000000 1000" # now with pressure if [ -e "$root/target-$DEAD" ] && [ -e "$root/target-$LIVE" ]; then fail "pressure fallback did nothing when liveness was unavailable" fi ok "pressure fallback still evicts when liveness is unavailable" (cd "$work" && git remote set-url origin "$origin") echo echo "=== 8: self-clear reports to the job summary ===" reset_cache rm -rf "$root/target-$DEAD" "$root/snapshot-$DEAD" "$root/target-$LIVE" : > "$scratch/summary" run_prune "1000000 1000" # nothing evictable left but the run's own cache assert_log "clearing own" "self-clear reported in the log" grep -q 'self-clear' "$scratch/summary" || fail "self-clear missing from the job summary" ok "self-clear reported to the job summary, not only the log" [ -d "$root/target-$OWN" ] || fail "self-clear left the own directory missing" [ -z "$(ls -A "$root/target-$OWN")" ] || fail "self-clear did not actually empty the directory" ok "own cache wiped and recreated empty" echo echo "=== 10: scoped to the cache root ===" reset_cache decoy="$scratch/other-project"; mkdir -p "$decoy/target-$DEAD"; touch "$decoy/target-$DEAD/blob" run_prune "1000000 1000" assert_kept "$decoy/target-$DEAD" "a cache outside the cache root is never touched" echo echo "=== 11: a live reader marker protects a cache, like a lock file does ===" reset_cache date +%s > "$root/.reading-target-$DEAD-job1" run_prune "1000000 1000" assert_kept "$root/target-$DEAD" "a cache being hardlink-cloned right now survives both passes" assert_log "currently cloning it" "the reader is named in the log, not silently honoured" rm -f "$root/.reading-target-$DEAD-job1" echo echo "=== 12: a reader marker published INSIDE the check-to-unlink window ===" reset_cache # A consumer publishes its marker whenever it starts a clone, which can be at # any instant — including after the pass has checked for markers and before it # unlinks. That window is real time, not a theoretical interleaving: `du -sk` # on a multi-GB cache runs for seconds, and the pass runs one on every # candidate. It is reproduced deterministically here by making that very `du` # publish the marker, which places it strictly after the check and strictly # before the unlink — exactly where a check-then-delete eviction cannot see # it. The candidate must still be standing afterwards, with its contents # intact and nothing left renamed aside. mkdir -p "$scratch/bin" real_du=$(command -v du) cat > "$scratch/bin/du" < "$root/.reading-target-$DEAD-racer" ;; esac done exec "$real_du" "\$@" EOF chmod +x "$scratch/bin/du" ( PATH="$scratch/bin:$outer_path"; run_prune "1000000 900000" ) [ -e "$root/.reading-target-$DEAD-racer" ] || fail "the racing marker was never published — scenario 12 proves nothing" assert_kept "$root/target-$DEAD" "a cache claimed inside the eviction window is not unlinked" assert_kept "$root/target-$DEAD/blob" "the reprieved cache still has its contents" assert_log "restored, not evicted" "the reprieve is reported, not silent" [ -z "$(ls -d "$root"/.evicting-* 2>/dev/null)" ] || fail "an aside directory was left behind after the reprieve" ok "nothing left renamed aside once the eviction is declined" rm -f "$root/.reading-target-$DEAD-racer" echo echo "=== 13: a deferred eviction is reclaimed, but not under a live reader ===" reset_cache aside="$root/.evicting-target-$DEAD-9999" mkdir -p "$aside"; head -c 4096 /dev/zero > "$aside/blob" date +%s > "$root/.reading-target-$DEAD-job1" # The settle window (scenario 14) gates this sweep first and would decide both # runs on its own. A directory's ctime is what that window reads and cannot be # backdated the way `touch -d` backdates an mtime, so the window is moved out # of the way rather than the directory aged into it. run_prune "1000000 900000" 0 assert_kept "$aside" "a deferred eviction is not reclaimed while a job is still reading it" assert_log "deferring its reclamation again" "the continued deferral is reported" rm -f "$root/.reading-target-$DEAD-job1" run_prune "1000000 900000" 0 assert_gone "$aside" "a deferred eviction is reclaimed once its reader is gone" assert_log "reclaiming deferred eviction" "the reclamation is reported" echo echo "=== 14: an aside another pass may still be evicting is left alone ===" reset_cache aside="$root/.evicting-target-$DEAD-9999" # Built the way the pass builds one — an old cache directory renamed a moment # ago — because the two ages that describe it disagree, and which of them the # window reads is the whole mechanism. `mkdir`-ing the aside directly would # give it a fresh mtime as well as a fresh ctime, and a fixture whose two # clocks agree cannot tell %Z from %Y: the settle window would read the wrong # one, never fire for any real aside, and this scenario would not notice. victim="$root/target-$DEAD-victim" mkdir -p "$victim"; head -c 4096 /dev/zero > "$victim/blob" touch -d '2020-01-01' "$victim" # an old cache, which is every cache mv -T "$victim" "$aside" # set aside a moment ago # Deliberately no reader marker: the reader gate would pass this straight # through, which is the whole point. An aside with no readers is exactly what # a pass that has just renamed one aside and not yet decided about it looks # like, and `rm -rf` traverses fd-relative — so reclaiming it out from under # that pass lets it republish a half-emptied tree under a live cache name. run_prune "1000000 900000" assert_kept "$aside" "an aside younger than the settle window is not reclaimed" assert_kept "$aside/blob" "and is left intact, not part-way emptied" assert_log "may still be evicting it" "the deferral gives its actual reason" echo echo "=== 15: a merged-but-undeleted branch is dead too ===" # This forge keeps a PR's branch after the merge, so `ls-remote` reports it # forever. Built the way that happens: a branch merged into dev with a merge # commit, still pushed, beside one branched at the same point and NOT merged. git="git -c user.email=t@t -c user.name=t -c commit.gpgsign=false" ( cd "$work" git checkout -q dev git checkout -q -b feat/merged $git commit -q --allow-empty -m merged git checkout -q dev $git merge -q --no-ff feat/merged -m "merge feat/merged" git checkout -q -b feat/unmerged $git commit -q --allow-empty -m unmerged git checkout -q dev git push -q origin dev feat/merged feat/unmerged ) MERGED=$(cache_key feat/merged); UNMERGED=$(cache_key feat/unmerged) reset_cache mk "target-$MERGED" '2030-01-01' mk "snapshot-$MERGED" '2030-01-01' mk "target-$UNMERGED" '2020-01-01' # older, deliberately: merge state decides, not age run_prune "1000000 900000" # 90% free: no pressure at all assert_log "merged-branch detection anchored on" "the pass says what it anchored ancestry on" assert_gone "$root/target-$MERGED" "a merged branch's cache is pruned though its branch is still on origin" assert_gone "$root/snapshot-$MERGED" "and so is its snapshot" assert_kept "$root/target-$UNMERGED" "an unmerged branch's cache survives, though it is the older of the two" assert_log "merged into dev" "the eviction names the branch it was merged into" echo echo "=== 16: 'cannot tell' is not death, per branch and per checkout ===" # A branch whose tip this checkout has never seen. Pushed from a second clone, # so `ls-remote` reports a SHA that `$work` holds no object for — which is # what "cannot determine" actually looks like, rather than a stubbed failure. other="$scratch/other"; git clone -q "$origin" "$other" ( cd "$other" git checkout -q -b feat/elsewhere origin/dev git -c user.email=t@t -c user.name=t -c commit.gpgsign=false commit -q --allow-empty -m elsewhere git push -q origin feat/elsewhere ) ELSEWHERE=$(cache_key feat/elsewhere) reset_cache mk "target-$ELSEWHERE" '2030-01-01' mk "target-$MERGED" '2030-01-01' run_prune "1000000 900000" assert_kept "$root/target-$ELSEWHERE" "a branch whose tip is not in this checkout is kept, not classified dead" assert_log "cannot tell merged from live" "and the undecidable branch is named, not silently skipped" assert_gone "$root/target-$MERGED" "while a branch it CAN decide is still pruned in the same pass" # A shallow checkout, where a missing object is the ordinary case rather than # a signal — so the whole merged half is withheld. The deleted-branch half is # unaffected, which is what keeps this a narrowing rather than an outage. shallow="$scratch/shallow"; git clone -q --depth 1 -b dev "file://$origin" "$shallow" [ "$(git -C "$shallow" rev-parse --is-shallow-repository)" = true ] \ || fail "the fixture clone is not shallow — scenario 16's second half proves nothing" reset_cache mk "target-$MERGED" '2030-01-01' ( cd "$shallow" CACHE_DF_OVERRIDE="1000000 900000" GITHUB_STEP_SUMMARY="$scratch/summary" \ bash "$prune" "$root" "$root/target-$OWN" "dev main" 10 ) > "$scratch/log" 2>&1 || { cat "$scratch/log"; fail "prune-cache.sh exited non-zero in a shallow checkout"; } assert_log "checkout is shallow" "a shallow checkout withholds the merged signal and says why" assert_kept "$root/target-$MERGED" "and keeps a merged branch's cache rather than guessing" assert_gone "$root/target-$DEAD" "while the deleted-branch signal still fires" echo echo "=== 17: the free-space requirement is measured off the clone's source ===" # A `df` that answers from the cache root's actual size, because the property # under test is that the pass STOPS once the requirement is met — which a # fixed CACHE_DF_OVERRIDE cannot express, since evicting never changes it. mkdir -p "$scratch/bin17" real_du=$(command -v du) build_seed_fixture() { rm -rf "$root"; mkdir -p "$root" # The source the seed is about to clone. 8 MB of dep-info, which # unshare_mutable_paths has to real-copy, beside 16 MB of .rlib that it # leaves hardlinked — so a requirement derived from the SIZE of the source # would be three times the one derived from its mutable set. mkdir -p "$root/snapshot-$DEV/debug/.fingerprint/unit" "$root/snapshot-$DEV/debug/deps" head -c $((8 * 1024 * 1024)) /dev/zero > "$root/snapshot-$DEV/debug/.fingerprint/unit/dep-lib" head -c $((16 * 1024 * 1024)) /dev/zero > "$root/snapshot-$DEV/debug/deps/libx.rlib" touch -d '2020-01-01' "$root/snapshot-$DEV/.cache-last-used" # Three live, unmerged branches' caches of 4 MB each, oldest first. local i=0 for b in a b c; do i=$((i + 1)) mkdir -p "$root/target-$(cache_key "feat/$b")" head -c $((4 * 1024 * 1024)) /dev/zero > "$root/target-$(cache_key "feat/$b")/blob" touch -d "202${i}-01-01" "$root/target-$(cache_key "feat/$b")/.cache-last-used" done # A volume with 2 MB to spare: under the requirement, over nothing else. cap=$(( $($real_du -sk "$root" | awk '{print $1}') + 2048 )) cat > "$scratch/bin17/df" < — the form cargo-cache/action.yml uses: # the same pass, told what the seed step it now runs ahead of will clone. The # headroom knobs are pinned so the arithmetic is the fixture's, not the # defaults' (whose 2 GiB floor would dwarf any fixture on a test host). seeded_rc=0 run_seeded_prune() { seeded_rc=0 PATH="$scratch/bin17:$outer_path" \ CACHE_CLONE_HEADROOM_PERCENT=100 CACHE_CLONE_HEADROOM_FLOOR_KB=1024 \ GITHUB_STEP_SUMMARY="$scratch/summary" \ bash "$prune" "$root" "$root/target-$(cache_key "$1")" "dev main" 0 \ "$(cache_key "$1")" "$(cache_key "$2")" "" \ > "$scratch/log" 2>&1 || seeded_rc=$? } build_seed_fixture run_seeded_prune feat/own dev [ "$seeded_rc" = 0 ] || { cat "$scratch/log"; fail "prune-cache.sh exited ${seeded_rc} with the requirement satisfiable"; } assert_log "measured from its mutable set" "the requirement says where it came from" assert_gone "$root/target-$(cache_key feat/a)" "the oldest cache is evicted to make room for the clone" assert_gone "$root/target-$(cache_key feat/b)" "and the next oldest, because one was not enough" assert_kept "$root/target-$(cache_key feat/c)" "and the pass STOPS there rather than draining the volume" assert_kept "$root/snapshot-$DEV" "the source the seed is about to clone is never a candidate" # Nothing eligible: every sibling is held open by a running job. The pass # cannot reach the requirement, and the seed that follows would fail against a # staging path naming none of this. build_seed_fixture for b in a b c; do date +%s > "$root/target-$(cache_key "feat/$b")/.ci-lock-ci-1"; done run_seeded_prune feat/own dev [ "$seeded_rc" = 1 ] || { cat "$scratch/log"; fail "expected exit 1 when the clone cannot fit, got ${seeded_rc}" ; } ok "a clone that cannot be made to fit fails the pass rather than the seed" assert_log "short by" "the failure names the shortfall" assert_log "held open by a running job" "and what was kept instead of it, with the reason" assert_kept "$root/target-$(cache_key feat/a)" "a locked cache is still not evicted, however tight the disk" echo echo "=== 18: the layout that produced the bug ===" # zemyna's volume on 2026-09-07: the base branch's snapshot and target dir, # plus one target dir for a branch merged the day before and never deleted. # Equal sizes and 90% free, so neither age nor pressure nor size can be what # decides — only the merge state. rm -rf "$root"; mkdir -p "$root" mk "snapshot-$DEV" '2026-09-01' mk "target-$DEV" '2026-09-01' mk "target-$MERGED" '2026-09-06' run_prune "1000000 900000" assert_kept "$root/snapshot-$DEV" "the base snapshot stays" assert_kept "$root/target-$DEV" "and the base target dir stays" assert_gone "$root/target-$MERGED" "and the merged-but-undeleted branch's cache is the one reclaimed" [ "$(grep -c 'pruned dead-branch cache' "$scratch/log")" = 1 ] \ || { cat "$scratch/log"; fail "expected exactly one eviction on the zemyna layout"; } ok "exactly one directory is evicted, and it is that one" echo echo "prune-cache-selftest: ${pass_count} assertions passed"