diff --git a/cargo-cache/action.yml b/cargo-cache/action.yml index 84d3777..4006673 100644 --- a/cargo-cache/action.yml +++ b/cargo-cache/action.yml @@ -29,9 +29,18 @@ inputs: required: false default: 'dev main' min-free-percent: - description: 'Prune when free space on the cache volume drops below this percentage.' + description: >- + An ADDITIONAL free-space floor, as a percentage of the cache volume. + The prune's own requirement is derived per run from what the seed is + about to clone — the mutable set it has to real-copy out of the source + snapshot, which is a property of that snapshot and not of the volume — + and this floor only ever raises it. 0, the default, leaves the derived + requirement as the only gate. Set it to keep headroom for something + other than the clone (the build's own output, another job on the same + volume); it will not make the clone fit, because it does not know how + big the clone is. required: false - default: '10' + default: '0' restore-mtimes: description: >- Restore every tracked file's mtime from git history. Requires a @@ -40,13 +49,20 @@ inputs: required: false default: 'true' prune: - description: 'Run the eviction pass (dead-branch liveness + disk pressure).' + description: >- + Run the eviction pass (dead-branch liveness + disk pressure). It runs + BEFORE the seed step, so what it frees is available to the clone that + step makes. required: false default: 'true' liveness-prune: description: >- - Within the prune pass, remove caches for branches that no longer exist - on origin. Set to false on a runner that cannot reach origin. + Within the prune pass, remove caches for branches that are dead: gone + from origin, or still on origin with a tip already merged into a + protected branch. The second signal is what reclaims anything at all on + a forge that keeps branches after merge, and it needs the protected + branches' commits in the checkout — a shallow one withholds it and says + so. Set to false on a runner that cannot reach origin. required: false default: 'true' own-ref: @@ -169,6 +185,39 @@ runs: echo "CI_WATERMARK_FILE=${WATERMARK}" } >> "$GITHUB_ENV" + # Runs BEFORE the seed, which is the only order in which its work can + # help: the eviction it performs is what makes room for the clone the seed + # step is about to make, and the requirement it evicts against is measured + # off the snapshot that clone will read. Running afterwards — where this + # step used to be — meant every run freed space for the NEXT one and the + # seed met whatever the last run happened to leave. + # + # Two things this ordering has to be safe against, and is: + # + # The source it is about to read is excluded from every pass by name + # (see protected_reason in prune-cache.sh), so pass 1 cannot take the + # snapshot out from under the seed that follows it. + # A concurrent job's target dir carries its lock from the instant it + # appears under its final name — the seed writes it into the staging + # tree before the rename — so there is no window in which running this + # earlier sees an unlocked directory somebody is using. + - if: ${{ inputs.prune == 'true' }} + shell: bash + env: + STALE_LOCK_SECONDS: ${{ inputs.stale-lock-seconds }} + CACHE_LIVENESS: ${{ inputs.liveness-prune }} + run: | + set -euo pipefail + SCRIPTS=$(cd "${{ github.action_path }}/.." && pwd)/scripts + bash "${SCRIPTS}/prune-cache.sh" \ + "${{ steps.resolve.outputs.cache-root }}" \ + "${{ steps.resolve.outputs.target-dir }}" \ + "${{ inputs.protected-branches }}" \ + "${{ inputs.min-free-percent }}" \ + "${{ steps.resolve.outputs.cache-key }}" \ + "${{ steps.resolve.outputs.base-key }}" \ + "${{ inputs.seed-fallback-dir }}" + # Seeds this ref's target dir from the base's published snapshot. See # scripts/seed-target-dir.sh — the staging-then-atomic-rename is what # makes concurrent jobs sharing one cache key safe by construction rather @@ -228,17 +277,3 @@ runs: CARGO_TARGET_DIR: ${{ steps.resolve.outputs.target-dir }} CI_WATERMARK_FILE: ${{ steps.resolve.outputs.watermark-file }} run: bash "$(cd "${{ github.action_path }}/.." && pwd)/scripts/restore-mtimes.sh" - - - if: ${{ inputs.prune == 'true' }} - shell: bash - env: - STALE_LOCK_SECONDS: ${{ inputs.stale-lock-seconds }} - CACHE_LIVENESS: ${{ inputs.liveness-prune }} - run: | - set -euo pipefail - SCRIPTS=$(cd "${{ github.action_path }}/.." && pwd)/scripts - bash "${SCRIPTS}/prune-cache.sh" \ - "${{ steps.resolve.outputs.cache-root }}" \ - "${{ steps.resolve.outputs.target-dir }}" \ - "${{ inputs.protected-branches }}" \ - "${{ inputs.min-free-percent }}" diff --git a/scripts/prune-cache-selftest.sh b/scripts/prune-cache-selftest.sh index ab7c8cf..ff5cca8 100755 --- a/scripts/prune-cache-selftest.sh +++ b/scripts/prune-cache-selftest.sh @@ -46,10 +46,36 @@ # empty a tree its owner may still restore under a live cache name. Its # fixture is an OLD directory renamed a moment ago — production's shape, # and what lets it tell the two timestamps apart. +# 15. A MERGED-BUT-UNDELETED BRANCH IS DEAD TOO. This forge keeps branches +# after merge, so `ls-remote` reports them forever and scenario 1's +# signal never fires for them — which is how three 40 GB caches sat on a +# full volume until somebody removed them by hand (gitdan-actions#20). A +# branch whose tip is an ancestor of a protected branch's tip is pruned +# like a deleted one; an unmerged branch beside it is not. +# 16. AND "CANNOT TELL" IS STILL NOT DEATH, at both granularities: a branch +# whose tip is not in this checkout is kept with a warning naming it, +# and a shallow checkout — where a missing object is the normal case — +# withholds the whole signal rather than reading it as "nothing merged". +# The deleted-branch signal keeps working in both. +# 17. THE FREE-SPACE REQUIREMENT IS MEASURED OFF THE SOURCE, not taken as a +# percentage of the volume: the pass evicts until the clone the seed is +# about to make fits, and stops there rather than draining the volume. +# When it cannot get there it FAILS, naming the shortfall and every +# directory it kept instead — because the seed would otherwise fail +# seconds later against a staging path that names nothing. +# 18. AND ON THE LAYOUT THAT PRODUCED THE BUG: three equal-sized caches, one +# of them a merged-but-undeleted branch's, with disk to spare. Exactly +# that one goes. Equal sizes and no pressure are the point — nothing but +# the merge state can be what decides. set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$script_dir/cache-lib.sh" prune="$script_dir/prune-cache.sh" +# Two scenarios below put a stub of a real tool on PATH for one command. +# Captured once, here, rather than read back at each of those sites: a `$PATH` +# read after the first of them is indistinguishable, to a static check, from +# reading the modification the subshell lost. +outer_path="$PATH" scratch=$(mktemp -d) trap 'rm -rf "$scratch"' EXIT @@ -235,7 +261,7 @@ done exec "$real_du" "\$@" EOF chmod +x "$scratch/bin/du" -( PATH="$scratch/bin:$PATH"; run_prune "1000000 900000" ) +( PATH="$scratch/bin:$outer_path"; run_prune "1000000 900000" ) [ -e "$root/.reading-target-$DEAD-racer" ] || fail "the racing marker was never published — scenario 12 proves nothing" assert_kept "$root/target-$DEAD" "a cache claimed inside the eviction window is not unlinked" assert_kept "$root/target-$DEAD/blob" "the reprieved cache still has its contents" @@ -286,5 +312,172 @@ assert_kept "$aside" "an aside younger than the settle window is not reclaimed" assert_kept "$aside/blob" "and is left intact, not part-way emptied" assert_log "may still be evicting it" "the deferral gives its actual reason" +echo +echo "=== 15: a merged-but-undeleted branch is dead too ===" +# This forge keeps a PR's branch after the merge, so `ls-remote` reports it +# forever. Built the way that happens: a branch merged into dev with a merge +# commit, still pushed, beside one branched at the same point and NOT merged. +git="git -c user.email=t@t -c user.name=t -c commit.gpgsign=false" +( + cd "$work" + git checkout -q dev + git checkout -q -b feat/merged + $git commit -q --allow-empty -m merged + git checkout -q dev + $git merge -q --no-ff feat/merged -m "merge feat/merged" + git checkout -q -b feat/unmerged + $git commit -q --allow-empty -m unmerged + git checkout -q dev + git push -q origin dev feat/merged feat/unmerged +) +MERGED=$(cache_key feat/merged); UNMERGED=$(cache_key feat/unmerged) +reset_cache +mk "target-$MERGED" '2030-01-01' +mk "snapshot-$MERGED" '2030-01-01' +mk "target-$UNMERGED" '2020-01-01' # older, deliberately: merge state decides, not age +run_prune "1000000 900000" # 90% free: no pressure at all +assert_log "merged-branch detection anchored on" "the pass says what it anchored ancestry on" +assert_gone "$root/target-$MERGED" "a merged branch's cache is pruned though its branch is still on origin" +assert_gone "$root/snapshot-$MERGED" "and so is its snapshot" +assert_kept "$root/target-$UNMERGED" "an unmerged branch's cache survives, though it is the older of the two" +assert_log "merged into dev" "the eviction names the branch it was merged into" + +echo +echo "=== 16: 'cannot tell' is not death, per branch and per checkout ===" +# A branch whose tip this checkout has never seen. Pushed from a second clone, +# so `ls-remote` reports a SHA that `$work` holds no object for — which is +# what "cannot determine" actually looks like, rather than a stubbed failure. +other="$scratch/other"; git clone -q "$origin" "$other" +( + cd "$other" + git checkout -q -b feat/elsewhere origin/dev + git -c user.email=t@t -c user.name=t -c commit.gpgsign=false commit -q --allow-empty -m elsewhere + git push -q origin feat/elsewhere +) +ELSEWHERE=$(cache_key feat/elsewhere) +reset_cache +mk "target-$ELSEWHERE" '2030-01-01' +mk "target-$MERGED" '2030-01-01' +run_prune "1000000 900000" +assert_kept "$root/target-$ELSEWHERE" "a branch whose tip is not in this checkout is kept, not classified dead" +assert_log "cannot tell merged from live" "and the undecidable branch is named, not silently skipped" +assert_gone "$root/target-$MERGED" "while a branch it CAN decide is still pruned in the same pass" + +# A shallow checkout, where a missing object is the ordinary case rather than +# a signal — so the whole merged half is withheld. The deleted-branch half is +# unaffected, which is what keeps this a narrowing rather than an outage. +shallow="$scratch/shallow"; git clone -q --depth 1 -b dev "file://$origin" "$shallow" +[ "$(git -C "$shallow" rev-parse --is-shallow-repository)" = true ] \ + || fail "the fixture clone is not shallow — scenario 16's second half proves nothing" +reset_cache +mk "target-$MERGED" '2030-01-01' +( + cd "$shallow" + CACHE_DF_OVERRIDE="1000000 900000" GITHUB_STEP_SUMMARY="$scratch/summary" \ + bash "$prune" "$root" "$root/target-$OWN" "dev main" 10 +) > "$scratch/log" 2>&1 || { cat "$scratch/log"; fail "prune-cache.sh exited non-zero in a shallow checkout"; } +assert_log "checkout is shallow" "a shallow checkout withholds the merged signal and says why" +assert_kept "$root/target-$MERGED" "and keeps a merged branch's cache rather than guessing" +assert_gone "$root/target-$DEAD" "while the deleted-branch signal still fires" + +echo +echo "=== 17: the free-space requirement is measured off the clone's source ===" +# A `df` that answers from the cache root's actual size, because the property +# under test is that the pass STOPS once the requirement is met — which a +# fixed CACHE_DF_OVERRIDE cannot express, since evicting never changes it. +mkdir -p "$scratch/bin17" +real_du=$(command -v du) +build_seed_fixture() { + rm -rf "$root"; mkdir -p "$root" + # The source the seed is about to clone. 8 MB of dep-info, which + # unshare_mutable_paths has to real-copy, beside 16 MB of .rlib that it + # leaves hardlinked — so a requirement derived from the SIZE of the source + # would be three times the one derived from its mutable set. + mkdir -p "$root/snapshot-$DEV/debug/.fingerprint/unit" "$root/snapshot-$DEV/debug/deps" + head -c $((8 * 1024 * 1024)) /dev/zero > "$root/snapshot-$DEV/debug/.fingerprint/unit/dep-lib" + head -c $((16 * 1024 * 1024)) /dev/zero > "$root/snapshot-$DEV/debug/deps/libx.rlib" + touch -d '2020-01-01' "$root/snapshot-$DEV/.cache-last-used" + # Three live, unmerged branches' caches of 4 MB each, oldest first. + local i=0 + for b in a b c; do + i=$((i + 1)) + mkdir -p "$root/target-$(cache_key "feat/$b")" + head -c $((4 * 1024 * 1024)) /dev/zero > "$root/target-$(cache_key "feat/$b")/blob" + touch -d "202${i}-01-01" "$root/target-$(cache_key "feat/$b")/.cache-last-used" + done + # A volume with 2 MB to spare: under the requirement, over nothing else. + cap=$(( $($real_du -sk "$root" | awk '{print $1}') + 2048 )) + cat > "$scratch/bin17/df" < — the form cargo-cache/action.yml uses: +# the same pass, told what the seed step it now runs ahead of will clone. The +# headroom knobs are pinned so the arithmetic is the fixture's, not the +# defaults' (whose 2 GiB floor would dwarf any fixture on a test host). +seeded_rc=0 +run_seeded_prune() { + seeded_rc=0 + PATH="$scratch/bin17:$outer_path" \ + CACHE_CLONE_HEADROOM_PERCENT=100 CACHE_CLONE_HEADROOM_FLOOR_KB=1024 \ + GITHUB_STEP_SUMMARY="$scratch/summary" \ + bash "$prune" "$root" "$root/target-$(cache_key "$1")" "dev main" 0 \ + "$(cache_key "$1")" "$(cache_key "$2")" "" \ + > "$scratch/log" 2>&1 || seeded_rc=$? +} + +build_seed_fixture +run_seeded_prune feat/own dev +[ "$seeded_rc" = 0 ] || { cat "$scratch/log"; fail "prune-cache.sh exited ${seeded_rc} with the requirement satisfiable"; } +assert_log "measured from its mutable set" "the requirement says where it came from" +assert_gone "$root/target-$(cache_key feat/a)" "the oldest cache is evicted to make room for the clone" +assert_gone "$root/target-$(cache_key feat/b)" "and the next oldest, because one was not enough" +assert_kept "$root/target-$(cache_key feat/c)" "and the pass STOPS there rather than draining the volume" +assert_kept "$root/snapshot-$DEV" "the source the seed is about to clone is never a candidate" + +# Nothing eligible: every sibling is held open by a running job. The pass +# cannot reach the requirement, and the seed that follows would fail against a +# staging path naming none of this. +build_seed_fixture +for b in a b c; do date +%s > "$root/target-$(cache_key "feat/$b")/.ci-lock-ci-1"; done +run_seeded_prune feat/own dev +[ "$seeded_rc" = 1 ] || { cat "$scratch/log"; fail "expected exit 1 when the clone cannot fit, got ${seeded_rc}" ; } +ok "a clone that cannot be made to fit fails the pass rather than the seed" +assert_log "short by" "the failure names the shortfall" +assert_log "held open by a running job" "and what was kept instead of it, with the reason" +assert_kept "$root/target-$(cache_key feat/a)" "a locked cache is still not evicted, however tight the disk" + +echo +echo "=== 18: the layout that produced the bug ===" +# zemyna's volume on 2026-09-07: the base branch's snapshot and target dir, +# plus one target dir for a branch merged the day before and never deleted. +# Equal sizes and 90% free, so neither age nor pressure nor size can be what +# decides — only the merge state. +rm -rf "$root"; mkdir -p "$root" +mk "snapshot-$DEV" '2026-09-01' +mk "target-$DEV" '2026-09-01' +mk "target-$MERGED" '2026-09-06' +run_prune "1000000 900000" +assert_kept "$root/snapshot-$DEV" "the base snapshot stays" +assert_kept "$root/target-$DEV" "and the base target dir stays" +assert_gone "$root/target-$MERGED" "and the merged-but-undeleted branch's cache is the one reclaimed" +[ "$(grep -c 'pruned dead-branch cache' "$scratch/log")" = 1 ] \ + || { cat "$scratch/log"; fail "expected exactly one eviction on the zemyna layout"; } +ok "exactly one directory is evicted, and it is that one" + echo echo "prune-cache-selftest: ${pass_count} assertions passed" diff --git a/scripts/prune-cache.sh b/scripts/prune-cache.sh index 6efa4fc..7e1827c 100755 --- a/scripts/prune-cache.sh +++ b/scripts/prune-cache.sh @@ -1,8 +1,16 @@ #!/usr/bin/env bash # Eviction for the per-ref cache directories on the persistent volume. # -# Usage: prune-cache.sh +# Usage: prune-cache.sh \ +# [own-key] [base-key] [fallback-dir] # protected-branches space-separated raw refs (e.g. "dev main") +# min-free-percent a FLOOR, not the gate — 0 to rely on the derived +# requirement alone (the default) +# own-key, base-key, fallback-dir +# the same three the seed step resolves its source from. +# Given them, this pass sizes the volume for the clone +# that step is about to make; without them it has only +# the percentage floor, and says so. # # Optional environment: # STALE_LOCK_SECONDS age past which a .ci-lock-* marker is treated as @@ -16,19 +24,41 @@ # # Three passes, in order: # -# 1. LIVENESS — every target-*/snapshot-* directory whose branch no longer -# exists on origin is removed UNCONDITIONALLY, not gated on free space. -# A directory for a branch deleted days ago is pure loss: nothing will -# ever read it again, since a merged PR's branch cannot be reopened. -# Waiting for disk pressure to notice means paying for it until then. -# Skipped entirely, loudly, if the liveness signal itself is -# unavailable — "couldn't determine" is never folded into "dead". -# 2. PRESSURE — if free space is still under the threshold, evict remaining -# (now necessarily live) directories oldest-first until it clears. -# 3. SELF-CLEAR — if pass 2 still isn't enough, wipe this run's own target -# dir and pay a cold rebuild, reported to the job summary as well as the -# log, because a warning on a green run is what lets a silently 4x-slower -# job go unnoticed. +# 1. LIVENESS — every target-*/snapshot-* directory whose branch is DEAD is +# removed UNCONDITIONALLY, not gated on free space. A directory for a +# branch nothing will build again is pure loss; waiting for disk pressure +# to notice means paying for it until then. Two signals make a branch +# dead, and the second exists because the first alone is inert on a forge +# that keeps branches after merge (gitdan-actions#20): +# +# DELETED — the branch is no longer on origin at all. +# MERGED — the branch is still on origin, but its tip is an ancestor +# of a protected branch's tip, so every commit it holds is +# already on the branch its cache would be re-cloned from. +# +# Skipped entirely, loudly, if the signal itself is unavailable — +# "couldn't determine" is never folded into "dead", for either signal +# and at either granularity: a checkout that cannot answer ancestry at +# all skips the merged half, and a single branch whose tip is not in the +# checkout is kept with a warning naming it. +# 2. PRESSURE — if free space is under the requirement, evict remaining +# (now necessarily live) directories oldest-first until it clears. The +# requirement is what the seed step is about to need to clone its source, +# MEASURED off that source (see clone_headroom_kb in cache-lib.sh), and a +# percentage floor only if one is configured. A percentage cannot express +# this: the failure it has to prevent is a clone running out of disk +# part-way through unsharing its mutable paths, and how much that needs +# is a property of the snapshot, not of the volume. +# 3. SELF-CLEAR — if pass 2 still isn't enough for the percentage floor, +# wipe this run's own target dir and pay a cold rebuild, reported to the +# job summary as well as the log, because a warning on a green run is +# what lets a silently 4x-slower job go unnoticed. It is not a way out of +# the derived requirement: a run that has an own target dir to wipe is a +# run whose seed reuses it and clones nothing, so that requirement is +# zero. Falling short of a NON-ZERO derived requirement fails the job +# here, naming the shortfall and what was kept instead of it — the seed +# would otherwise fail seconds later against a half-unshared staging +# tree, which is the failure this pass exists to pre-empt. # # Reactive-only, with no hard cap on cache size: a workspace's natural working # set is what it is, and bounding the footprint preemptively means wiping @@ -75,16 +105,34 @@ # than reimplementing a lookalike is what makes the classification sound; any # drift between two spellings would silently misclassify every directory. # +# The merged half reads the TIP SHA out of that same `ls-remote` output and +# asks `git merge-base --is-ancestor` against each protected branch's tip, +# using the objects in this job's own checkout. Ancestry is only decidable +# where the objects are there to decide it, so the answer "I cannot tell" +# exists and is distinct from "not merged" everywhere it can arise: +# +# - a shallow checkout makes a MISSING object prove nothing, so the whole +# signal is withheld rather than read as "no branch is merged"; +# - a protected tip that is not in the checkout is not used as an anchor; +# - a branch tip that is not in the checkout is kept, loudly. +# +# A squash or rebase merge leaves no ancestor relationship at all, so its +# branch reads as live here. That is a missed reclamation, not a wrong one, +# and the deleted-branch signal still covers it once the branch is removed. +# # Only ever globs inside . Another project's volume is a different # Docker named volume and is not mounted in this container at all, so "stays # scoped to this repo's cache" holds structurally, not by convention. set -euo pipefail . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cache-lib.sh" -ROOT="${1:?usage: prune-cache.sh }" +ROOT="${1:?usage: prune-cache.sh [own-key] [base-key] [fallback-dir]}" OWN_DIR="${2:?}" PROTECTED_REFS="${3:-}" -MIN_FREE_PCT="${4:-10}" +MIN_FREE_PCT="${4:-0}" +OWN_KEY="${5:-}" +BASE_KEY="${6:-}" +FALLBACK="${7:-}" # Mirrored by daniel/gitdan's ci-cache-reclaim.sh, whose copy must be >= this # one — raising this without raising theirs first lets that script treat a lock # this side still honours as abandoned. Same direction and same reasoning as @@ -102,6 +150,34 @@ STALE_LOCK_SECONDS="${STALE_LOCK_SECONDS:-7200}" # the volume is under pressure. EVICTION_ASIDE_SETTLE_SECONDS="${EVICTION_ASIDE_SETTLE_SECONDS:-60}" +# What the seed step is about to do, resolved through the same function that +# step resolves it with (cache-lib.sh's seed_source_candidates). Empty when it +# will clone nothing at all: its own target dir already exists and it reuses +# it, or no source exists and it starts cold. Either way the derived +# requirement is zero, because nothing is about to be copied. +# +# gb() is for reporting only. Every comparison below is in KB, because +# `read_df` reports KB and rounding a threshold to a tenth of a GB either +# passes a run that cannot fit or evicts a cache the run did not need. +gb() { awk -v k="${1:-0}" 'BEGIN { printf "%.1f", k / 1048576 }'; } + +SEED_SRC="" +CLONE_KB=0 +if [ -n "$OWN_KEY" ]; then + SEED_SRC=$(seed_clone_source "$ROOT" "$OWN_KEY" "$BASE_KEY" "$FALLBACK") + if [ -n "$SEED_SRC" ]; then + # A full walk of the source, and the reason this pass moved ahead of the + # seed rather than staying where it was: the number is only useful before + # the clone it describes. + CLONE_KB=$(clone_headroom_kb "$SEED_SRC") + echo "clone requirement: seeding from $(basename "$SEED_SRC") needs $(gb "$CLONE_KB") GB free (measured from its mutable set)" + else + echo "clone requirement: none — this run reuses its own cache or starts cold, so nothing will be copied" + fi +else + echo "clone requirement: not derivable — no cache key was passed to this pass; the ${MIN_FREE_PCT}% floor is the only gate" +fi + declare -A protected_ns=() for ref in $PROTECTED_REFS; do suffix=$(cache_key "$ref") @@ -109,14 +185,26 @@ for ref in $PROTECTED_REFS; do protected_ns["snapshot-${suffix}"]=1 done -is_protected() { +# Prints why is off limits to every pass, or nothing when it is a +# candidate. The reason is not decoration: it is what the failure report at +# the bottom lists against each directory it kept while running out of space. +# +# SEED_SRC is the third exclusion and the one this script did not used to need. +# The prune ran after the seed, so the source had already been cloned and the +# reader marker over it was gone; running BEFORE the seed puts the directory +# this run is about to read squarely in the candidate set, and pass 1 would +# take it the moment its branch merged. +protected_reason() { local dir="$1" name name=$(basename "$dir") - [ "$dir" = "$OWN_DIR" ] && return 0 - [ -n "${protected_ns[$name]:-}" ] && return 0 + [ "$dir" = "$OWN_DIR" ] && { printf 'this run own cache'; return 0; } + [ -n "$SEED_SRC" ] && [ "$dir" = "$SEED_SRC" ] && { printf 'the source this run is about to clone'; return 0; } + [ -n "${protected_ns[$name]:-}" ] && { printf 'a protected branch cache'; return 0; } return 1 } +is_protected() { protected_reason "$1" >/dev/null; } + # is_locked [name] # # `name` is the directory's own name for reporting and for the reader-marker @@ -277,6 +365,11 @@ done echo "=== pass 1: liveness (unconditional, not gated on free space) ===" declare -A live_ns=() +# The tip SHA origin reports for the branch each directory name belongs to. +# Same output, same loop, one field over — reading it from a second `git` call +# would be reading a different instant. +declare -A live_tip=() +declare -A remote_tip_of=() LIVENESS_AVAILABLE=0 LIVENESS_REASON="" if [ "${CACHE_LIVENESS:-true}" = "0" ] || [ "${CACHE_LIVENESS:-true}" = "false" ]; then @@ -289,9 +382,13 @@ elif remote_heads=$(timeout 20 git ls-remote --heads origin 2>&1); then case "$line" in *refs/heads/*) ;; *) continue ;; esac branch="${line#*refs/heads/}" [ -n "$branch" ] || continue + sha="${line%%[[:space:]]*}" suffix=$(cache_key "$branch") live_ns["target-${suffix}"]=1 live_ns["snapshot-${suffix}"]=1 + live_tip["target-${suffix}"]="$sha" + live_tip["snapshot-${suffix}"]="$sha" + remote_tip_of["$branch"]="$sha" branch_count=$((branch_count + 1)) done <<< "$remote_heads" echo "liveness: ${branch_count} live branches on origin" @@ -299,6 +396,77 @@ else LIVENESS_REASON="git ls-remote --heads origin failed or timed out" fi +# The merged half of pass 1, and whether this checkout can answer it at all. +# Every branch of this decision that ends in "no" ends in the signal being +# WITHHELD, never in a directory being classified dead by default. +MERGED_AVAILABLE=0 +MERGED_REASON="" +PROTECTED_TIPS=() +declare -A merged_verdict=() +declare -A merged_into=() +MERGED_INTO="" +if [ "$LIVENESS_AVAILABLE" = "1" ]; then + if ! git rev-parse --git-dir >/dev/null 2>&1; then + MERGED_REASON="not inside a git checkout" + elif [ "$(git rev-parse --is-shallow-repository 2>/dev/null || echo unknown)" != "false" ]; then + # In a shallow clone an absent commit is the normal case, so `--is-ancestor` + # answers about the graph that was fetched rather than the one that exists. + MERGED_REASON="the checkout is shallow, so a commit missing from it says nothing about ancestry" + else + for ref in $PROTECTED_REFS; do + tip="${remote_tip_of[$ref]:-}" + [ -n "$tip" ] || continue + git cat-file -e "${tip}^{commit}" 2>/dev/null || continue + PROTECTED_TIPS+=("${ref}:${tip}") + done + if [ "${#PROTECTED_TIPS[@]}" -gt 0 ]; then + MERGED_AVAILABLE=1 + echo "liveness: merged-branch detection anchored on ${PROTECTED_TIPS[*]%%:*}" + else + MERGED_REASON="none of the protected branch tips (${PROTECTED_REFS:-none configured}) is present in this checkout" + fi + fi + [ "$MERGED_AVAILABLE" = "1" ] || \ + echo "::warning::liveness: ${MERGED_REASON} — merged-but-undeleted branches keep their caches this run" +fi + +# is_merged_dead +# +# True when the branch this directory belongs to is still on origin but every +# commit it holds is already on a protected branch — a merged PR whose branch +# the forge did not delete, which is the case zemyna hits on every merge and +# which the deleted-branch signal above can never see. +# +# Memoised per tip because target- and snapshot- share one branch, +# and because the "cannot tell" warning belongs to the branch rather than to +# each of its directories. +is_merged_dead() { + local name="$1" tip="${live_tip[$1]:-}" entry ref psha + MERGED_INTO="" + [ "$MERGED_AVAILABLE" = "1" ] || return 1 + [ -n "$tip" ] || return 1 + case "${merged_verdict[$tip]:-}" in + dead) MERGED_INTO="${merged_into[$tip]}"; return 0 ;; + live|unknown) return 1 ;; + esac + if ! git cat-file -e "${tip}^{commit}" 2>/dev/null; then + merged_verdict["$tip"]=unknown + echo "::warning::prune: ${name}: its branch tip ${tip} is not in this checkout — cannot tell merged from live, keeping it" + return 1 + fi + for entry in "${PROTECTED_TIPS[@]}"; do + ref="${entry%%:*}"; psha="${entry#*:}" + if git merge-base --is-ancestor "$tip" "$psha" 2>/dev/null; then + merged_verdict["$tip"]=dead + merged_into["$tip"]="$ref" + MERGED_INTO="$ref" + return 0 + fi + done + merged_verdict["$tip"]=live + return 1 +} + if [ "$LIVENESS_AVAILABLE" = "1" ]; then pruned_any=0 # Tracked separately so the line below cannot contradict the decline lines @@ -309,12 +477,20 @@ if [ "$LIVENESS_AVAILABLE" = "1" ]; then [ -d "$dir" ] || continue name=$(basename "$dir") is_protected "$dir" && continue - [ -n "${live_ns[$name]:-}" ] && continue + if [ -z "${live_ns[$name]:-}" ]; then + why="no matching branch on origin" + why_summary="branch no longer exists on origin" + elif is_merged_dead "$name"; then + why="merged into ${MERGED_INTO}, whose tip already contains its every commit" + why_summary="merged into \`${MERGED_INTO}\`" + else + continue + fi is_locked "$dir" && { spared_any=1; continue; } dir_gb=$(usage_gb "$dir") evict_dir "$dir" || { spared_any=1; continue; } - echo "::warning::pruned dead-branch cache ${name} (${dir_gb} GB) — no matching branch on origin" - summary_line "- pruned dead-branch cache \`${name}\` (${dir_gb} GB) — branch no longer exists on origin" + echo "::warning::pruned dead-branch cache ${name} (${dir_gb} GB) — ${why}" + summary_line "- pruned dead-branch cache \`${name}\` (${dir_gb} GB) — ${why_summary}" pruned_any=1 done if [ "$pruned_any" = "0" ]; then @@ -329,16 +505,45 @@ else fi echo -echo "=== pass 2/3: disk pressure (threshold: free < ${MIN_FREE_PCT}%) ===" +echo "=== pass 2/3: disk pressure ===" read -r TOTAL_KB FREE_KB <<< "$(read_df "$ROOT")" -THRESHOLD_KB=$(( TOTAL_KB * MIN_FREE_PCT / 100 )) +PCT_KB=$(( TOTAL_KB * MIN_FREE_PCT / 100 )) -if [ "$FREE_KB" -ge "$THRESHOLD_KB" ]; then - echo "cache: $(basename "$OWN_DIR") $(usage_gb "$OWN_DIR") GB | $(report_df host "$FREE_KB" "$TOTAL_KB")" +# The two floors, and which of them governs. They are kept apart all the way +# down rather than collapsed here, because falling short of them means +# different things: the derived one predicts that the very next step cannot +# finish, and the percentage one is a hygiene target for the volume. +REQUIRED_KB="$CLONE_KB" +GOVERNS="the clone this run is about to make" +if [ "$PCT_KB" -gt "$REQUIRED_KB" ]; then + REQUIRED_KB="$PCT_KB" + GOVERNS="the ${MIN_FREE_PCT}% floor" +fi +echo "required: $(gb "$REQUIRED_KB") GB free — ${GOVERNS} (clone $(gb "$CLONE_KB") GB, floor $(gb "$PCT_KB") GB)" + +own_report() { + if [ -d "$OWN_DIR" ]; then + echo "cache: $(basename "$OWN_DIR") $(usage_gb "$OWN_DIR") GB | $(report_df host "$1" "$2")" + else + # Ordinary now that this pass runs ahead of the seed: on a branch's first + # run of the day the directory does not exist yet, and reporting 0.0 GB + # for it would read as an emptied cache. + echo "cache: $(basename "$OWN_DIR") not seeded yet | $(report_df host "$1" "$2")" + fi +} + +if [ "$FREE_KB" -ge "$REQUIRED_KB" ]; then + own_report "$FREE_KB" "$TOTAL_KB" exit 0 fi -echo "::warning::$(report_df disk "$FREE_KB" "$TOTAL_KB") < ${MIN_FREE_PCT}% threshold" +echo "::warning::$(report_df disk "$FREE_KB" "$TOTAL_KB") < the $(gb "$REQUIRED_KB") GB this run requires" + +# What survived the pass, and why, in the order the pass considered them. Read +# only by the failure report at the bottom: a run that cannot fit its clone is +# a run whose log has to answer "then what is all that space?" without anyone +# having to reconstruct the pass by hand. +KEPT=() # A plain loop over a pre-materialised, pre-sorted list rather than a live # `find | while` pipeline, so `rm -rf` inside the loop cannot make a running @@ -346,21 +551,56 @@ echo "::warning::$(report_df disk "$FREE_KB" "$TOTAL_KB") < ${MIN_FREE_PCT}% thr mapfile -t LRU < <(list_by_lru) for dir in "${LRU[@]}"; do [ -d "$dir" ] || continue - is_protected "$dir" && continue + if reason=$(protected_reason "$dir"); then + KEPT+=("$(basename "$dir") — ${reason}") + continue + fi read -r TOTAL_KB FREE_KB <<< "$(read_df "$ROOT")" - [ "$FREE_KB" -ge "$THRESHOLD_KB" ] && break - is_locked "$dir" && continue + [ "$FREE_KB" -ge "$REQUIRED_KB" ] && break + if is_locked "$dir"; then + KEPT+=("$(basename "$dir") — held open by a running job") + continue + fi dir_gb=$(usage_gb "$dir") - evict_dir "$dir" || continue + if ! evict_dir "$dir"; then + KEPT+=("$(basename "$dir") — claimed by a job while its eviction was in flight") + continue + fi echo "::warning::evicted $(basename "$dir") (${dir_gb} GB, LRU under disk pressure)" summary_line "- evicted \`$(basename "$dir")\` (${dir_gb} GB, LRU under disk pressure)" done read -r TOTAL_KB FREE_KB <<< "$(read_df "$ROOT")" -if [ "$FREE_KB" -lt "$THRESHOLD_KB" ]; then + +# Falling short of the DERIVED requirement is a failure, not a warning. The +# seed step is next, it will clone that source, and it will run out of disk +# part-way through unsharing the clone's mutable paths — reported against a +# staging path, with nothing in the message about which cache was holding the +# space. Failing here says that instead. +# +# There is nothing to self-clear on this path and it is not skipped in error: +# a non-zero requirement means the seed is about to CLONE, which means this +# run has no own target dir to wipe (seed_clone_source returns nothing when it +# does), so pass 3 has no candidate. See the header. +if [ "$FREE_KB" -lt "$CLONE_KB" ]; then + echo "::error::prune: $(gb "$FREE_KB") GB free after evicting every eligible cache, but seeding from $(basename "$SEED_SRC") needs $(gb "$CLONE_KB") GB — short by $(gb "$(( CLONE_KB - FREE_KB ))") GB" + summary_line "- **out of disk**: seeding from \`$(basename "$SEED_SRC")\` needs $(gb "$CLONE_KB") GB, $(gb "$FREE_KB") GB free" + echo "prune: kept, and why:" + for entry in ${KEPT[@]+"${KEPT[@]}"}; do echo " ${entry}"; done + [ "${#KEPT[@]}" -gt 0 ] || echo " (nothing — the volume holds no cache directories at all)" + exit 1 +fi + +if [ "$FREE_KB" -lt "$PCT_KB" ]; then OWN_GB=$(usage_gb "$OWN_DIR") echo "::warning::still under threshold after evicting every eligible sibling; clearing own $(basename "$OWN_DIR") (was ${OWN_GB} GB) — this run pays a cold rebuild" summary_line "- **self-clear**: \`$(basename "$OWN_DIR")\` (was ${OWN_GB} GB) wiped — this run pays a cold rebuild" + # Recreated empty rather than left absent, and that is what keeps this path + # out of the requirement above: the seed reuses an own target dir that + # exists, whatever is in it, so a self-cleared run clones nothing and needs + # no headroom. Leaving it absent would send that run to the base snapshot + # instead, needing a clone this pass has just spent its last eligible bytes + # not sizing for. rm -rf "$OWN_DIR" mkdir -p "$OWN_DIR" else