fix(prune): reclaim merged branches and size the volume for the clone #21
+54
-19
@@ -29,9 +29,18 @@ inputs:
|
||||
required: false
|
||||
default: 'dev main'
|
||||
min-free-percent:
|
||||
description: 'Prune when free space on the cache volume drops below this percentage.'
|
||||
description: >-
|
||||
An ADDITIONAL free-space floor, as a percentage of the cache volume.
|
||||
The prune's own requirement is derived per run from what the seed is
|
||||
about to clone — the mutable set it has to real-copy out of the source
|
||||
snapshot, which is a property of that snapshot and not of the volume —
|
||||
and this floor only ever raises it. 0, the default, leaves the derived
|
||||
requirement as the only gate. Set it to keep headroom for something
|
||||
other than the clone (the build's own output, another job on the same
|
||||
volume); it will not make the clone fit, because it does not know how
|
||||
big the clone is.
|
||||
required: false
|
||||
default: '10'
|
||||
default: '0'
|
||||
restore-mtimes:
|
||||
description: >-
|
||||
Restore every tracked file's mtime from git history. Requires a
|
||||
@@ -40,13 +49,20 @@ inputs:
|
||||
required: false
|
||||
default: 'true'
|
||||
prune:
|
||||
description: 'Run the eviction pass (dead-branch liveness + disk pressure).'
|
||||
description: >-
|
||||
Run the eviction pass (dead-branch liveness + disk pressure). It runs
|
||||
BEFORE the seed step, so what it frees is available to the clone that
|
||||
step makes.
|
||||
required: false
|
||||
default: 'true'
|
||||
liveness-prune:
|
||||
description: >-
|
||||
Within the prune pass, remove caches for branches that no longer exist
|
||||
on origin. Set to false on a runner that cannot reach origin.
|
||||
Within the prune pass, remove caches for branches that are dead: gone
|
||||
from origin, or still on origin with a tip already merged into a
|
||||
protected branch. The second signal is what reclaims anything at all on
|
||||
a forge that keeps branches after merge, and it needs the protected
|
||||
branches' commits in the checkout — a shallow one withholds it and says
|
||||
so. Set to false on a runner that cannot reach origin.
|
||||
required: false
|
||||
default: 'true'
|
||||
own-ref:
|
||||
@@ -169,6 +185,39 @@ runs:
|
||||
echo "CI_WATERMARK_FILE=${WATERMARK}"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
# Runs BEFORE the seed, which is the only order in which its work can
|
||||
# help: the eviction it performs is what makes room for the clone the seed
|
||||
# step is about to make, and the requirement it evicts against is measured
|
||||
# off the snapshot that clone will read. Running afterwards — where this
|
||||
# step used to be — meant every run freed space for the NEXT one and the
|
||||
# seed met whatever the last run happened to leave.
|
||||
#
|
||||
# Two things this ordering has to be safe against, and is:
|
||||
#
|
||||
# The source it is about to read is excluded from every pass by name
|
||||
# (see protected_reason in prune-cache.sh), so pass 1 cannot take the
|
||||
# snapshot out from under the seed that follows it.
|
||||
# A concurrent job's target dir carries its lock from the instant it
|
||||
# appears under its final name — the seed writes it into the staging
|
||||
# tree before the rename — so there is no window in which running this
|
||||
# earlier sees an unlocked directory somebody is using.
|
||||
- if: ${{ inputs.prune == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
STALE_LOCK_SECONDS: ${{ inputs.stale-lock-seconds }}
|
||||
CACHE_LIVENESS: ${{ inputs.liveness-prune }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SCRIPTS=$(cd "${{ github.action_path }}/.." && pwd)/scripts
|
||||
bash "${SCRIPTS}/prune-cache.sh" \
|
||||
"${{ steps.resolve.outputs.cache-root }}" \
|
||||
"${{ steps.resolve.outputs.target-dir }}" \
|
||||
"${{ inputs.protected-branches }}" \
|
||||
"${{ inputs.min-free-percent }}" \
|
||||
"${{ steps.resolve.outputs.cache-key }}" \
|
||||
"${{ steps.resolve.outputs.base-key }}" \
|
||||
"${{ inputs.seed-fallback-dir }}"
|
||||
|
||||
# Seeds this ref's target dir from the base's published snapshot. See
|
||||
# scripts/seed-target-dir.sh — the staging-then-atomic-rename is what
|
||||
# makes concurrent jobs sharing one cache key safe by construction rather
|
||||
@@ -228,17 +277,3 @@ runs:
|
||||
CARGO_TARGET_DIR: ${{ steps.resolve.outputs.target-dir }}
|
||||
CI_WATERMARK_FILE: ${{ steps.resolve.outputs.watermark-file }}
|
||||
run: bash "$(cd "${{ github.action_path }}/.." && pwd)/scripts/restore-mtimes.sh"
|
||||
|
||||
- if: ${{ inputs.prune == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
STALE_LOCK_SECONDS: ${{ inputs.stale-lock-seconds }}
|
||||
CACHE_LIVENESS: ${{ inputs.liveness-prune }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SCRIPTS=$(cd "${{ github.action_path }}/.." && pwd)/scripts
|
||||
bash "${SCRIPTS}/prune-cache.sh" \
|
||||
"${{ steps.resolve.outputs.cache-root }}" \
|
||||
"${{ steps.resolve.outputs.target-dir }}" \
|
||||
"${{ inputs.protected-branches }}" \
|
||||
"${{ inputs.min-free-percent }}"
|
||||
|
||||
@@ -46,10 +46,36 @@
|
||||
# empty a tree its owner may still restore under a live cache name. Its
|
||||
# fixture is an OLD directory renamed a moment ago — production's shape,
|
||||
# and what lets it tell the two timestamps apart.
|
||||
# 15. A MERGED-BUT-UNDELETED BRANCH IS DEAD TOO. This forge keeps branches
|
||||
# after merge, so `ls-remote` reports them forever and scenario 1's
|
||||
# signal never fires for them — which is how three 40 GB caches sat on a
|
||||
# full volume until somebody removed them by hand (gitdan-actions#20). A
|
||||
# branch whose tip is an ancestor of a protected branch's tip is pruned
|
||||
# like a deleted one; an unmerged branch beside it is not.
|
||||
# 16. AND "CANNOT TELL" IS STILL NOT DEATH, at both granularities: a branch
|
||||
# whose tip is not in this checkout is kept with a warning naming it,
|
||||
# and a shallow checkout — where a missing object is the normal case —
|
||||
# withholds the whole signal rather than reading it as "nothing merged".
|
||||
# The deleted-branch signal keeps working in both.
|
||||
# 17. THE FREE-SPACE REQUIREMENT IS MEASURED OFF THE SOURCE, not taken as a
|
||||
# percentage of the volume: the pass evicts until the clone the seed is
|
||||
# about to make fits, and stops there rather than draining the volume.
|
||||
# When it cannot get there it FAILS, naming the shortfall and every
|
||||
# directory it kept instead — because the seed would otherwise fail
|
||||
# seconds later against a staging path that names nothing.
|
||||
# 18. AND ON THE LAYOUT THAT PRODUCED THE BUG: three equal-sized caches, one
|
||||
# of them a merged-but-undeleted branch's, with disk to spare. Exactly
|
||||
# that one goes. Equal sizes and no pressure are the point — nothing but
|
||||
# the merge state can be what decides.
|
||||
set -euo pipefail
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$script_dir/cache-lib.sh"
|
||||
prune="$script_dir/prune-cache.sh"
|
||||
# Two scenarios below put a stub of a real tool on PATH for one command.
|
||||
# Captured once, here, rather than read back at each of those sites: a `$PATH`
|
||||
# read after the first of them is indistinguishable, to a static check, from
|
||||
# reading the modification the subshell lost.
|
||||
outer_path="$PATH"
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
@@ -235,7 +261,7 @@ done
|
||||
exec "$real_du" "\$@"
|
||||
EOF
|
||||
chmod +x "$scratch/bin/du"
|
||||
( PATH="$scratch/bin:$PATH"; run_prune "1000000 900000" )
|
||||
( PATH="$scratch/bin:$outer_path"; run_prune "1000000 900000" )
|
||||
[ -e "$root/.reading-target-$DEAD-racer" ] || fail "the racing marker was never published — scenario 12 proves nothing"
|
||||
assert_kept "$root/target-$DEAD" "a cache claimed inside the eviction window is not unlinked"
|
||||
assert_kept "$root/target-$DEAD/blob" "the reprieved cache still has its contents"
|
||||
@@ -286,5 +312,172 @@ assert_kept "$aside" "an aside younger than the settle window is not reclaimed"
|
||||
assert_kept "$aside/blob" "and is left intact, not part-way emptied"
|
||||
assert_log "may still be evicting it" "the deferral gives its actual reason"
|
||||
|
||||
echo
|
||||
echo "=== 15: a merged-but-undeleted branch is dead too ==="
|
||||
# This forge keeps a PR's branch after the merge, so `ls-remote` reports it
|
||||
# forever. Built the way that happens: a branch merged into dev with a merge
|
||||
# commit, still pushed, beside one branched at the same point and NOT merged.
|
||||
git="git -c user.email=t@t -c user.name=t -c commit.gpgsign=false"
|
||||
(
|
||||
cd "$work"
|
||||
git checkout -q dev
|
||||
git checkout -q -b feat/merged
|
||||
$git commit -q --allow-empty -m merged
|
||||
git checkout -q dev
|
||||
$git merge -q --no-ff feat/merged -m "merge feat/merged"
|
||||
git checkout -q -b feat/unmerged
|
||||
$git commit -q --allow-empty -m unmerged
|
||||
git checkout -q dev
|
||||
git push -q origin dev feat/merged feat/unmerged
|
||||
)
|
||||
MERGED=$(cache_key feat/merged); UNMERGED=$(cache_key feat/unmerged)
|
||||
reset_cache
|
||||
mk "target-$MERGED" '2030-01-01'
|
||||
mk "snapshot-$MERGED" '2030-01-01'
|
||||
mk "target-$UNMERGED" '2020-01-01' # older, deliberately: merge state decides, not age
|
||||
run_prune "1000000 900000" # 90% free: no pressure at all
|
||||
assert_log "merged-branch detection anchored on" "the pass says what it anchored ancestry on"
|
||||
assert_gone "$root/target-$MERGED" "a merged branch's cache is pruned though its branch is still on origin"
|
||||
assert_gone "$root/snapshot-$MERGED" "and so is its snapshot"
|
||||
assert_kept "$root/target-$UNMERGED" "an unmerged branch's cache survives, though it is the older of the two"
|
||||
assert_log "merged into dev" "the eviction names the branch it was merged into"
|
||||
|
||||
echo
|
||||
echo "=== 16: 'cannot tell' is not death, per branch and per checkout ==="
|
||||
# A branch whose tip this checkout has never seen. Pushed from a second clone,
|
||||
# so `ls-remote` reports a SHA that `$work` holds no object for — which is
|
||||
# what "cannot determine" actually looks like, rather than a stubbed failure.
|
||||
other="$scratch/other"; git clone -q "$origin" "$other"
|
||||
(
|
||||
cd "$other"
|
||||
git checkout -q -b feat/elsewhere origin/dev
|
||||
git -c user.email=t@t -c user.name=t -c commit.gpgsign=false commit -q --allow-empty -m elsewhere
|
||||
git push -q origin feat/elsewhere
|
||||
)
|
||||
ELSEWHERE=$(cache_key feat/elsewhere)
|
||||
reset_cache
|
||||
mk "target-$ELSEWHERE" '2030-01-01'
|
||||
mk "target-$MERGED" '2030-01-01'
|
||||
run_prune "1000000 900000"
|
||||
assert_kept "$root/target-$ELSEWHERE" "a branch whose tip is not in this checkout is kept, not classified dead"
|
||||
assert_log "cannot tell merged from live" "and the undecidable branch is named, not silently skipped"
|
||||
assert_gone "$root/target-$MERGED" "while a branch it CAN decide is still pruned in the same pass"
|
||||
|
||||
# A shallow checkout, where a missing object is the ordinary case rather than
|
||||
# a signal — so the whole merged half is withheld. The deleted-branch half is
|
||||
# unaffected, which is what keeps this a narrowing rather than an outage.
|
||||
shallow="$scratch/shallow"; git clone -q --depth 1 -b dev "file://$origin" "$shallow"
|
||||
[ "$(git -C "$shallow" rev-parse --is-shallow-repository)" = true ] \
|
||||
|| fail "the fixture clone is not shallow — scenario 16's second half proves nothing"
|
||||
reset_cache
|
||||
mk "target-$MERGED" '2030-01-01'
|
||||
(
|
||||
cd "$shallow"
|
||||
CACHE_DF_OVERRIDE="1000000 900000" GITHUB_STEP_SUMMARY="$scratch/summary" \
|
||||
bash "$prune" "$root" "$root/target-$OWN" "dev main" 10
|
||||
) > "$scratch/log" 2>&1 || { cat "$scratch/log"; fail "prune-cache.sh exited non-zero in a shallow checkout"; }
|
||||
assert_log "checkout is shallow" "a shallow checkout withholds the merged signal and says why"
|
||||
assert_kept "$root/target-$MERGED" "and keeps a merged branch's cache rather than guessing"
|
||||
assert_gone "$root/target-$DEAD" "while the deleted-branch signal still fires"
|
||||
|
||||
echo
|
||||
echo "=== 17: the free-space requirement is measured off the clone's source ==="
|
||||
# A `df` that answers from the cache root's actual size, because the property
|
||||
# under test is that the pass STOPS once the requirement is met — which a
|
||||
# fixed CACHE_DF_OVERRIDE cannot express, since evicting never changes it.
|
||||
mkdir -p "$scratch/bin17"
|
||||
real_du=$(command -v du)
|
||||
build_seed_fixture() {
|
||||
rm -rf "$root"; mkdir -p "$root"
|
||||
# The source the seed is about to clone. 8 MB of dep-info, which
|
||||
# unshare_mutable_paths has to real-copy, beside 16 MB of .rlib that it
|
||||
# leaves hardlinked — so a requirement derived from the SIZE of the source
|
||||
# would be three times the one derived from its mutable set.
|
||||
mkdir -p "$root/snapshot-$DEV/debug/.fingerprint/unit" "$root/snapshot-$DEV/debug/deps"
|
||||
head -c $((8 * 1024 * 1024)) /dev/zero > "$root/snapshot-$DEV/debug/.fingerprint/unit/dep-lib"
|
||||
head -c $((16 * 1024 * 1024)) /dev/zero > "$root/snapshot-$DEV/debug/deps/libx.rlib"
|
||||
touch -d '2020-01-01' "$root/snapshot-$DEV/.cache-last-used"
|
||||
# Three live, unmerged branches' caches of 4 MB each, oldest first.
|
||||
local i=0
|
||||
for b in a b c; do
|
||||
i=$((i + 1))
|
||||
mkdir -p "$root/target-$(cache_key "feat/$b")"
|
||||
head -c $((4 * 1024 * 1024)) /dev/zero > "$root/target-$(cache_key "feat/$b")/blob"
|
||||
touch -d "202${i}-01-01" "$root/target-$(cache_key "feat/$b")/.cache-last-used"
|
||||
done
|
||||
# A volume with 2 MB to spare: under the requirement, over nothing else.
|
||||
cap=$(( $($real_du -sk "$root" | awk '{print $1}') + 2048 ))
|
||||
cat > "$scratch/bin17/df" <<DFEOF
|
||||
#!/usr/bin/env bash
|
||||
used=\$($real_du -sk "$root" | awk '{print \$1}')
|
||||
echo "Filesystem 1024-blocks Used Available Capacity Mounted-on"
|
||||
echo "fake $cap \$used \$(( $cap - used )) 50% $root"
|
||||
DFEOF
|
||||
chmod +x "$scratch/bin17/df"
|
||||
}
|
||||
(
|
||||
cd "$work"
|
||||
for b in a b c; do
|
||||
git checkout -q dev
|
||||
git checkout -q -b "feat/$b"
|
||||
git -c user.email=t@t -c user.name=t -c commit.gpgsign=false commit -q --allow-empty -m "$b"
|
||||
done
|
||||
git checkout -q dev
|
||||
git push -q origin feat/a feat/b feat/c
|
||||
)
|
||||
# run_seeded_prune <own-ref> <base-ref> — the form cargo-cache/action.yml uses:
|
||||
# the same pass, told what the seed step it now runs ahead of will clone. The
|
||||
# headroom knobs are pinned so the arithmetic is the fixture's, not the
|
||||
# defaults' (whose 2 GiB floor would dwarf any fixture on a test host).
|
||||
seeded_rc=0
|
||||
run_seeded_prune() {
|
||||
seeded_rc=0
|
||||
PATH="$scratch/bin17:$outer_path" \
|
||||
CACHE_CLONE_HEADROOM_PERCENT=100 CACHE_CLONE_HEADROOM_FLOOR_KB=1024 \
|
||||
GITHUB_STEP_SUMMARY="$scratch/summary" \
|
||||
bash "$prune" "$root" "$root/target-$(cache_key "$1")" "dev main" 0 \
|
||||
"$(cache_key "$1")" "$(cache_key "$2")" "" \
|
||||
> "$scratch/log" 2>&1 || seeded_rc=$?
|
||||
}
|
||||
|
||||
build_seed_fixture
|
||||
run_seeded_prune feat/own dev
|
||||
[ "$seeded_rc" = 0 ] || { cat "$scratch/log"; fail "prune-cache.sh exited ${seeded_rc} with the requirement satisfiable"; }
|
||||
assert_log "measured from its mutable set" "the requirement says where it came from"
|
||||
assert_gone "$root/target-$(cache_key feat/a)" "the oldest cache is evicted to make room for the clone"
|
||||
assert_gone "$root/target-$(cache_key feat/b)" "and the next oldest, because one was not enough"
|
||||
assert_kept "$root/target-$(cache_key feat/c)" "and the pass STOPS there rather than draining the volume"
|
||||
assert_kept "$root/snapshot-$DEV" "the source the seed is about to clone is never a candidate"
|
||||
|
||||
# Nothing eligible: every sibling is held open by a running job. The pass
|
||||
# cannot reach the requirement, and the seed that follows would fail against a
|
||||
# staging path naming none of this.
|
||||
build_seed_fixture
|
||||
for b in a b c; do date +%s > "$root/target-$(cache_key "feat/$b")/.ci-lock-ci-1"; done
|
||||
run_seeded_prune feat/own dev
|
||||
[ "$seeded_rc" = 1 ] || { cat "$scratch/log"; fail "expected exit 1 when the clone cannot fit, got ${seeded_rc}" ; }
|
||||
ok "a clone that cannot be made to fit fails the pass rather than the seed"
|
||||
assert_log "short by" "the failure names the shortfall"
|
||||
assert_log "held open by a running job" "and what was kept instead of it, with the reason"
|
||||
assert_kept "$root/target-$(cache_key feat/a)" "a locked cache is still not evicted, however tight the disk"
|
||||
|
||||
echo
|
||||
echo "=== 18: the layout that produced the bug ==="
|
||||
# zemyna's volume on 2026-09-07: the base branch's snapshot and target dir,
|
||||
# plus one target dir for a branch merged the day before and never deleted.
|
||||
# Equal sizes and 90% free, so neither age nor pressure nor size can be what
|
||||
# decides — only the merge state.
|
||||
rm -rf "$root"; mkdir -p "$root"
|
||||
mk "snapshot-$DEV" '2026-09-01'
|
||||
mk "target-$DEV" '2026-09-01'
|
||||
mk "target-$MERGED" '2026-09-06'
|
||||
run_prune "1000000 900000"
|
||||
assert_kept "$root/snapshot-$DEV" "the base snapshot stays"
|
||||
assert_kept "$root/target-$DEV" "and the base target dir stays"
|
||||
assert_gone "$root/target-$MERGED" "and the merged-but-undeleted branch's cache is the one reclaimed"
|
||||
[ "$(grep -c 'pruned dead-branch cache' "$scratch/log")" = 1 ] \
|
||||
|| { cat "$scratch/log"; fail "expected exactly one eviction on the zemyna layout"; }
|
||||
ok "exactly one directory is evicted, and it is that one"
|
||||
|
||||
echo
|
||||
echo "prune-cache-selftest: ${pass_count} assertions passed"
|
||||
|
||||
+272
-32
@@ -1,8 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# Eviction for the per-ref cache directories on the persistent volume.
|
||||
#
|
||||
# Usage: prune-cache.sh <cache-root> <own-target-dir> <protected-branches> <min-free-percent>
|
||||
# Usage: prune-cache.sh <cache-root> <own-target-dir> <protected-branches> \
|
||||
# <min-free-percent> [own-key] [base-key] [fallback-dir]
|
||||
# protected-branches space-separated raw refs (e.g. "dev main")
|
||||
# min-free-percent a FLOOR, not the gate — 0 to rely on the derived
|
||||
# requirement alone (the default)
|
||||
# own-key, base-key, fallback-dir
|
||||
# the same three the seed step resolves its source from.
|
||||
# Given them, this pass sizes the volume for the clone
|
||||
# that step is about to make; without them it has only
|
||||
# the percentage floor, and says so.
|
||||
#
|
||||
# Optional environment:
|
||||
# STALE_LOCK_SECONDS age past which a .ci-lock-* marker is treated as
|
||||
@@ -16,19 +24,41 @@
|
||||
#
|
||||
# Three passes, in order:
|
||||
#
|
||||
# 1. LIVENESS — every target-*/snapshot-* directory whose branch no longer
|
||||
# exists on origin is removed UNCONDITIONALLY, not gated on free space.
|
||||
# A directory for a branch deleted days ago is pure loss: nothing will
|
||||
# ever read it again, since a merged PR's branch cannot be reopened.
|
||||
# Waiting for disk pressure to notice means paying for it until then.
|
||||
# Skipped entirely, loudly, if the liveness signal itself is
|
||||
# unavailable — "couldn't determine" is never folded into "dead".
|
||||
# 2. PRESSURE — if free space is still under the threshold, evict remaining
|
||||
# (now necessarily live) directories oldest-first until it clears.
|
||||
# 3. SELF-CLEAR — if pass 2 still isn't enough, wipe this run's own target
|
||||
# dir and pay a cold rebuild, reported to the job summary as well as the
|
||||
# log, because a warning on a green run is what lets a silently 4x-slower
|
||||
# job go unnoticed.
|
||||
# 1. LIVENESS — every target-*/snapshot-* directory whose branch is DEAD is
|
||||
# removed UNCONDITIONALLY, not gated on free space. A directory for a
|
||||
# branch nothing will build again is pure loss; waiting for disk pressure
|
||||
# to notice means paying for it until then. Two signals make a branch
|
||||
# dead, and the second exists because the first alone is inert on a forge
|
||||
# that keeps branches after merge (gitdan-actions#20):
|
||||
#
|
||||
# DELETED — the branch is no longer on origin at all.
|
||||
# MERGED — the branch is still on origin, but its tip is an ancestor
|
||||
# of a protected branch's tip, so every commit it holds is
|
||||
# already on the branch its cache would be re-cloned from.
|
||||
#
|
||||
# Skipped entirely, loudly, if the signal itself is unavailable —
|
||||
# "couldn't determine" is never folded into "dead", for either signal
|
||||
# and at either granularity: a checkout that cannot answer ancestry at
|
||||
# all skips the merged half, and a single branch whose tip is not in the
|
||||
# checkout is kept with a warning naming it.
|
||||
# 2. PRESSURE — if free space is under the requirement, evict remaining
|
||||
# (now necessarily live) directories oldest-first until it clears. The
|
||||
# requirement is what the seed step is about to need to clone its source,
|
||||
# MEASURED off that source (see clone_headroom_kb in cache-lib.sh), and a
|
||||
# percentage floor only if one is configured. A percentage cannot express
|
||||
# this: the failure it has to prevent is a clone running out of disk
|
||||
# part-way through unsharing its mutable paths, and how much that needs
|
||||
# is a property of the snapshot, not of the volume.
|
||||
# 3. SELF-CLEAR — if pass 2 still isn't enough for the percentage floor,
|
||||
# wipe this run's own target dir and pay a cold rebuild, reported to the
|
||||
# job summary as well as the log, because a warning on a green run is
|
||||
# what lets a silently 4x-slower job go unnoticed. It is not a way out of
|
||||
# the derived requirement: a run that has an own target dir to wipe is a
|
||||
# run whose seed reuses it and clones nothing, so that requirement is
|
||||
# zero. Falling short of a NON-ZERO derived requirement fails the job
|
||||
# here, naming the shortfall and what was kept instead of it — the seed
|
||||
# would otherwise fail seconds later against a half-unshared staging
|
||||
# tree, which is the failure this pass exists to pre-empt.
|
||||
#
|
||||
# Reactive-only, with no hard cap on cache size: a workspace's natural working
|
||||
# set is what it is, and bounding the footprint preemptively means wiping
|
||||
@@ -75,16 +105,34 @@
|
||||
# than reimplementing a lookalike is what makes the classification sound; any
|
||||
# drift between two spellings would silently misclassify every directory.
|
||||
#
|
||||
# The merged half reads the TIP SHA out of that same `ls-remote` output and
|
||||
# asks `git merge-base --is-ancestor` against each protected branch's tip,
|
||||
# using the objects in this job's own checkout. Ancestry is only decidable
|
||||
# where the objects are there to decide it, so the answer "I cannot tell"
|
||||
# exists and is distinct from "not merged" everywhere it can arise:
|
||||
#
|
||||
# - a shallow checkout makes a MISSING object prove nothing, so the whole
|
||||
# signal is withheld rather than read as "no branch is merged";
|
||||
# - a protected tip that is not in the checkout is not used as an anchor;
|
||||
# - a branch tip that is not in the checkout is kept, loudly.
|
||||
#
|
||||
# A squash or rebase merge leaves no ancestor relationship at all, so its
|
||||
# branch reads as live here. That is a missed reclamation, not a wrong one,
|
||||
# and the deleted-branch signal still covers it once the branch is removed.
|
||||
#
|
||||
# Only ever globs inside <cache-root>. Another project's volume is a different
|
||||
# Docker named volume and is not mounted in this container at all, so "stays
|
||||
# scoped to this repo's cache" holds structurally, not by convention.
|
||||
set -euo pipefail
|
||||
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cache-lib.sh"
|
||||
|
||||
ROOT="${1:?usage: prune-cache.sh <cache-root> <own-target-dir> <protected-branches> <min-free-percent>}"
|
||||
ROOT="${1:?usage: prune-cache.sh <cache-root> <own-target-dir> <protected-branches> <min-free-percent> [own-key] [base-key] [fallback-dir]}"
|
||||
OWN_DIR="${2:?}"
|
||||
PROTECTED_REFS="${3:-}"
|
||||
MIN_FREE_PCT="${4:-10}"
|
||||
MIN_FREE_PCT="${4:-0}"
|
||||
OWN_KEY="${5:-}"
|
||||
BASE_KEY="${6:-}"
|
||||
FALLBACK="${7:-}"
|
||||
# Mirrored by daniel/gitdan's ci-cache-reclaim.sh, whose copy must be >= this
|
||||
# one — raising this without raising theirs first lets that script treat a lock
|
||||
# this side still honours as abandoned. Same direction and same reasoning as
|
||||
@@ -102,6 +150,34 @@ STALE_LOCK_SECONDS="${STALE_LOCK_SECONDS:-7200}"
|
||||
# the volume is under pressure.
|
||||
EVICTION_ASIDE_SETTLE_SECONDS="${EVICTION_ASIDE_SETTLE_SECONDS:-60}"
|
||||
|
||||
# What the seed step is about to do, resolved through the same function that
|
||||
# step resolves it with (cache-lib.sh's seed_source_candidates). Empty when it
|
||||
# will clone nothing at all: its own target dir already exists and it reuses
|
||||
# it, or no source exists and it starts cold. Either way the derived
|
||||
# requirement is zero, because nothing is about to be copied.
|
||||
#
|
||||
# gb() is for reporting only. Every comparison below is in KB, because
|
||||
# `read_df` reports KB and rounding a threshold to a tenth of a GB either
|
||||
# passes a run that cannot fit or evicts a cache the run did not need.
|
||||
gb() { awk -v k="${1:-0}" 'BEGIN { printf "%.1f", k / 1048576 }'; }
|
||||
|
||||
SEED_SRC=""
|
||||
CLONE_KB=0
|
||||
if [ -n "$OWN_KEY" ]; then
|
||||
SEED_SRC=$(seed_clone_source "$ROOT" "$OWN_KEY" "$BASE_KEY" "$FALLBACK")
|
||||
if [ -n "$SEED_SRC" ]; then
|
||||
# A full walk of the source, and the reason this pass moved ahead of the
|
||||
# seed rather than staying where it was: the number is only useful before
|
||||
# the clone it describes.
|
||||
CLONE_KB=$(clone_headroom_kb "$SEED_SRC")
|
||||
echo "clone requirement: seeding from $(basename "$SEED_SRC") needs $(gb "$CLONE_KB") GB free (measured from its mutable set)"
|
||||
else
|
||||
echo "clone requirement: none — this run reuses its own cache or starts cold, so nothing will be copied"
|
||||
fi
|
||||
else
|
||||
echo "clone requirement: not derivable — no cache key was passed to this pass; the ${MIN_FREE_PCT}% floor is the only gate"
|
||||
fi
|
||||
|
||||
declare -A protected_ns=()
|
||||
for ref in $PROTECTED_REFS; do
|
||||
suffix=$(cache_key "$ref")
|
||||
@@ -109,14 +185,26 @@ for ref in $PROTECTED_REFS; do
|
||||
protected_ns["snapshot-${suffix}"]=1
|
||||
done
|
||||
|
||||
is_protected() {
|
||||
# Prints why <dir> is off limits to every pass, or nothing when it is a
|
||||
# candidate. The reason is not decoration: it is what the failure report at
|
||||
# the bottom lists against each directory it kept while running out of space.
|
||||
#
|
||||
# SEED_SRC is the third exclusion and the one this script did not used to need.
|
||||
# The prune ran after the seed, so the source had already been cloned and the
|
||||
# reader marker over it was gone; running BEFORE the seed puts the directory
|
||||
# this run is about to read squarely in the candidate set, and pass 1 would
|
||||
# take it the moment its branch merged.
|
||||
protected_reason() {
|
||||
local dir="$1" name
|
||||
name=$(basename "$dir")
|
||||
[ "$dir" = "$OWN_DIR" ] && return 0
|
||||
[ -n "${protected_ns[$name]:-}" ] && return 0
|
||||
[ "$dir" = "$OWN_DIR" ] && { printf 'this run own cache'; return 0; }
|
||||
[ -n "$SEED_SRC" ] && [ "$dir" = "$SEED_SRC" ] && { printf 'the source this run is about to clone'; return 0; }
|
||||
[ -n "${protected_ns[$name]:-}" ] && { printf 'a protected branch cache'; return 0; }
|
||||
return 1
|
||||
}
|
||||
|
||||
is_protected() { protected_reason "$1" >/dev/null; }
|
||||
|
||||
# is_locked <dir> [name]
|
||||
#
|
||||
# `name` is the directory's own name for reporting and for the reader-marker
|
||||
@@ -277,6 +365,11 @@ done
|
||||
|
||||
echo "=== pass 1: liveness (unconditional, not gated on free space) ==="
|
||||
declare -A live_ns=()
|
||||
# The tip SHA origin reports for the branch each directory name belongs to.
|
||||
# Same output, same loop, one field over — reading it from a second `git` call
|
||||
# would be reading a different instant.
|
||||
declare -A live_tip=()
|
||||
declare -A remote_tip_of=()
|
||||
LIVENESS_AVAILABLE=0
|
||||
LIVENESS_REASON=""
|
||||
if [ "${CACHE_LIVENESS:-true}" = "0" ] || [ "${CACHE_LIVENESS:-true}" = "false" ]; then
|
||||
@@ -289,9 +382,13 @@ elif remote_heads=$(timeout 20 git ls-remote --heads origin 2>&1); then
|
||||
case "$line" in *refs/heads/*) ;; *) continue ;; esac
|
||||
branch="${line#*refs/heads/}"
|
||||
[ -n "$branch" ] || continue
|
||||
sha="${line%%[[:space:]]*}"
|
||||
suffix=$(cache_key "$branch")
|
||||
live_ns["target-${suffix}"]=1
|
||||
live_ns["snapshot-${suffix}"]=1
|
||||
live_tip["target-${suffix}"]="$sha"
|
||||
live_tip["snapshot-${suffix}"]="$sha"
|
||||
remote_tip_of["$branch"]="$sha"
|
||||
branch_count=$((branch_count + 1))
|
||||
done <<< "$remote_heads"
|
||||
echo "liveness: ${branch_count} live branches on origin"
|
||||
@@ -299,6 +396,77 @@ else
|
||||
LIVENESS_REASON="git ls-remote --heads origin failed or timed out"
|
||||
fi
|
||||
|
||||
# The merged half of pass 1, and whether this checkout can answer it at all.
|
||||
# Every branch of this decision that ends in "no" ends in the signal being
|
||||
# WITHHELD, never in a directory being classified dead by default.
|
||||
MERGED_AVAILABLE=0
|
||||
MERGED_REASON=""
|
||||
PROTECTED_TIPS=()
|
||||
declare -A merged_verdict=()
|
||||
declare -A merged_into=()
|
||||
MERGED_INTO=""
|
||||
if [ "$LIVENESS_AVAILABLE" = "1" ]; then
|
||||
if ! git rev-parse --git-dir >/dev/null 2>&1; then
|
||||
MERGED_REASON="not inside a git checkout"
|
||||
elif [ "$(git rev-parse --is-shallow-repository 2>/dev/null || echo unknown)" != "false" ]; then
|
||||
# In a shallow clone an absent commit is the normal case, so `--is-ancestor`
|
||||
# answers about the graph that was fetched rather than the one that exists.
|
||||
MERGED_REASON="the checkout is shallow, so a commit missing from it says nothing about ancestry"
|
||||
else
|
||||
for ref in $PROTECTED_REFS; do
|
||||
tip="${remote_tip_of[$ref]:-}"
|
||||
[ -n "$tip" ] || continue
|
||||
git cat-file -e "${tip}^{commit}" 2>/dev/null || continue
|
||||
PROTECTED_TIPS+=("${ref}:${tip}")
|
||||
done
|
||||
if [ "${#PROTECTED_TIPS[@]}" -gt 0 ]; then
|
||||
MERGED_AVAILABLE=1
|
||||
echo "liveness: merged-branch detection anchored on ${PROTECTED_TIPS[*]%%:*}"
|
||||
else
|
||||
MERGED_REASON="none of the protected branch tips (${PROTECTED_REFS:-none configured}) is present in this checkout"
|
||||
fi
|
||||
fi
|
||||
[ "$MERGED_AVAILABLE" = "1" ] || \
|
||||
echo "::warning::liveness: ${MERGED_REASON} — merged-but-undeleted branches keep their caches this run"
|
||||
fi
|
||||
|
||||
# is_merged_dead <dir-name>
|
||||
#
|
||||
# True when the branch this directory belongs to is still on origin but every
|
||||
# commit it holds is already on a protected branch — a merged PR whose branch
|
||||
# the forge did not delete, which is the case zemyna hits on every merge and
|
||||
# which the deleted-branch signal above can never see.
|
||||
#
|
||||
# Memoised per tip because target-<key> and snapshot-<key> share one branch,
|
||||
# and because the "cannot tell" warning belongs to the branch rather than to
|
||||
# each of its directories.
|
||||
is_merged_dead() {
|
||||
local name="$1" tip="${live_tip[$1]:-}" entry ref psha
|
||||
MERGED_INTO=""
|
||||
[ "$MERGED_AVAILABLE" = "1" ] || return 1
|
||||
[ -n "$tip" ] || return 1
|
||||
case "${merged_verdict[$tip]:-}" in
|
||||
dead) MERGED_INTO="${merged_into[$tip]}"; return 0 ;;
|
||||
live|unknown) return 1 ;;
|
||||
esac
|
||||
if ! git cat-file -e "${tip}^{commit}" 2>/dev/null; then
|
||||
merged_verdict["$tip"]=unknown
|
||||
echo "::warning::prune: ${name}: its branch tip ${tip} is not in this checkout — cannot tell merged from live, keeping it"
|
||||
return 1
|
||||
fi
|
||||
for entry in "${PROTECTED_TIPS[@]}"; do
|
||||
ref="${entry%%:*}"; psha="${entry#*:}"
|
||||
if git merge-base --is-ancestor "$tip" "$psha" 2>/dev/null; then
|
||||
merged_verdict["$tip"]=dead
|
||||
merged_into["$tip"]="$ref"
|
||||
MERGED_INTO="$ref"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
merged_verdict["$tip"]=live
|
||||
return 1
|
||||
}
|
||||
|
||||
if [ "$LIVENESS_AVAILABLE" = "1" ]; then
|
||||
pruned_any=0
|
||||
# Tracked separately so the line below cannot contradict the decline lines
|
||||
@@ -309,12 +477,20 @@ if [ "$LIVENESS_AVAILABLE" = "1" ]; then
|
||||
[ -d "$dir" ] || continue
|
||||
name=$(basename "$dir")
|
||||
is_protected "$dir" && continue
|
||||
[ -n "${live_ns[$name]:-}" ] && continue
|
||||
if [ -z "${live_ns[$name]:-}" ]; then
|
||||
why="no matching branch on origin"
|
||||
why_summary="branch no longer exists on origin"
|
||||
elif is_merged_dead "$name"; then
|
||||
why="merged into ${MERGED_INTO}, whose tip already contains its every commit"
|
||||
why_summary="merged into \`${MERGED_INTO}\`"
|
||||
else
|
||||
continue
|
||||
fi
|
||||
is_locked "$dir" && { spared_any=1; continue; }
|
||||
dir_gb=$(usage_gb "$dir")
|
||||
evict_dir "$dir" || { spared_any=1; continue; }
|
||||
echo "::warning::pruned dead-branch cache ${name} (${dir_gb} GB) — no matching branch on origin"
|
||||
summary_line "- pruned dead-branch cache \`${name}\` (${dir_gb} GB) — branch no longer exists on origin"
|
||||
echo "::warning::pruned dead-branch cache ${name} (${dir_gb} GB) — ${why}"
|
||||
summary_line "- pruned dead-branch cache \`${name}\` (${dir_gb} GB) — ${why_summary}"
|
||||
pruned_any=1
|
||||
done
|
||||
if [ "$pruned_any" = "0" ]; then
|
||||
@@ -329,16 +505,45 @@ else
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "=== pass 2/3: disk pressure (threshold: free < ${MIN_FREE_PCT}%) ==="
|
||||
echo "=== pass 2/3: disk pressure ==="
|
||||
read -r TOTAL_KB FREE_KB <<< "$(read_df "$ROOT")"
|
||||
THRESHOLD_KB=$(( TOTAL_KB * MIN_FREE_PCT / 100 ))
|
||||
PCT_KB=$(( TOTAL_KB * MIN_FREE_PCT / 100 ))
|
||||
|
||||
if [ "$FREE_KB" -ge "$THRESHOLD_KB" ]; then
|
||||
echo "cache: $(basename "$OWN_DIR") $(usage_gb "$OWN_DIR") GB | $(report_df host "$FREE_KB" "$TOTAL_KB")"
|
||||
# The two floors, and which of them governs. They are kept apart all the way
|
||||
# down rather than collapsed here, because falling short of them means
|
||||
# different things: the derived one predicts that the very next step cannot
|
||||
# finish, and the percentage one is a hygiene target for the volume.
|
||||
REQUIRED_KB="$CLONE_KB"
|
||||
GOVERNS="the clone this run is about to make"
|
||||
if [ "$PCT_KB" -gt "$REQUIRED_KB" ]; then
|
||||
REQUIRED_KB="$PCT_KB"
|
||||
GOVERNS="the ${MIN_FREE_PCT}% floor"
|
||||
fi
|
||||
echo "required: $(gb "$REQUIRED_KB") GB free — ${GOVERNS} (clone $(gb "$CLONE_KB") GB, floor $(gb "$PCT_KB") GB)"
|
||||
|
||||
own_report() {
|
||||
if [ -d "$OWN_DIR" ]; then
|
||||
echo "cache: $(basename "$OWN_DIR") $(usage_gb "$OWN_DIR") GB | $(report_df host "$1" "$2")"
|
||||
else
|
||||
# Ordinary now that this pass runs ahead of the seed: on a branch's first
|
||||
# run of the day the directory does not exist yet, and reporting 0.0 GB
|
||||
# for it would read as an emptied cache.
|
||||
echo "cache: $(basename "$OWN_DIR") not seeded yet | $(report_df host "$1" "$2")"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$FREE_KB" -ge "$REQUIRED_KB" ]; then
|
||||
own_report "$FREE_KB" "$TOTAL_KB"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "::warning::$(report_df disk "$FREE_KB" "$TOTAL_KB") < ${MIN_FREE_PCT}% threshold"
|
||||
echo "::warning::$(report_df disk "$FREE_KB" "$TOTAL_KB") < the $(gb "$REQUIRED_KB") GB this run requires"
|
||||
|
||||
# What survived the pass, and why, in the order the pass considered them. Read
|
||||
# only by the failure report at the bottom: a run that cannot fit its clone is
|
||||
# a run whose log has to answer "then what is all that space?" without anyone
|
||||
# having to reconstruct the pass by hand.
|
||||
KEPT=()
|
||||
|
||||
# A plain loop over a pre-materialised, pre-sorted list rather than a live
|
||||
# `find | while` pipeline, so `rm -rf` inside the loop cannot make a running
|
||||
@@ -346,21 +551,56 @@ echo "::warning::$(report_df disk "$FREE_KB" "$TOTAL_KB") < ${MIN_FREE_PCT}% thr
|
||||
mapfile -t LRU < <(list_by_lru)
|
||||
for dir in "${LRU[@]}"; do
|
||||
[ -d "$dir" ] || continue
|
||||
is_protected "$dir" && continue
|
||||
if reason=$(protected_reason "$dir"); then
|
||||
KEPT+=("$(basename "$dir") — ${reason}")
|
||||
continue
|
||||
fi
|
||||
read -r TOTAL_KB FREE_KB <<< "$(read_df "$ROOT")"
|
||||
[ "$FREE_KB" -ge "$THRESHOLD_KB" ] && break
|
||||
is_locked "$dir" && continue
|
||||
[ "$FREE_KB" -ge "$REQUIRED_KB" ] && break
|
||||
if is_locked "$dir"; then
|
||||
KEPT+=("$(basename "$dir") — held open by a running job")
|
||||
continue
|
||||
fi
|
||||
dir_gb=$(usage_gb "$dir")
|
||||
evict_dir "$dir" || continue
|
||||
if ! evict_dir "$dir"; then
|
||||
KEPT+=("$(basename "$dir") — claimed by a job while its eviction was in flight")
|
||||
continue
|
||||
fi
|
||||
echo "::warning::evicted $(basename "$dir") (${dir_gb} GB, LRU under disk pressure)"
|
||||
summary_line "- evicted \`$(basename "$dir")\` (${dir_gb} GB, LRU under disk pressure)"
|
||||
done
|
||||
|
||||
read -r TOTAL_KB FREE_KB <<< "$(read_df "$ROOT")"
|
||||
if [ "$FREE_KB" -lt "$THRESHOLD_KB" ]; then
|
||||
|
||||
# Falling short of the DERIVED requirement is a failure, not a warning. The
|
||||
# seed step is next, it will clone that source, and it will run out of disk
|
||||
# part-way through unsharing the clone's mutable paths — reported against a
|
||||
# staging path, with nothing in the message about which cache was holding the
|
||||
# space. Failing here says that instead.
|
||||
#
|
||||
# There is nothing to self-clear on this path and it is not skipped in error:
|
||||
# a non-zero requirement means the seed is about to CLONE, which means this
|
||||
# run has no own target dir to wipe (seed_clone_source returns nothing when it
|
||||
# does), so pass 3 has no candidate. See the header.
|
||||
if [ "$FREE_KB" -lt "$CLONE_KB" ]; then
|
||||
echo "::error::prune: $(gb "$FREE_KB") GB free after evicting every eligible cache, but seeding from $(basename "$SEED_SRC") needs $(gb "$CLONE_KB") GB — short by $(gb "$(( CLONE_KB - FREE_KB ))") GB"
|
||||
summary_line "- **out of disk**: seeding from \`$(basename "$SEED_SRC")\` needs $(gb "$CLONE_KB") GB, $(gb "$FREE_KB") GB free"
|
||||
echo "prune: kept, and why:"
|
||||
for entry in ${KEPT[@]+"${KEPT[@]}"}; do echo " ${entry}"; done
|
||||
[ "${#KEPT[@]}" -gt 0 ] || echo " (nothing — the volume holds no cache directories at all)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$FREE_KB" -lt "$PCT_KB" ]; then
|
||||
OWN_GB=$(usage_gb "$OWN_DIR")
|
||||
echo "::warning::still under threshold after evicting every eligible sibling; clearing own $(basename "$OWN_DIR") (was ${OWN_GB} GB) — this run pays a cold rebuild"
|
||||
summary_line "- **self-clear**: \`$(basename "$OWN_DIR")\` (was ${OWN_GB} GB) wiped — this run pays a cold rebuild"
|
||||
# Recreated empty rather than left absent, and that is what keeps this path
|
||||
# out of the requirement above: the seed reuses an own target dir that
|
||||
# exists, whatever is in it, so a self-cleared run clones nothing and needs
|
||||
# no headroom. Leaving it absent would send that run to the base snapshot
|
||||
# instead, needing a clone this pass has just spent its last eligible bytes
|
||||
# not sizing for.
|
||||
rm -rf "$OWN_DIR"
|
||||
mkdir -p "$OWN_DIR"
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user