fix(prune-cache): close the reader-marker check-then-delete window #2
@@ -43,7 +43,9 @@
|
|||||||
# 14. AND NOT WHILE ANOTHER PASS MAY STILL BE EVICTING IT. An aside with no
|
# 14. AND NOT WHILE ANOTHER PASS MAY STILL BE EVICTING IT. An aside with no
|
||||||
# readers is indistinguishable from one a concurrent pass has just
|
# readers is indistinguishable from one a concurrent pass has just
|
||||||
# renamed and not yet decided about; reclaiming that one lets `rm -rf`
|
# renamed and not yet decided about; reclaiming that one lets `rm -rf`
|
||||||
# empty a tree its owner may still restore under a live cache name.
|
# empty a tree its owner may still restore under a live cache name. Its
|
||||||
|
# fixture is an OLD directory renamed a moment ago — production's shape,
|
||||||
|
# and the only shape that can tell the two timestamps apart.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||||
. "$script_dir/cache-lib.sh"
|
. "$script_dir/cache-lib.sh"
|
||||||
@@ -264,13 +266,21 @@ echo
|
|||||||
echo "=== 14: an aside another pass may still be evicting is left alone ==="
|
echo "=== 14: an aside another pass may still be evicting is left alone ==="
|
||||||
reset_cache
|
reset_cache
|
||||||
aside="$root/.evicting-target-$DEAD-9999"
|
aside="$root/.evicting-target-$DEAD-9999"
|
||||||
mkdir -p "$aside"; head -c 4096 /dev/zero > "$aside/blob"
|
# Built the way the pass builds one — an old cache directory renamed a moment
|
||||||
|
# ago — because the two ages that describe it disagree, and which of them the
|
||||||
|
# window reads is the whole mechanism. `mkdir`-ing the aside directly would
|
||||||
|
# give it a fresh mtime as well as a fresh ctime, and a fixture whose two
|
||||||
|
# clocks agree cannot tell %Z from %Y: the settle window would read the wrong
|
||||||
|
# one, never fire for any real aside, and this scenario would not notice.
|
||||||
|
victim="$root/target-$DEAD-victim"
|
||||||
|
mkdir -p "$victim"; head -c 4096 /dev/zero > "$victim/blob"
|
||||||
|
touch -d '2020-01-01' "$victim" # an old cache, which is every cache
|
||||||
|
mv -T "$victim" "$aside" # set aside a moment ago
|
||||||
# Deliberately no reader marker: the reader gate would pass this straight
|
# Deliberately no reader marker: the reader gate would pass this straight
|
||||||
# through, which is the whole point. An aside with no readers is exactly what
|
# through, which is the whole point. An aside with no readers is exactly what
|
||||||
# a pass that has just renamed one aside and not yet decided about it looks
|
# a pass that has just renamed one aside and not yet decided about it looks
|
||||||
# like, and `rm -rf` traverses fd-relative — so reclaiming it out from under
|
# like, and `rm -rf` traverses fd-relative — so reclaiming it out from under
|
||||||
# that pass lets it republish a half-emptied tree under a live cache name.
|
# that pass lets it republish a half-emptied tree under a live cache name.
|
||||||
# Nothing here fakes an age: the directory really was set aside a moment ago.
|
|
||||||
run_prune "1000000 900000"
|
run_prune "1000000 900000"
|
||||||
assert_kept "$aside" "an aside younger than the settle window is not reclaimed"
|
assert_kept "$aside" "an aside younger than the settle window is not reclaimed"
|
||||||
assert_kept "$aside/blob" "and is left intact, not part-way emptied"
|
assert_kept "$aside/blob" "and is left intact, not part-way emptied"
|
||||||
|
|||||||
@@ -89,9 +89,10 @@ STALE_LOCK_SECONDS="${STALE_LOCK_SECONDS:-7200}"
|
|||||||
# An aside directory is in flight for one rename plus one marker glob —
|
# An aside directory is in flight for one rename plus one marker glob —
|
||||||
# milliseconds. Anything older belongs to a pass that died between the two, so
|
# milliseconds. Anything older belongs to a pass that died between the two, so
|
||||||
# an age is what separates "another pass is mid-eviction" from "a leftover",
|
# an age is what separates "another pass is mid-eviction" from "a leftover",
|
||||||
# and it separates them without having to identify the pass that created it: a
|
# and it separates them without having to identify the pass that created it.
|
||||||
# PID is meaningless across the job containers these passes run in, and
|
# The `$$` in an aside's name was that pass's PID inside its own job
|
||||||
# recycles. Three orders of magnitude of headroom over the operation it covers,
|
# container, so testing it with `kill -0` from a different one is not
|
||||||
|
# unreliable, it is meaningless — and PIDs recycle besides. Three orders of magnitude of headroom over the operation it covers,
|
||||||
# and short enough that a genuine leftover is reclaimed by the next run rather
|
# and short enough that a genuine leftover is reclaimed by the next run rather
|
||||||
# than lingering while the volume is under pressure.
|
# than lingering while the volume is under pressure.
|
||||||
EVICTION_ASIDE_SETTLE_SECONDS="${EVICTION_ASIDE_SETTLE_SECONDS:-60}"
|
EVICTION_ASIDE_SETTLE_SECONDS="${EVICTION_ASIDE_SETTLE_SECONDS:-60}"
|
||||||
|
|||||||
Reference in New Issue
Block a user