diff --git a/scripts/seed-target-dir-selftest.sh b/scripts/seed-target-dir-selftest.sh index 452f55e..5997aec 100755 --- a/scripts/seed-target-dir-selftest.sh +++ b/scripts/seed-target-dir-selftest.sh @@ -29,14 +29,24 @@ # at no point is a partially-populated directory visible under the final # name. This is the property that replaces "the runner only has one job # slot" with an actual guarantee. -# 8. SEED VS PUBLISH ROTATION — the race scenario 7 does NOT cover, and the +# 8a. SEED VS PUBLISH ROTATION — the race scenario 7 does NOT cover, and the # one that actually mattered: a consumer hardlink-cloning a snapshot -# while the publisher of that snapshot rotates it and unlinks the -# generation being read. Two seeds racing on a DESTINATION is a different -# race from a seed racing a publisher on its SOURCE, and only the second -# one can truncate a tree. Against the unguarded version this scenario -# reproduces a silent partial clone reported as success — 20,328 of -# 48,805 entries, `seed: cloned in 1s`, exit 0, seeded-from=base-snapshot. +# while the publisher of that snapshot rotates it. Two seeds racing on a +# DESTINATION is a different race from a seed racing a publisher on its +# SOURCE, and only the second one can truncate a tree. What it pins is +# that the consumer notices its source was REPLACED and re-clones, +# ending up with the whole generation now published — not that the +# rotation interfered with the copy, which is not guaranteed and is what +# made the racing version of this scenario flaky (issue #3). +# 8b. AND A SILENTLY TRUNCATED WALK — the same tear seen through the other +# check: a subtree unlinked out of the parent's listing before `cp -al` +# reads it is never visited, the copy exits 0 and the source's identity +# never changes. Against the unguarded version this is a partial tree +# renamed into place and reported as a success — 20,328 of 48,805 +# entries, `seed: cloned in 1s`, exit 0, seeded-from=base-snapshot. +# Both scenarios force their interleaving rather than racing for it, and +# each asserts WHICH check caught the tear, so neither stays green if +# the check it exercises is removed. # 9. AN UNREADABLE SOURCE FAILS LOUDLY — the clone reports a distinct status # instead of renaming whatever it managed to produce into place, and the # seed SCRIPT turns that status into a failed job rather than a silent @@ -75,9 +85,13 @@ make_tree() { seed() { bash "$script_dir/seed-target-dir.sh" "$@" > "$scratch/log" 2>&1 || { tail -40 "$scratch/log"; fail "seed-target-dir.sh exited non-zero"; }; } -# Always succeeds and always prints a number: an absent directory is "0 -# entries so far", which is the normal state at the top of the progress poll -# below, not an error worth aborting the suite over. +# The same script, with the scenario-8 stub directory ahead of the real +# coreutils on PATH. Kept separate so no other scenario can pick a stub up by +# accident, and so the caller keeps the exit status instead of aborting on it. +seed_with_stub() { PATH="$scratch/bin:$PATH" bash "$script_dir/seed-target-dir.sh" "$@"; } + +# Always succeeds and always prints a number: a directory that does not exist +# is 0 entries, not an error worth aborting the suite over. tree_entries() { local n n=$(find "$1" -mindepth 1 2>/dev/null | wc -l) || n=0 @@ -85,12 +99,56 @@ tree_entries() { return 0 } -# A tree wide enough that a hardlink clone of it takes long enough to be -# caught mid-walk. The race under test is a real interleaving, not a mocked -# one, so the fixture has to be big enough for the window to exist: a -# four-file tree clones in microseconds and no scheduling could ever land -# inside it. Built by cloning one small template directory N times, which is N -# forks rather than N*M file creations. +# Waits for a file a concurrently running script will create. The publisher in +# scenario 8a is started from inside the consumer's process tree rather than +# by this script, so its completion cannot be waited on as a job. +wait_for_file() { + local path="$1" what="$2" deadline + deadline=$(( $(date +%s) + 60 )) + until [ -e "$path" ]; do + [ "$(date +%s)" -lt "$deadline" ] || fail "$what" + sleep 0.1 + done +} + +# Reads the clone's own torn-read report back and asserts WHICH of its three +# checks fired: the copy's exit status, the entry count, or the source's +# identity. Scenarios 8a and 8b each force exactly one of the three, and a +# scenario that only asserted "some tear was reported" would stay green if +# the check it exercises were deleted and one of the others happened to fire +# in its place. +# +# assert_tear +assert_tear() { + local log="$1" want_rc="$2" want_count="$3" want_inode="$4" msg="$5" line + local re='cp rc=([0-9]+), ([0-9]+)/([0-9]+) entries, source inode ([0-9]+) -> ([0-9]+)' + # `|| line=""` rather than a bare assignment: no match makes grep exit 1, + # which under `set -e` would abort the suite with no message at all — the + # exact case this assertion exists to report. + line=$(grep -o 'was torn ([^)]*)' "$log" | head -1) || line="" + [ -n "$line" ] || { tail -20 "$log"; fail "no torn read was reported at all ($msg)"; } + [[ $line =~ $re ]] || fail "unrecognised torn-read report: ${line}" + [ "${BASH_REMATCH[1]}" = "$want_rc" ] || fail "expected cp to exit ${want_rc}: ${line}" + case "$want_count" in + same) [ "${BASH_REMATCH[2]}" -eq "${BASH_REMATCH[3]}" ] || fail "expected a whole staging tree: ${line}" ;; + short) [ "${BASH_REMATCH[2]}" -lt "${BASH_REMATCH[3]}" ] || fail "expected a short staging tree: ${line}" ;; + *) fail "assert_tear: bad entry-count expectation '${want_count}'" ;; + esac + case "$want_inode" in + same) [ "${BASH_REMATCH[4]}" = "${BASH_REMATCH[5]}" ] || fail "expected the source's identity to hold: ${line}" ;; + differs) [ "${BASH_REMATCH[4]}" != "${BASH_REMATCH[5]}" ] || fail "expected the source's identity to change: ${line}" ;; + *) fail "assert_tear: bad identity expectation '${want_inode}'" ;; + esac + ok "$msg (${line})" +} + +# A tree of many sibling subtrees, built by cloning one small template +# directory N times — N forks rather than N*M file creations. The shape is +# what scenarios 8a and 8b need: two generations of DIFFERENT entry count, so +# an assertion can tell which one a consumer ended up with, and enough +# sibling subtrees that removing one moves the count. Size no longer has to +# buy a race window — both scenarios force their interleaving — so it stays +# small enough to be free. make_wide_tree() { local d="$1" marker="$2" ndirs="$3" i mkdir -p "$d/debug/deps/.tmpl" "$d/debug/.fingerprint/x" @@ -170,62 +228,149 @@ fi ok "neither racing job fell through to a cold start" echo -echo "=== 8: seeding while the base republishes the snapshot underneath it ===" +echo "=== 8a: a rotation landing inside the clone's identity window ===" +# The race scenario 7 does NOT cover: a seed racing a publisher on its SOURCE +# rather than two seeds racing on a DESTINATION. What this pins is that a +# consumer whose source is REPLACED WHOLESALE mid-clone notices the +# substitution and re-clones, ending up holding the whole generation now +# published — NOT that the rotation necessarily interfered with the copy, +# which is not guaranteed and is what made the racing version of this +# scenario flaky (issue #3). +# +# The interleaving is forced rather than hoped for, the way +# prune-cache-selftest.sh's scenario 12 forces a marker into the +# check-to-unlink window: the consumer's own `cp` performs the rotation, so +# it lands strictly after the entry count and inode that open the identity +# window and strictly before the inode that closes it. The publisher is the +# real publish-snapshot.sh running concurrently; only WHEN it runs is +# arranged. ROT=$(cache_key feat/rotate) rm -rf "$root/snapshot-$BASE_KEY" "$root/target-$BASE_KEY" -# Generation 1 is wide (the consumer will still be walking it when the swap -# happens); the generation replacing it is small, so the publisher's own -# staging clone does not itself outlast the consumer's. -make_wide_tree "$root/snapshot-$BASE_KEY" gen1 800 -make_wide_tree "$root/target-$BASE_KEY" gen2 8 +# The two generations must differ in entry count, or "the seeded tree matches +# the published snapshot" would hold for whichever one the consumer ended up +# with and the fixture could not distinguish the outcomes at all. +make_wide_tree "$root/snapshot-$BASE_KEY" gen1 12 +make_wide_tree "$root/target-$BASE_KEY" gen2 3 gen1_entries=$(tree_entries "$root/snapshot-$BASE_KEY") +gen1_inode=$(stat -c '%i' "$root/snapshot-$BASE_KEY") +[ "$gen1_entries" -ne "$(tree_entries "$root/target-$BASE_KEY")" ] \ + || fail "both generations have ${gen1_entries} entries — this fixture cannot tell them apart" -( bash "$script_dir/seed-target-dir.sh" "$ROT" "$BASE_KEY" "$root" jobRot > "$scratch/logRot" 2>&1; echo $? > "$scratch/rcRot" ) & -seed_pid=$! +mkdir -p "$scratch/bin" +real_cp=$(command -v cp) +cat > "$scratch/bin/cp" < "$scratch/rotated" + rc=0; "$real_cp" "\$@" || rc=\$? + # Concurrently: the publisher's post-swap drain wait is against THIS + # consumer's marker, which is held until the identity read that follows + # this cp returns, so running the publish inline would deadlock the two + # sides against each other. + ( bash "$script_dir/publish-snapshot.sh" "$BASE_KEY" "$root" pubRot > "$scratch/logPub" 2>&1 + echo \$? > "$scratch/rcPub" ) & + # Hand control back only once the swap is on disk, so the identity read + # immediately after this cp is guaranteed to resolve to the new generation. + deadline=\$(( \$(date +%s) + 60 )) + while [ "\$(stat -c '%i' "$root/snapshot-$BASE_KEY" 2>/dev/null)" = "$gen1_inode" ]; do + [ "\$(date +%s)" -lt "\$deadline" ] || { echo "stub cp: the publisher never swapped the snapshot" >&2; exit 90; } + sleep 0.05 + done + exit \$rc +fi +exec "$real_cp" "\$@" +EOF +chmod +x "$scratch/bin/cp" -# Rotate only once the clone is demonstrably mid-walk. Gating on observed -# progress rather than on a sleep is what makes the interleaving reproducible -# instead of a coin flip that passes on a fast machine for the wrong reason. -threshold=$(( gen1_entries / 5 )) -progress=0 -deadline=$(( $(date +%s) + 60 )) -while :; do - progress=$(tree_entries "$root/.stage-jobRot") - if [ "$progress" -ge "$threshold" ]; then break; fi - if ! kill -0 "$seed_pid" 2>/dev/null; then - fail "the seed finished before its clone could be caught mid-walk (fixture too small for this machine?)" - fi - if [ "$(date +%s)" -ge "$deadline" ]; then - fail "the staging clone never reached ${threshold} of ${gen1_entries} entries" - fi -done -ok "caught the consumer's clone mid-walk at ${progress}/${gen1_entries} entries" +rcRot=0 +seed_with_stub "$ROT" "$BASE_KEY" "$root" jobRot > "$scratch/logRot" 2>&1 || rcRot=$? +[ -e "$scratch/rotated" ] \ + || fail "the stubbed cp never fired: no rotation was placed in the window, so this scenario proves nothing" +ok "the rotation was placed inside the consumer's identity window" +wait_for_file "$scratch/rcPub" "the publisher never finished" +[ "$(cat "$scratch/rcPub")" = "0" ] || { tail -40 "$scratch/logPub"; fail "publish-snapshot.sh exited non-zero"; } +[ "$rcRot" = "0" ] || { tail -40 "$scratch/logRot"; fail "the seed exited non-zero"; } +ok "the seed completed" -bash "$script_dir/publish-snapshot.sh" "$BASE_KEY" "$root" pubRot > "$scratch/logPub" 2>&1 \ - || { tail -40 "$scratch/logPub"; fail "publish-snapshot.sh exited non-zero"; } -wait "$seed_pid" +# WHICH check caught the rotation is the point of this scenario, so it is +# asserted rather than assumed. The copy succeeded and the staging tree holds +# every entry the source had when the walk began, so neither cp's exit status +# nor the entry count is a witness here — the source's identity changing +# under the walk is the only one. Reading the report back is what keeps this +# scenario sensitive to losing that single comparison. +assert_tear "$scratch/logRot" 0 same differs "the substitution was caught by the source's identity alone" rot_dir="$root/target-$ROT" -[ "$(cat "$scratch/rcRot")" = "0" ] || { tail -40 "$scratch/logRot"; fail "the seed exited non-zero"; } -ok "the seed completed" -# THE assertion. Before the guard, this is where it failed: the seed reported -# `cloned in 1s` and exit 0 while target- held less than half the entries -# of the snapshot it claimed to have cloned. Comparing against the snapshot as -# it stands NOW is the right bar either way — a clone that raced the rotation -# must end up holding one complete generation, and a consumer caught mid-walk -# re-reads, so that generation is the new one. snap_entries=$(tree_entries "$root/snapshot-$BASE_KEY") rot_entries=$(tree_entries "$rot_dir") [ "$rot_entries" -eq "$snap_entries" ] \ - || fail "the seeded tree is truncated: ${rot_entries} entries against the snapshot's ${snap_entries} (was ${gen1_entries} before the rotation)" -ok "the seeded tree is complete (${rot_entries} entries, no silent truncation)" + || fail "the seeded tree is not the generation now published: ${rot_entries} entries against the snapshot's ${snap_entries} (generation 1 had ${gen1_entries})" +ok "the seed re-cloned and holds the whole published generation (${rot_entries} entries)" assert_content "$rot_dir/debug/.fingerprint/x/dep-lib-x" gen2 "the seeded tree holds one whole generation, not a splice of two" -grep -q 'was torn' "$scratch/logRot" || fail "the rotation was not detected as a torn read" -ok "the torn read was detected and reported, not swallowed" -[ -z "$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' -o -name '.publish-*' \) -print -quit)" ] \ - || fail "scratch left behind: $(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' -o -name '.publish-*' \) -print)" +leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' -o -name '.publish-*' \) -print) +[ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}" ok "no staging, reader-marker or deferred-generation scratch left behind" +rm -f "$scratch/bin/cp" +echo +echo "=== 8b: a subtree unlinked out from under the walk, silently ===" +# The other way a clone tears, and the one with nothing to report: a subtree +# that leaves the parent's listing before `cp -al` reads it is simply never +# visited. The copy exits 0 and the source's identity never changes, so the +# entry count taken before the walk is the only witness there is — this is +# the mode that used to publish a partial tree and call it a success. +# +# Modelled by renaming the subtree out and back around the consumer's own cp: +# out before the walk starts (the only way to be missed without an error), +# back before the retry, because the real thing that removes entries — a +# publish rotating a generation away — has a whole generation at the path by +# the time the retry looks. +TRUNC=$(cache_key feat/truncate) +TRUNC_BASE=$(cache_key release/1) +make_wide_tree "$root/snapshot-$TRUNC_BASE" trunkgen 6 +victim="$root/snapshot-$TRUNC_BASE/debug/deps/d3" +trunc_entries=$(tree_entries "$root/snapshot-$TRUNC_BASE") +[ "$(tree_entries "$victim")" -gt 0 ] \ + || fail "the subtree this scenario removes is empty — its removal would not change the entry count" + +cat > "$scratch/bin/cp" < "$scratch/unlinked" + mv "$victim" "$scratch/held" + rc=0; "$real_cp" "\$@" || rc=\$? + mv "$scratch/held" "$victim" + exit \$rc +fi +exec "$real_cp" "\$@" +EOF +chmod +x "$scratch/bin/cp" + +rcTrunc=0 +seed_with_stub "$TRUNC" "$TRUNC_BASE" "$root" jobTrunc > "$scratch/logTrunc" 2>&1 || rcTrunc=$? +[ -e "$scratch/unlinked" ] \ + || fail "the stubbed cp never fired: nothing was unlinked mid-walk, so this scenario proves nothing" +ok "a subtree was taken out of the source's listing before the walk read it" +[ "$rcTrunc" = "0" ] || { tail -40 "$scratch/logTrunc"; fail "the seed exited non-zero"; } +ok "the seed completed" +assert_tear "$scratch/logTrunc" 0 short same "the silent truncation was caught by the entry count alone" + +trunc_dir="$root/target-$TRUNC" +[ "$(tree_entries "$trunc_dir")" -eq "$trunc_entries" ] \ + || fail "the seeded tree is short: $(tree_entries "$trunc_dir") entries against the source's ${trunc_entries}" +ok "the seed re-cloned and holds every entry the source has (${trunc_entries})" +[ -e "$victim" ] || fail "the fixture did not put the subtree back" +[ -e "$trunc_dir/debug/deps/d3/f0" ] \ + || fail "the subtree missed by the first walk is absent from the seeded tree" +ok "the subtree the first walk never saw is present in the seeded tree" +leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' \) -print) +[ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}" +ok "no staging or reader-marker scratch left behind" +rm -f "$scratch/bin/cp" echo echo "=== 9: a source that cannot be read fails loudly ===" rc=0