#!/usr/bin/env bash # Regression test for seed-target-dir.sh: which source a run seeds from, and # what happens when two jobs sharing one cache key seed at the same time. # # Runs the ACTUAL script against a real scratch cache directory with fake # target trees standing in for cargo output — no compiler needed, so this is # the fast half of the suite. hardlink-clone-selftest.sh covers the parts that # need a real build. # # What each scenario demonstrates: # # 1. BASE SNAPSHOT PREFERRED — a PR whose base has published a snapshot # seeds from it, and the seeded directory really is a hardlink clone # (shared inodes), not a copy. # 2. OWN DIR WINS — a second run of the same ref reuses what is already # there and does not re-seed over its own work. # 3. OWN SNAPSHOT AS SELF-RESTORE — a publisher whose live target dir was # evicted restores from the snapshot it last published, instead of # rebuilding cold. # 4. FALLBACK DIR — with no snapshot at all, an explicitly configured # fallback (a pre-existing flat cache, during a migration) is used. # 5. COLD — with nothing available, the directory is created empty rather # than the script failing. # 6. LOCK FILES STRIPPED — Cargo's in-place-flock'd lock files never # survive a clone, because a shared lock inode would make two branches # contend on one mutex. # 7. CONCURRENT SEED IS ATOMIC — two seeds racing on one cache key: the # loser discards its staging copy and uses the winner's directory, and # at no point is a partially-populated directory visible under the final # name. This is the property that replaces "the runner only has one job # slot" with an actual guarantee. # 8a. SEED VS PUBLISH ROTATION — the race scenario 7 does NOT cover, and the # one that actually mattered: a consumer hardlink-cloning a snapshot # while the publisher of that snapshot rotates it. Two seeds racing on a # DESTINATION is a different race from a seed racing a publisher on its # SOURCE, and only the second one can truncate a tree. What it pins is # that the consumer notices its source was REPLACED and re-clones, # ending up with the whole generation now published — not that the # rotation interfered with the copy, which is not guaranteed and is what # made the racing version of this scenario flaky (issue #3). # 8b. AND A SILENTLY TRUNCATED WALK — the same tear seen through the other # check: a subtree unlinked out of the parent's listing before `cp -al` # reads it is never visited, the copy exits 0 and the source's identity # never changes. Against the unguarded version this is a partial tree # renamed into place and reported as a success — 20,328 of 48,805 # entries, `seed: cloned in 1s`, exit 0, seeded-from=base-snapshot. # 8c. AND A COPY THAT SAYS SO ITSELF — the third witness, and the only one # the tool volunteers: `cp -al` exiting non-zero over a tree that both # other checks read as whole. Pins that a copy's own failure report is # never overruled by two inferences that saw nothing. # 8d. AND AN IDENTITY THAT COULD NOT BE READ AT ALL — the fourth. A failed # identity read is reported as the string `missing`, so two of them # compare equal to each other; without the term that rejects the # sentinel, a clone whose source could not be identified at either end # is published on the strength of two errors. # All four scenarios force their interleaving rather than racing for it, # and each asserts WHICH check caught the tear, so none stays green if # the check it exercises is removed. # 9. AN UNREADABLE SOURCE FAILS LOUDLY — the clone reports a distinct status # instead of renaming whatever it managed to produce into place, and the # seed SCRIPT turns that status into a failed job rather than a silent # cold build. Retries are what make a torn read survivable; exhausting # them must not degrade into "start cold and rebuild everything", which # would turn a corrupt-cache bug into an invisible 4x-slower CI job. # (The publisher's half of the rotation race — deferring reclamation # while a reader is still in flight — lives in # publish-snapshot-selftest.sh, next to the swap it modifies.) # 10. A TREE THAT CANNOT BE PRIVATELY OWNED IS DISCARDED — the other way a # clone must refuse to publish, and the one that is not about tearing at # all: if unsharing the mutable paths fails, the staging tree still # aliases its source, so renaming it into place would wire two branches # onto one set of fingerprints. That is the silent stale-reuse bug the # whole scheme exists to prevent, so the failure has to abort the clone # rather than be swallowed. set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$script_dir/cache-lib.sh" scratch=$(mktemp -d) trap 'rm -rf "$scratch"' EXIT root="$scratch/cache" mkdir -p "$root" pass_count=0 fail() { echo "ASSERTION FAILED: $*" >&2; exit 1; } ok() { pass_count=$((pass_count + 1)); echo "PASS: $*"; } assert_file() { [ -e "$1" ] || fail "expected $1 to exist ($2)"; ok "$2"; } assert_absent() { [ -e "$1" ] && fail "expected $1 to be gone ($2)"; ok "$2"; } assert_content() { [ "$(cat "$1")" = "$2" ] || fail "expected '$2' in $1, got '$(cat "$1")' ($3)"; ok "$3"; } # A plausible target tree: a big shared artifact, a mutable fingerprint, a # build-script output, and a lock file. make_tree() { local d="$1" marker="$2" mkdir -p "$d/debug/deps" "$d/debug/.fingerprint/x" "$d/debug/build/x/out" echo "$marker" > "$d/debug/deps/libx.rlib" echo "$marker" > "$d/debug/.fingerprint/x/dep-lib-x" echo "$marker" > "$d/debug/build/x/out/gen.txt" : > "$d/debug/.cargo-lock" } seed() { bash "$script_dir/seed-target-dir.sh" "$@" > "$scratch/log" 2>&1 || { tail -40 "$scratch/log"; fail "seed-target-dir.sh exited non-zero"; }; } # The same script, with the stub directory scenarios 8a and 8b write into # ahead of the real coreutils on PATH. Kept separate so no other scenario can # pick a stub up by accident, and so the caller keeps the exit status instead # of aborting on it. Both halves live here rather than in the first scenario # that needs them, so either scenario can be run, moved or mutated alone. mkdir -p "$scratch/bin" real_cp=$(command -v cp) real_stat=$(command -v stat) seed_with_stub() { PATH="$scratch/bin:$PATH" bash "$script_dir/seed-target-dir.sh" "$@"; } # Always succeeds and always prints a number: a directory that does not exist # is 0 entries, not an error worth aborting the suite over. tree_entries() { local n n=$(find "$1" -mindepth 1 2>/dev/null | wc -l) || n=0 printf '%s\n' "$n" return 0 } # Waits for a file a concurrently running script will create. The publisher in # scenario 8a is started from inside the consumer's process tree rather than # by this script, so its completion cannot be waited on as a job. wait_for_file() { local path="$1" what="$2" deadline deadline=$(( $(date +%s) + 60 )) until [ -e "$path" ]; do [ "$(date +%s)" -lt "$deadline" ] || fail "$what" sleep 0.1 done } # Reads the clone's own torn-read report back and asserts WHICH of its four # checks fired: the copy's exit status, the entry count, the source's identity, # or that identity being unreadable at all. Scenarios 8a to 8d each force # exactly one of the four, and a scenario that only asserted "some tear was # reported" would stay green if the check it exercises were deleted and one of # the others happened to fire in its place. # # assert_tear assert_tear() { local log="$1" want_rc="$2" want_count="$3" want_inode="$4" msg="$5" line # The identity fields are matched as `|missing` rather than as # "anything up to the space", so a report whose inode field is neither is a # loud parse failure instead of a silently-compared string. local re='cp rc=([0-9]+), ([0-9]+)/([0-9]+) entries, source inode ([0-9]+|missing) -> ([0-9]+|missing)' # `|| line=""` rather than a bare assignment: no match makes grep exit 1, # which under `set -e` would abort the suite with no message at all — the # exact case this assertion exists to report. line=$(grep -o 'was torn ([^)]*)' "$log" | head -1) || line="" [ -n "$line" ] || { tail -20 "$log"; fail "no torn read was reported at all ($msg)"; } [[ $line =~ $re ]] || fail "unrecognised torn-read report: ${line}" [ "${BASH_REMATCH[1]}" = "$want_rc" ] || fail "expected cp to exit ${want_rc}: ${line}" case "$want_count" in same) [ "${BASH_REMATCH[2]}" -eq "${BASH_REMATCH[3]}" ] || fail "expected a whole staging tree: ${line}" ;; short) [ "${BASH_REMATCH[2]}" -lt "${BASH_REMATCH[3]}" ] || fail "expected a short staging tree: ${line}" ;; *) fail "assert_tear: bad entry-count expectation '${want_count}'" ;; esac case "$want_inode" in # `same` demands a READ identity that held, not merely two equal strings: # two failed reads are both `missing` and would otherwise satisfy it, which # is the exact confusion scenario 8d exists to pin. same) [ "${BASH_REMATCH[4]}" != missing ] || fail "expected the source's identity to be readable: ${line}" [ "${BASH_REMATCH[4]}" = "${BASH_REMATCH[5]}" ] || fail "expected the source's identity to hold: ${line}" ;; differs) [ "${BASH_REMATCH[4]}" != "${BASH_REMATCH[5]}" ] || fail "expected the source's identity to change: ${line}" ;; unreadable) [ "${BASH_REMATCH[4]}" = missing ] && [ "${BASH_REMATCH[5]}" = missing ] \ || fail "expected neither identity read to have resolved: ${line}" ;; *) fail "assert_tear: bad identity expectation '${want_inode}'" ;; esac ok "$msg (${line})" } # A tree of many sibling subtrees, built by cloning one small template # directory N times — N forks rather than N*M file creations. The shape is # what scenarios 8a and 8b need: two generations of DIFFERENT entry count, so # an assertion can tell which one a consumer ended up with, and enough # sibling subtrees that removing one moves the count. Size no longer has to # buy a race window — both scenarios force their interleaving — so it stays # small enough to be free. make_wide_tree() { local d="$1" marker="$2" ndirs="$3" i mkdir -p "$d/debug/deps/.tmpl" "$d/debug/.fingerprint/x" for i in $(seq 0 59); do echo "$marker" > "$d/debug/deps/.tmpl/f$i"; done for i in $(seq -w 1 "$ndirs"); do cp -al "$d/debug/deps/.tmpl" "$d/debug/deps/d$i"; done rm -rf "$d/debug/deps/.tmpl" echo "$marker" > "$d/debug/.fingerprint/x/dep-lib-x" } BASE_KEY=$(cache_key dev) OWN_KEY=$(cache_key feat/thing) echo "=== 1: base snapshot preferred, and cloned by hardlink ===" make_tree "$root/snapshot-$BASE_KEY" base-content seed "$OWN_KEY" "$BASE_KEY" "$root" job1 own="$root/target-$OWN_KEY" assert_content "$own/debug/deps/libx.rlib" base-content "seeded from the base snapshot" [ "$(stat -c '%i' "$own/debug/deps/libx.rlib")" = "$(stat -c '%i' "$root/snapshot-$BASE_KEY/debug/deps/libx.rlib")" ] \ || fail "artifact was copied, not hardlinked" ok "artifact shares an inode with the snapshot (hardlink clone, not a copy)" [ "$(stat -c '%i' "$own/debug/.fingerprint/x/dep-lib-x")" != "$(stat -c '%i' "$root/snapshot-$BASE_KEY/debug/.fingerprint/x/dep-lib-x")" ] \ || fail "fingerprint still shares an inode with the snapshot" ok "fingerprint is privately owned (unshare_mutable_paths ran)" echo echo "=== 6: Cargo lock files never survive a clone ===" assert_absent "$own/debug/.cargo-lock" "cloned .cargo-lock removed" echo echo "=== 2: an existing own dir is reused, never re-seeded over ===" echo own-work > "$own/debug/deps/libx.rlib" seed "$OWN_KEY" "$BASE_KEY" "$root" job1 assert_content "$own/debug/deps/libx.rlib" own-work "own directory reused as-is" grep -q 'reusing this ref' "$scratch/log" || fail "expected the reuse path in the log" ok "reuse is reported in the log" echo echo "=== 3: a publisher restores from its own snapshot after eviction ===" make_tree "$root/snapshot-$BASE_KEY" published-dev seed "$BASE_KEY" "" "$root" job1 assert_content "$root/target-$BASE_KEY/debug/deps/libx.rlib" published-dev "publisher self-restored from its own snapshot" echo echo "=== 4: explicit fallback dir when no snapshot exists ===" OTHER=$(cache_key feat/other) make_tree "$scratch/legacy-flat" legacy seed "$OTHER" "$(cache_key nosuch)" "$root" job1 "$scratch/legacy-flat" assert_content "$root/target-$OTHER/debug/deps/libx.rlib" legacy "seeded from the fallback dir" echo echo "=== 5: cold start when nothing is available ===" COLD=$(cache_key feat/cold) seed "$COLD" "$(cache_key nosuch)" "$root" job1 [ -d "$root/target-$COLD" ] || fail "cold start did not create the directory" [ -z "$(ls -A "$root/target-$COLD")" ] || fail "cold start directory is not empty" ok "cold start creates an empty directory rather than failing" echo echo "=== 7: two jobs racing on one cache key ===" RACE=$(cache_key feat/race) make_tree "$root/snapshot-$BASE_KEY" race-source # Both jobs seed concurrently from the same snapshot into the same key. Each # stages under its own tag, so the only interaction is the final rename. ( bash "$script_dir/seed-target-dir.sh" "$RACE" "$BASE_KEY" "$root" jobA > "$scratch/logA" 2>&1 ) & ( bash "$script_dir/seed-target-dir.sh" "$RACE" "$BASE_KEY" "$root" jobB > "$scratch/logB" 2>&1 ) & wait race_dir="$root/target-$RACE" assert_content "$race_dir/debug/deps/libx.rlib" race-source "the surviving directory is complete" [ -z "$(find "$root" -maxdepth 1 -name '.stage-*' -print -quit)" ] || fail "a staging directory was left behind" ok "no staging directory survived the race" # Exactly one job may claim it seeded; the other must report either the # concurrent-peer path or a plain reuse (if it started after the winner # finished). Neither may report a cold start. if grep -q 'starts cold' "$scratch/logA" "$scratch/logB"; then cat "$scratch/logA" "$scratch/logB"; fail "a racing job reported a cold start" fi ok "neither racing job fell through to a cold start" echo echo "=== 8a: a rotation landing inside the clone's identity window ===" # The race scenario 7 does NOT cover: a seed racing a publisher on its SOURCE # rather than two seeds racing on a DESTINATION. What this pins is that a # consumer whose source is REPLACED WHOLESALE mid-clone notices the # substitution and re-clones, ending up holding the whole generation now # published — NOT that the rotation necessarily interfered with the copy, # which is not guaranteed and is what made the racing version of this # scenario flaky (issue #3). # # The interleaving is forced rather than hoped for, the way # prune-cache-selftest.sh's scenario 12 forces a marker into the # check-to-unlink window: the consumer's own `cp` performs the rotation, so # it lands strictly after the entry count and inode that open the identity # window and strictly before the inode that closes it. The publisher is the # real publish-snapshot.sh running concurrently; only WHEN it runs is # arranged. ROT=$(cache_key feat/rotate) rm -rf "$root/snapshot-$BASE_KEY" "$root/target-$BASE_KEY" # The two generations must differ in entry count, or "the seeded tree matches # the published snapshot" would hold for whichever one the consumer ended up # with and the fixture could not distinguish the outcomes at all. make_wide_tree "$root/snapshot-$BASE_KEY" gen1 12 make_wide_tree "$root/target-$BASE_KEY" gen2 3 gen1_entries=$(tree_entries "$root/snapshot-$BASE_KEY") gen1_inode=$(stat -c '%i' "$root/snapshot-$BASE_KEY") [ "$gen1_entries" -ne "$(tree_entries "$root/target-$BASE_KEY")" ] \ || fail "both generations have ${gen1_entries} entries — this fixture cannot tell them apart" cat > "$scratch/bin/cp" < "$scratch/rotated" rc=0; "$real_cp" "\$@" || rc=\$? # Concurrently: the publisher's post-swap drain wait is against THIS # consumer's marker, which is held until the identity read that follows # this cp returns, so running the publish inline would deadlock the two # sides against each other. ( bash "$script_dir/publish-snapshot.sh" "$BASE_KEY" "$root" pubRot > "$scratch/logPub" 2>&1 echo \$? > "$scratch/rcPub" ) & # Hand control back only once the swap is on disk, so the identity read # immediately after this cp is guaranteed to resolve to the new generation. deadline=\$(( \$(date +%s) + 60 )) while [ "\$(stat -c '%i' "$root/snapshot-$BASE_KEY" 2>/dev/null)" = "$gen1_inode" ]; do [ "\$(date +%s)" -lt "\$deadline" ] || { echo "stub cp: the publisher never swapped the snapshot" >&2; exit 90; } sleep 0.05 done exit \$rc fi exec "$real_cp" "\$@" EOF chmod +x "$scratch/bin/cp" rcRot=0 seed_with_stub "$ROT" "$BASE_KEY" "$root" jobRot > "$scratch/logRot" 2>&1 || rcRot=$? [ -e "$scratch/rotated" ] \ || fail "the stubbed cp never fired: no rotation was placed in the window, so this scenario proves nothing" ok "the rotation was placed inside the consumer's identity window" wait_for_file "$scratch/rcPub" "the publisher never finished" [ "$(cat "$scratch/rcPub")" = "0" ] || { tail -40 "$scratch/logPub"; fail "publish-snapshot.sh exited non-zero"; } [ "$rcRot" = "0" ] || { tail -40 "$scratch/logRot"; fail "the seed exited non-zero"; } ok "the seed completed" # WHICH check caught the rotation is the point of this scenario, so it is # asserted rather than assumed. The copy succeeded and the staging tree holds # every entry the source had when the walk began, so neither cp's exit status # nor the entry count is a witness here — the source's identity changing # under the walk is the only one. Reading the report back is what keeps this # scenario sensitive to losing that single comparison. assert_tear "$scratch/logRot" 0 same differs "the substitution was caught by the source's identity alone" rot_dir="$root/target-$ROT" snap_entries=$(tree_entries "$root/snapshot-$BASE_KEY") rot_entries=$(tree_entries "$rot_dir") [ "$rot_entries" -eq "$snap_entries" ] \ || fail "the seeded tree is not the generation now published: ${rot_entries} entries against the snapshot's ${snap_entries} (generation 1 had ${gen1_entries})" ok "the seed re-cloned and holds the whole published generation (${rot_entries} entries)" assert_content "$rot_dir/debug/.fingerprint/x/dep-lib-x" gen2 "the seeded tree holds one whole generation, not a splice of two" leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' -o -name '.publish-*' \) -print) [ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}" ok "no staging, reader-marker or deferred-generation scratch left behind" rm -f "$scratch/bin/cp" echo echo "=== 8b: a subtree unlinked out from under the walk, silently ===" # The other way a clone tears, and the one with nothing to report: a subtree # that leaves the parent's listing before `cp -al` reads it is simply never # visited. The copy exits 0 and the source's identity never changes, so the # entry count taken before the walk is the only witness there is — this is # the mode that used to publish a partial tree and call it a success. # # Modelled by renaming the subtree out and back around the consumer's own cp: # out before the walk starts (the only way to be missed without an error), # back before the retry, because the real thing that removes entries — a # publish rotating a generation away — has a whole generation at the path by # the time the retry looks. TRUNC=$(cache_key feat/truncate) TRUNC_BASE=$(cache_key release/1) make_wide_tree "$root/snapshot-$TRUNC_BASE" trunkgen 6 victim="$root/snapshot-$TRUNC_BASE/debug/deps/d3" trunc_entries=$(tree_entries "$root/snapshot-$TRUNC_BASE") [ "$(tree_entries "$victim")" -gt 0 ] \ || fail "the subtree this scenario removes is empty — its removal would not change the entry count" cat > "$scratch/bin/cp" < "$scratch/unlinked" mv "$victim" "$scratch/held" rc=0; "$real_cp" "\$@" || rc=\$? mv "$scratch/held" "$victim" exit \$rc fi exec "$real_cp" "\$@" EOF chmod +x "$scratch/bin/cp" rcTrunc=0 seed_with_stub "$TRUNC" "$TRUNC_BASE" "$root" jobTrunc > "$scratch/logTrunc" 2>&1 || rcTrunc=$? [ -e "$scratch/unlinked" ] \ || fail "the stubbed cp never fired: nothing was unlinked mid-walk, so this scenario proves nothing" ok "a subtree was taken out of the source's listing before the walk read it" [ "$rcTrunc" = "0" ] || { tail -40 "$scratch/logTrunc"; fail "the seed exited non-zero"; } ok "the seed completed" assert_tear "$scratch/logTrunc" 0 short same "the silent truncation was caught by the entry count alone" trunc_dir="$root/target-$TRUNC" [ "$(tree_entries "$trunc_dir")" -eq "$trunc_entries" ] \ || fail "the seeded tree is short: $(tree_entries "$trunc_dir") entries against the source's ${trunc_entries}" ok "the seed re-cloned and holds every entry the source has (${trunc_entries})" [ -e "$victim" ] || fail "the fixture did not put the subtree back" [ -e "$trunc_dir/debug/deps/d3/f0" ] \ || fail "the subtree missed by the first walk is absent from the seeded tree" ok "the subtree the first walk never saw is present in the seeded tree" leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' \) -print) [ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}" ok "no staging or reader-marker scratch left behind" rm -f "$scratch/bin/cp" echo echo "=== 8c: a copy that reports failure over a tree that looks whole ===" # The third witness, and the only one the tool volunteers rather than leaving # to be inferred: `cp -al` exiting non-zero. What this pins is that a copy's # own failure report is never overruled by the other two checks agreeing that # the tree looks intact. # # Both of those are inferences with blind spots. The entry count is # `find | wc -l` over two trees, so a failure below a directory NEITHER walk # could enumerate moves it not at all, and an entry lost against one gained # cancels out. The inode comparison sees only a source replaced wholesale. A # `cp -a` that links every entry and still fails — it could not preserve a # directory's ownership, say — is invisible to both, and that is the shape # forced here: the report reads 0 entries short and one unchanged inode, so # the exit status is the only witness there is. # # Forced with the PATH stub the suite already uses for 8a and 8b, on the # consumer's own top-level clone: the whole tree really is copied, and then # the failure is reported, once. The retry finds nothing wrong, so the # scenario also pins that the seed RECOVERS rather than merely refusing — # a status that is honoured but not survivable would fail every job whose # runner hiccuped once. CPSTAT=$(cache_key feat/cp-status) CPSTAT_BASE=$(cache_key release/2) make_wide_tree "$root/snapshot-$CPSTAT_BASE" cpstatgen 4 cpstat_entries=$(tree_entries "$root/snapshot-$CPSTAT_BASE") cat > "$scratch/bin/cp" < "$scratch/cp_reported_failure" "$real_cp" "\$@" echo "cp: failed to preserve ownership for '\${@: -1}': Operation not permitted" >&2 exit 1 fi exec "$real_cp" "\$@" EOF chmod +x "$scratch/bin/cp" rcCpStat=0 seed_with_stub "$CPSTAT" "$CPSTAT_BASE" "$root" jobCpStat > "$scratch/logCpStat" 2>&1 || rcCpStat=$? [ -e "$scratch/cp_reported_failure" ] \ || fail "the stubbed cp never fired: no copy reported failure, so this scenario proves nothing" ok "a copy reported failure over a staging tree that was in fact complete" [ "$rcCpStat" = "0" ] || { tail -40 "$scratch/logCpStat"; fail "the seed exited non-zero"; } ok "the seed completed" # `same` on both inferences is the whole point: neither of them saw anything, # so deleting the exit-status check leaves nothing to report and this # assertion is the one that goes red. assert_tear "$scratch/logCpStat" 1 same same "the failure was caught by the copy's exit status alone" cpstat_dir="$root/target-$CPSTAT" [ "$(tree_entries "$cpstat_dir")" -eq "$cpstat_entries" ] \ || fail "the seeded tree is short: $(tree_entries "$cpstat_dir") entries against the source's ${cpstat_entries}" ok "the seed re-cloned and holds every entry the source has (${cpstat_entries})" assert_content "$cpstat_dir/debug/.fingerprint/x/dep-lib-x" cpstatgen "the seeded tree is the source's content" leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' \) -print) [ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}" ok "no staging or reader-marker scratch left behind" rm -f "$scratch/bin/cp" echo echo "=== 8d: an identity that could not be read at either end ===" # The fourth check, `[ "$i_before" != missing ]`, and the one whose absence is # hardest to see. `_dir_inode` folds every stat failure into the string # `missing`, so two FAILED identity reads compare equal TO EACH OTHER: drop # this check and a clone whose source could not be identified before or after # the walk satisfies `i_before = i_after` on the strength of two errors, and # is renamed into place having proven nothing about the tree it holds. # # It cannot be forced by taking the source away. A source that is really gone # fails `cp -al` too, so the exit status would fire in this check's place and # the scenario would pin 8c's property over again. What isolates it is the # identity read failing while the copy SUCCEEDS — a transient stat error over # a source that is otherwise perfectly readable — so the stub goes on `stat` # rather than on the tree, and narrowly: only the `%i` reads of this clone's # own source, which is the sole caller of `stat -c '%i'` on this path. The # `%Y` reads the reader markers do are left alone. # # BOTH reads of the attempt have to fail. If only one did, the surviving one # would differ from `missing` and the identity COMPARISON would become the # witness instead — 8a's property, not this one. The assertion that the stub # fired is therefore a count rather than a flag. IDENT=$(cache_key feat/identity-unreadable) IDENT_BASE=$(cache_key release/3) make_wide_tree "$root/snapshot-$IDENT_BASE" identgen 4 ident_src="$root/snapshot-$IDENT_BASE" ident_entries=$(tree_entries "$ident_src") cat > "$scratch/bin/stat" </dev/null || echo 0)" -lt 2 ]; then printf 'x' >> "$scratch/identity_reads_failed" echo "stat: cannot statx '\$3': Input/output error" >&2 exit 1 fi exec "$real_stat" "\$@" EOF chmod +x "$scratch/bin/stat" rcIdent=0 seed_with_stub "$IDENT" "$IDENT_BASE" "$root" jobIdent > "$scratch/logIdent" 2>&1 || rcIdent=$? ident_failures=$(wc -c < "$scratch/identity_reads_failed" 2>/dev/null || echo 0) [ "$ident_failures" -eq 2 ] \ || fail "expected both identity reads of one attempt to fail, got ${ident_failures} — a single failure would make the identity comparison the witness instead, which is scenario 8a's property" ok "neither identity read of the first attempt resolved" [ "$rcIdent" = "0" ] || { tail -40 "$scratch/logIdent"; fail "the seed exited non-zero"; } ok "the seed completed" assert_tear "$scratch/logIdent" 0 same unreadable "the unreadable identity was caught by the sentinel check alone" ident_dir="$root/target-$IDENT" [ "$(tree_entries "$ident_dir")" -eq "$ident_entries" ] \ || fail "the seeded tree is short: $(tree_entries "$ident_dir") entries against the source's ${ident_entries}" ok "the seed re-cloned once the identity was readable again (${ident_entries} entries)" assert_content "$ident_dir/debug/.fingerprint/x/dep-lib-x" identgen "the seeded tree is the source's content" leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' \) -print) [ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}" ok "no staging or reader-marker scratch left behind" rm -f "$scratch/bin/stat" echo echo "=== 9: a source that cannot be read fails loudly ===" rc=0 hardlink_clone_into "$root/nosuch-source" "$root/target-nosuch" nosuch-tag > "$scratch/logMissing" 2>&1 || rc=$? [ "$rc" -eq 2 ] || fail "expected status 2 for an unreadable source, got ${rc}" ok "an unreadable source returns the distinct hard-failure status" assert_absent "$root/target-nosuch" "nothing was renamed into place" # The status only matters if the script acts on it. A source that exists but # cannot be read exercises the whole path: retries exhaust, the function # returns 2, and seed-target-dir.sh must exit non-zero rather than falling # through to its cold-start branch. if [ "$(id -u)" = "0" ]; then echo "SKIP: running as root — mode bits do not deny access" else UNREADABLE=$(cache_key feat/unreadable) VICTIM=$(cache_key feat/victim) make_tree "$root/snapshot-$UNREADABLE" locked-away chmod 000 "$root/snapshot-$UNREADABLE" rc=0 CACHE_CLONE_ATTEMPTS=2 bash "$script_dir/seed-target-dir.sh" \ "$VICTIM" "$UNREADABLE" "$root" jobUnread > "$scratch/logUnread" 2>&1 || rc=$? chmod 755 "$root/snapshot-$UNREADABLE" [ "$rc" -ne 0 ] || { tail -20 "$scratch/logUnread"; fail "the seed reported success against a source it could not read"; } ok "the seed exits non-zero when its source cannot be cloned" grep -q 'refusing to build against a partial cache' "$scratch/logUnread" \ || { tail -20 "$scratch/logUnread"; fail "the failure was not reported as such"; } ok "the failure names the reason rather than exiting silently" if grep -q 'starts cold' "$scratch/logUnread"; then tail -20 "$scratch/logUnread"; fail "an unreadable source degraded into a silent cold build" fi ok "it does not degrade into a silent cold build" assert_absent "$root/target-$VICTIM" "no target dir was left behind by the failed seed" fi echo echo "=== 10: a staging tree that cannot be privately owned is never published ===" # unshare_mutable_paths turns the hardlinked copies of Cargo's mutable # metadata back into private inodes, and its failure status is load-bearing in # a way no other check covers: a staging tree whose dep-info files still point # at the SOURCE's inodes is not torn — every entry is present and the source # never moved — so all four torn-clone checks pass it. Publish it anyway and # this branch's build rewrites the base branch's dep-info, which is the silent # stale-artifact reuse documented at length in cache-lib.sh. The only thing # standing between that tree and DST is the status being propagated. # # Forced with the same PATH stub shape as 8a to 8d, on the narrowest possible # target: `_unshare_files` copies each shared file as `cp -p -- # .unshare.`, so refusing exactly the `.d` copies leaves the directory # unshares (`cp -a`) and the clone itself (`cp -al`) untouched, and the # failure that reaches the clone is unambiguously this one. The dep-info file # has to live outside .fingerprint/ and build/, or it would be inside a # subtree already replaced wholesale and never reach the per-file pass. ALIAS=$(cache_key feat/unshare-fails) ALIAS_BASE=$(cache_key release/4) make_tree "$root/snapshot-$ALIAS_BASE" aliasgen alias_dep="$root/snapshot-$ALIAS_BASE/debug/deps/libx.d" echo aliasgen > "$alias_dep" cat > "$scratch/bin/cp" < "$scratch/depinfo_unshare_refused" echo "cp: cannot create regular file '\$4': No space left on device" >&2 exit 1 fi exec "$real_cp" "\$@" EOF chmod +x "$scratch/bin/cp" rcAlias=0 seed_with_stub "$ALIAS" "$ALIAS_BASE" "$root" jobAlias > "$scratch/logAlias" 2>&1 || rcAlias=$? [ -e "$scratch/depinfo_unshare_refused" ] \ || fail "the stubbed cp never fired: no unshare was refused, so this scenario proves nothing" ok "a dep-info unshare was refused inside the clone" [ "$rcAlias" != "0" ] || { tail -40 "$scratch/logAlias"; fail "the seed reported success over a staging tree that still aliased its source"; } ok "the seed exits non-zero when the staging tree cannot be privately owned" # Which failure aborted the clone is asserted, not assumed: without this the # scenario would stay green if the clone had failed for any other reason — # the same wrong-reason pass that let a mutation survive scenario 9 (issue #5). grep -q 'failed to unshare dep-info files' "$scratch/logAlias" \ || { tail -20 "$scratch/logAlias"; fail "the dep-info unshare failure was not the one reported"; } ok "the refused unshare is what was reported" grep -q 'could not privately own the mutable paths' "$scratch/logAlias" \ || { tail -20 "$scratch/logAlias"; fail "the clone did not report why it refused to publish"; } ok "the clone names aliasing as its reason for refusing" assert_absent "$root/target-$ALIAS" "nothing was renamed into place" [ "$(stat -c '%h' "$alias_dep")" = "1" ] \ || fail "the source's dep-info file is still hardlinked from somewhere: $(stat -c '%h' "$alias_dep") links" ok "the discarded staging tree took its hardlinks to the source with it" leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' \) -print) [ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}" ok "no staging or reader-marker scratch left behind" rm -f "$scratch/bin/cp" echo echo "seed-target-dir-selftest: ${pass_count} assertions passed"