#!/usr/bin/env bash
# Regression test for seed-target-dir.sh: which source a run seeds from, and
# what happens when two jobs sharing one cache key seed at the same time.
#
# Runs the ACTUAL script against a real scratch cache directory with fake
# target trees standing in for cargo output — no compiler needed, so this is
# the fast half of the suite. hardlink-clone-selftest.sh covers the parts that
# need a real build.
#
# What each scenario demonstrates:
#
#   1. BASE SNAPSHOT PREFERRED — a PR whose base has published a snapshot
#      seeds from it, and the seeded directory really is a hardlink clone
#      (shared inodes), not a copy.
#   2. OWN DIR WINS — a second run of the same ref reuses what is already
#      there and does not re-seed over its own work.
#   3. OWN SNAPSHOT AS SELF-RESTORE — a publisher whose live target dir was
#      evicted restores from the snapshot it last published, instead of
#      rebuilding cold.
#   4. FALLBACK DIR — with no snapshot at all, an explicitly configured
#      fallback (a pre-existing flat cache, during a migration) is used.
#   5. COLD — with nothing available, the directory is created empty rather
#      than the script failing.
#   6. LOCK FILES STRIPPED — Cargo's in-place-flock'd lock files never
#      survive a clone, because a shared lock inode would make two branches
#      contend on one mutex.
#   7. CONCURRENT SEED IS ATOMIC — two seeds racing on one cache key: the
#      loser discards its staging copy and uses the winner's directory, and
#      at no point is a partially-populated directory visible under the final
#      name. This is the property that replaces "the runner only has one job
#      slot" with an actual guarantee.
#  8a. SEED VS PUBLISH ROTATION — the race scenario 7 does NOT cover, and the
#      one that actually mattered: a consumer hardlink-cloning a snapshot
#      while the publisher of that snapshot rotates it. Two seeds racing on a
#      DESTINATION is a different race from a seed racing a publisher on its
#      SOURCE, and only the second one can truncate a tree. What it pins is
#      that the consumer notices its source was REPLACED and re-clones,
#      ending up with the whole generation now published — not that the
#      rotation interfered with the copy, which is not guaranteed and is what
#      made the racing version of this scenario flaky (issue #3).
#  8b. AND A SILENTLY TRUNCATED WALK — the same tear seen through the other
#      check: a subtree unlinked out of the parent's listing before `cp -al`
#      reads it is never visited, the copy exits 0 and the source's identity
#      never changes. Against the unguarded version this is a partial tree
#      renamed into place and reported as a success — 20,328 of 48,805
#      entries, `seed: cloned in 1s`, exit 0, seeded-from=base-snapshot.
#      Both scenarios force their interleaving rather than racing for it, and
#      each asserts WHICH check caught the tear, so neither stays green if
#      the check it exercises is removed.
#   9. AN UNREADABLE SOURCE FAILS LOUDLY — the clone reports a distinct status
#      instead of renaming whatever it managed to produce into place, and the
#      seed SCRIPT turns that status into a failed job rather than a silent
#      cold build. Retries are what make a torn read survivable; exhausting
#      them must not degrade into "start cold and rebuild everything", which
#      would turn a corrupt-cache bug into an invisible 4x-slower CI job.
#      (The publisher's half of the rotation race — deferring reclamation
#      while a reader is still in flight — lives in
#      publish-snapshot-selftest.sh, next to the swap it modifies.)
set -euo pipefail
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$script_dir/cache-lib.sh"

scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
root="$scratch/cache"
mkdir -p "$root"
pass_count=0

fail() { echo "ASSERTION FAILED: $*" >&2; exit 1; }
ok() { pass_count=$((pass_count + 1)); echo "PASS: $*"; }
assert_file() { [ -e "$1" ] || fail "expected $1 to exist ($2)"; ok "$2"; }
assert_absent() { [ -e "$1" ] && fail "expected $1 to be gone ($2)"; ok "$2"; }
assert_content() { [ "$(cat "$1")" = "$2" ] || fail "expected '$2' in $1, got '$(cat "$1")' ($3)"; ok "$3"; }

# A plausible target tree: a big shared artifact, a mutable fingerprint, a
# build-script output, and a lock file.
make_tree() {
  local d="$1" marker="$2"
  mkdir -p "$d/debug/deps" "$d/debug/.fingerprint/x" "$d/debug/build/x/out"
  echo "$marker" > "$d/debug/deps/libx.rlib"
  echo "$marker" > "$d/debug/.fingerprint/x/dep-lib-x"
  echo "$marker" > "$d/debug/build/x/out/gen.txt"
  : > "$d/debug/.cargo-lock"
}

seed() { bash "$script_dir/seed-target-dir.sh" "$@" > "$scratch/log" 2>&1 || { tail -40 "$scratch/log"; fail "seed-target-dir.sh exited non-zero"; }; }

# The same script, with the scenario-8 stub directory ahead of the real
# coreutils on PATH. Kept separate so no other scenario can pick a stub up by
# accident, and so the caller keeps the exit status instead of aborting on it.
seed_with_stub() { PATH="$scratch/bin:$PATH" bash "$script_dir/seed-target-dir.sh" "$@"; }

# Always succeeds and always prints a number: a directory that does not exist
# is 0 entries, not an error worth aborting the suite over.
tree_entries() {
  local n
  n=$(find "$1" -mindepth 1 2>/dev/null | wc -l) || n=0
  printf '%s\n' "$n"
  return 0
}

# Waits for a file a concurrently running script will create. The publisher in
# scenario 8a is started from inside the consumer's process tree rather than
# by this script, so its completion cannot be waited on as a job.
wait_for_file() {
  local path="$1" what="$2" deadline
  deadline=$(( $(date +%s) + 60 ))
  until [ -e "$path" ]; do
    [ "$(date +%s)" -lt "$deadline" ] || fail "$what"
    sleep 0.1
  done
}

# Reads the clone's own torn-read report back and asserts WHICH of its three
# checks fired: the copy's exit status, the entry count, or the source's
# identity. Scenarios 8a and 8b each force exactly one of the three, and a
# scenario that only asserted "some tear was reported" would stay green if
# the check it exercises were deleted and one of the others happened to fire
# in its place.
#
#   assert_tear <log> <cp-status> <same|short> <same|differs> <message>
assert_tear() {
  local log="$1" want_rc="$2" want_count="$3" want_inode="$4" msg="$5" line
  local re='cp rc=([0-9]+), ([0-9]+)/([0-9]+) entries, source inode ([0-9]+) -> ([0-9]+)'
  # `|| line=""` rather than a bare assignment: no match makes grep exit 1,
  # which under `set -e` would abort the suite with no message at all — the
  # exact case this assertion exists to report.
  line=$(grep -o 'was torn ([^)]*)' "$log" | head -1) || line=""
  [ -n "$line" ] || { tail -20 "$log"; fail "no torn read was reported at all ($msg)"; }
  [[ $line =~ $re ]] || fail "unrecognised torn-read report: ${line}"
  [ "${BASH_REMATCH[1]}" = "$want_rc" ] || fail "expected cp to exit ${want_rc}: ${line}"
  case "$want_count" in
    same)  [ "${BASH_REMATCH[2]}" -eq "${BASH_REMATCH[3]}" ] || fail "expected a whole staging tree: ${line}" ;;
    short) [ "${BASH_REMATCH[2]}" -lt "${BASH_REMATCH[3]}" ] || fail "expected a short staging tree: ${line}" ;;
    *) fail "assert_tear: bad entry-count expectation '${want_count}'" ;;
  esac
  case "$want_inode" in
    same)    [ "${BASH_REMATCH[4]}" = "${BASH_REMATCH[5]}" ] || fail "expected the source's identity to hold: ${line}" ;;
    differs) [ "${BASH_REMATCH[4]}" != "${BASH_REMATCH[5]}" ] || fail "expected the source's identity to change: ${line}" ;;
    *) fail "assert_tear: bad identity expectation '${want_inode}'" ;;
  esac
  ok "$msg (${line})"
}

# A tree of many sibling subtrees, built by cloning one small template
# directory N times — N forks rather than N*M file creations. The shape is
# what scenarios 8a and 8b need: two generations of DIFFERENT entry count, so
# an assertion can tell which one a consumer ended up with, and enough
# sibling subtrees that removing one moves the count. Size no longer has to
# buy a race window — both scenarios force their interleaving — so it stays
# small enough to be free.
make_wide_tree() {
  local d="$1" marker="$2" ndirs="$3" i
  mkdir -p "$d/debug/deps/.tmpl" "$d/debug/.fingerprint/x"
  for i in $(seq 0 59); do echo "$marker" > "$d/debug/deps/.tmpl/f$i"; done
  for i in $(seq -w 1 "$ndirs"); do cp -al "$d/debug/deps/.tmpl" "$d/debug/deps/d$i"; done
  rm -rf "$d/debug/deps/.tmpl"
  echo "$marker" > "$d/debug/.fingerprint/x/dep-lib-x"
}

BASE_KEY=$(cache_key dev)
OWN_KEY=$(cache_key feat/thing)

echo "=== 1: base snapshot preferred, and cloned by hardlink ==="
make_tree "$root/snapshot-$BASE_KEY" base-content
seed "$OWN_KEY" "$BASE_KEY" "$root" job1
own="$root/target-$OWN_KEY"
assert_content "$own/debug/deps/libx.rlib" base-content "seeded from the base snapshot"
[ "$(stat -c '%i' "$own/debug/deps/libx.rlib")" = "$(stat -c '%i' "$root/snapshot-$BASE_KEY/debug/deps/libx.rlib")" ] \
  || fail "artifact was copied, not hardlinked"
ok "artifact shares an inode with the snapshot (hardlink clone, not a copy)"
[ "$(stat -c '%i' "$own/debug/.fingerprint/x/dep-lib-x")" != "$(stat -c '%i' "$root/snapshot-$BASE_KEY/debug/.fingerprint/x/dep-lib-x")" ] \
  || fail "fingerprint still shares an inode with the snapshot"
ok "fingerprint is privately owned (unshare_mutable_paths ran)"

echo
echo "=== 6: Cargo lock files never survive a clone ==="
assert_absent "$own/debug/.cargo-lock" "cloned .cargo-lock removed"

echo
echo "=== 2: an existing own dir is reused, never re-seeded over ==="
echo own-work > "$own/debug/deps/libx.rlib"
seed "$OWN_KEY" "$BASE_KEY" "$root" job1
assert_content "$own/debug/deps/libx.rlib" own-work "own directory reused as-is"
grep -q 'reusing this ref' "$scratch/log" || fail "expected the reuse path in the log"
ok "reuse is reported in the log"

echo
echo "=== 3: a publisher restores from its own snapshot after eviction ==="
make_tree "$root/snapshot-$BASE_KEY" published-dev
seed "$BASE_KEY" "" "$root" job1
assert_content "$root/target-$BASE_KEY/debug/deps/libx.rlib" published-dev "publisher self-restored from its own snapshot"

echo
echo "=== 4: explicit fallback dir when no snapshot exists ==="
OTHER=$(cache_key feat/other)
make_tree "$scratch/legacy-flat" legacy
seed "$OTHER" "$(cache_key nosuch)" "$root" job1 "$scratch/legacy-flat"
assert_content "$root/target-$OTHER/debug/deps/libx.rlib" legacy "seeded from the fallback dir"

echo
echo "=== 5: cold start when nothing is available ==="
COLD=$(cache_key feat/cold)
seed "$COLD" "$(cache_key nosuch)" "$root" job1
[ -d "$root/target-$COLD" ] || fail "cold start did not create the directory"
[ -z "$(ls -A "$root/target-$COLD")" ] || fail "cold start directory is not empty"
ok "cold start creates an empty directory rather than failing"

echo
echo "=== 7: two jobs racing on one cache key ==="
RACE=$(cache_key feat/race)
make_tree "$root/snapshot-$BASE_KEY" race-source
# Both jobs seed concurrently from the same snapshot into the same key. Each
# stages under its own tag, so the only interaction is the final rename.
( bash "$script_dir/seed-target-dir.sh" "$RACE" "$BASE_KEY" "$root" jobA > "$scratch/logA" 2>&1 ) &
( bash "$script_dir/seed-target-dir.sh" "$RACE" "$BASE_KEY" "$root" jobB > "$scratch/logB" 2>&1 ) &
wait
race_dir="$root/target-$RACE"
assert_content "$race_dir/debug/deps/libx.rlib" race-source "the surviving directory is complete"
[ -z "$(find "$root" -maxdepth 1 -name '.stage-*' -print -quit)" ] || fail "a staging directory was left behind"
ok "no staging directory survived the race"
# Exactly one job may claim it seeded; the other must report either the
# concurrent-peer path or a plain reuse (if it started after the winner
# finished). Neither may report a cold start.
if grep -q 'starts cold' "$scratch/logA" "$scratch/logB"; then
  cat "$scratch/logA" "$scratch/logB"; fail "a racing job reported a cold start"
fi
ok "neither racing job fell through to a cold start"

echo
echo "=== 8a: a rotation landing inside the clone's identity window ==="
# The race scenario 7 does NOT cover: a seed racing a publisher on its SOURCE
# rather than two seeds racing on a DESTINATION. What this pins is that a
# consumer whose source is REPLACED WHOLESALE mid-clone notices the
# substitution and re-clones, ending up holding the whole generation now
# published — NOT that the rotation necessarily interfered with the copy,
# which is not guaranteed and is what made the racing version of this
# scenario flaky (issue #3).
#
# The interleaving is forced rather than hoped for, the way
# prune-cache-selftest.sh's scenario 12 forces a marker into the
# check-to-unlink window: the consumer's own `cp` performs the rotation, so
# it lands strictly after the entry count and inode that open the identity
# window and strictly before the inode that closes it. The publisher is the
# real publish-snapshot.sh running concurrently; only WHEN it runs is
# arranged.
ROT=$(cache_key feat/rotate)
rm -rf "$root/snapshot-$BASE_KEY" "$root/target-$BASE_KEY"
# The two generations must differ in entry count, or "the seeded tree matches
# the published snapshot" would hold for whichever one the consumer ended up
# with and the fixture could not distinguish the outcomes at all.
make_wide_tree "$root/snapshot-$BASE_KEY" gen1 12
make_wide_tree "$root/target-$BASE_KEY" gen2 3
gen1_entries=$(tree_entries "$root/snapshot-$BASE_KEY")
gen1_inode=$(stat -c '%i' "$root/snapshot-$BASE_KEY")
[ "$gen1_entries" -ne "$(tree_entries "$root/target-$BASE_KEY")" ] \
  || fail "both generations have ${gen1_entries} entries — this fixture cannot tell them apart"

mkdir -p "$scratch/bin"
real_cp=$(command -v cp)
cat > "$scratch/bin/cp" <<EOF
#!/usr/bin/env bash
# Fires once, and only on the consumer's own top-level hardlink clone —
# identified by its destination, which is that seed's private staging path.
# Every other cp in the process tree (the unshare copies, and the publisher's
# own staging clone) falls through to the real one.
if [ "\${@: -1}" = "$root/.stage-jobRot" ] && [ ! -e "$scratch/rotated" ]; then
  : > "$scratch/rotated"
  rc=0; "$real_cp" "\$@" || rc=\$?
  # Concurrently: the publisher's post-swap drain wait is against THIS
  # consumer's marker, which is held until the identity read that follows
  # this cp returns, so running the publish inline would deadlock the two
  # sides against each other.
  ( bash "$script_dir/publish-snapshot.sh" "$BASE_KEY" "$root" pubRot > "$scratch/logPub" 2>&1
    echo \$? > "$scratch/rcPub" ) &
  # Hand control back only once the swap is on disk, so the identity read
  # immediately after this cp is guaranteed to resolve to the new generation.
  deadline=\$(( \$(date +%s) + 60 ))
  while [ "\$(stat -c '%i' "$root/snapshot-$BASE_KEY" 2>/dev/null)" = "$gen1_inode" ]; do
    [ "\$(date +%s)" -lt "\$deadline" ] || { echo "stub cp: the publisher never swapped the snapshot" >&2; exit 90; }
    sleep 0.05
  done
  exit \$rc
fi
exec "$real_cp" "\$@"
EOF
chmod +x "$scratch/bin/cp"

rcRot=0
seed_with_stub "$ROT" "$BASE_KEY" "$root" jobRot > "$scratch/logRot" 2>&1 || rcRot=$?
[ -e "$scratch/rotated" ] \
  || fail "the stubbed cp never fired: no rotation was placed in the window, so this scenario proves nothing"
ok "the rotation was placed inside the consumer's identity window"
wait_for_file "$scratch/rcPub" "the publisher never finished"
[ "$(cat "$scratch/rcPub")" = "0" ] || { tail -40 "$scratch/logPub"; fail "publish-snapshot.sh exited non-zero"; }
[ "$rcRot" = "0" ] || { tail -40 "$scratch/logRot"; fail "the seed exited non-zero"; }
ok "the seed completed"

# WHICH check caught the rotation is the point of this scenario, so it is
# asserted rather than assumed. The copy succeeded and the staging tree holds
# every entry the source had when the walk began, so neither cp's exit status
# nor the entry count is a witness here — the source's identity changing
# under the walk is the only one. Reading the report back is what keeps this
# scenario sensitive to losing that single comparison.
assert_tear "$scratch/logRot" 0 same differs "the substitution was caught by the source's identity alone"

rot_dir="$root/target-$ROT"
snap_entries=$(tree_entries "$root/snapshot-$BASE_KEY")
rot_entries=$(tree_entries "$rot_dir")
[ "$rot_entries" -eq "$snap_entries" ] \
  || fail "the seeded tree is not the generation now published: ${rot_entries} entries against the snapshot's ${snap_entries} (generation 1 had ${gen1_entries})"
ok "the seed re-cloned and holds the whole published generation (${rot_entries} entries)"
assert_content "$rot_dir/debug/.fingerprint/x/dep-lib-x" gen2 "the seeded tree holds one whole generation, not a splice of two"
leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' -o -name '.publish-*' \) -print)
[ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}"
ok "no staging, reader-marker or deferred-generation scratch left behind"
rm -f "$scratch/bin/cp"

echo
echo "=== 8b: a subtree unlinked out from under the walk, silently ==="
# The other way a clone tears, and the one with nothing to report: a subtree
# that leaves the parent's listing before `cp -al` reads it is simply never
# visited. The copy exits 0 and the source's identity never changes, so the
# entry count taken before the walk is the only witness there is — this is
# the mode that used to publish a partial tree and call it a success.
#
# Modelled by renaming the subtree out and back around the consumer's own cp:
# out before the walk starts (the only way to be missed without an error),
# back before the retry, because the real thing that removes entries — a
# publish rotating a generation away — has a whole generation at the path by
# the time the retry looks.
TRUNC=$(cache_key feat/truncate)
TRUNC_BASE=$(cache_key release/1)
make_wide_tree "$root/snapshot-$TRUNC_BASE" trunkgen 6
victim="$root/snapshot-$TRUNC_BASE/debug/deps/d3"
trunc_entries=$(tree_entries "$root/snapshot-$TRUNC_BASE")
[ "$(tree_entries "$victim")" -gt 0 ] \
  || fail "the subtree this scenario removes is empty — its removal would not change the entry count"

cat > "$scratch/bin/cp" <<EOF
#!/usr/bin/env bash
if [ "\${@: -1}" = "$root/.stage-jobTrunc" ] && [ ! -e "$scratch/unlinked" ]; then
  : > "$scratch/unlinked"
  mv "$victim" "$scratch/held"
  rc=0; "$real_cp" "\$@" || rc=\$?
  mv "$scratch/held" "$victim"
  exit \$rc
fi
exec "$real_cp" "\$@"
EOF
chmod +x "$scratch/bin/cp"

rcTrunc=0
seed_with_stub "$TRUNC" "$TRUNC_BASE" "$root" jobTrunc > "$scratch/logTrunc" 2>&1 || rcTrunc=$?
[ -e "$scratch/unlinked" ] \
  || fail "the stubbed cp never fired: nothing was unlinked mid-walk, so this scenario proves nothing"
ok "a subtree was taken out of the source's listing before the walk read it"
[ "$rcTrunc" = "0" ] || { tail -40 "$scratch/logTrunc"; fail "the seed exited non-zero"; }
ok "the seed completed"
assert_tear "$scratch/logTrunc" 0 short same "the silent truncation was caught by the entry count alone"

trunc_dir="$root/target-$TRUNC"
[ "$(tree_entries "$trunc_dir")" -eq "$trunc_entries" ] \
  || fail "the seeded tree is short: $(tree_entries "$trunc_dir") entries against the source's ${trunc_entries}"
ok "the seed re-cloned and holds every entry the source has (${trunc_entries})"
[ -e "$victim" ] || fail "the fixture did not put the subtree back"
[ -e "$trunc_dir/debug/deps/d3/f0" ] \
  || fail "the subtree missed by the first walk is absent from the seeded tree"
ok "the subtree the first walk never saw is present in the seeded tree"
leftovers=$(find "$root" -maxdepth 1 \( -name '.stage-*' -o -name '.reading-*' \) -print)
[ -z "$leftovers" ] || fail "scratch left behind: ${leftovers}"
ok "no staging or reader-marker scratch left behind"
rm -f "$scratch/bin/cp"
echo
echo "=== 9: a source that cannot be read fails loudly ==="
rc=0
hardlink_clone_into "$root/nosuch-source" "$root/target-nosuch" nosuch-tag > "$scratch/logMissing" 2>&1 || rc=$?
[ "$rc" -eq 2 ] || fail "expected status 2 for an unreadable source, got ${rc}"
ok "an unreadable source returns the distinct hard-failure status"
assert_absent "$root/target-nosuch" "nothing was renamed into place"

# The status only matters if the script acts on it. A source that exists but
# cannot be read exercises the whole path: retries exhaust, the function
# returns 2, and seed-target-dir.sh must exit non-zero rather than falling
# through to its cold-start branch.
if [ "$(id -u)" = "0" ]; then
  echo "SKIP: running as root — mode bits do not deny access"
else
  UNREADABLE=$(cache_key feat/unreadable)
  VICTIM=$(cache_key feat/victim)
  make_tree "$root/snapshot-$UNREADABLE" locked-away
  chmod 000 "$root/snapshot-$UNREADABLE"
  rc=0
  CACHE_CLONE_ATTEMPTS=2 bash "$script_dir/seed-target-dir.sh" \
    "$VICTIM" "$UNREADABLE" "$root" jobUnread > "$scratch/logUnread" 2>&1 || rc=$?
  chmod 755 "$root/snapshot-$UNREADABLE"
  [ "$rc" -ne 0 ] || { tail -20 "$scratch/logUnread"; fail "the seed reported success against a source it could not read"; }
  ok "the seed exits non-zero when its source cannot be cloned"
  grep -q 'refusing to build against a partial cache' "$scratch/logUnread" \
    || { tail -20 "$scratch/logUnread"; fail "the failure was not reported as such"; }
  ok "the failure names the reason rather than exiting silently"
  if grep -q 'starts cold' "$scratch/logUnread"; then
    tail -20 "$scratch/logUnread"; fail "an unreadable source degraded into a silent cold build"
  fi
  ok "it does not degrade into a silent cold build"
  assert_absent "$root/target-$VICTIM" "no target dir was left behind by the failed seed"
fi

echo
echo "seed-target-dir-selftest: ${pass_count} assertions passed"