claude
1f42064d20
publish-snapshot.sh's header described the silent-truncation failure mode as the one "this script's own selftest (scenario 8)" reproduces. Wrong twice: the scenario that reproduces a truncated clone is in seed-target-dir-selftest.sh, and since #4 split the old scenario 8 into 8a and 8b it is 8b. publish-snapshot-selftest.sh does have a scenario 8 — the abandoned-marker case — so the pointer landed on a real scenario with a plausible number that tests something else. Names the suite as well as the number, and says what the local scenario 8 actually is so the collision cannot re-form. Closes #6
114 lines
5.6 KiB
Bash
Executable File
114 lines
5.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Publish side: atomically republish this reference branch's live target dir
|
|
# as the immutable snapshot other branches seed from.
|
|
#
|
|
# Usage: publish-snapshot.sh <own-key> <cache-root> <tag>
|
|
#
|
|
# Run only after the build has already succeeded, and only on a branch that is
|
|
# a publisher (see the cargo-cache-publish action, which makes that decision).
|
|
# A red build must never overwrite a known-good snapshot: the caller's step
|
|
# ordering is the whole gate here, so keep this step last.
|
|
#
|
|
# The swap is two renames, not one, because POSIX rename() can only replace an
|
|
# EMPTY directory and a snapshot from a prior publish is not one:
|
|
#
|
|
# 1. stage the new snapshot at .stage-<tag> (copy time is off every
|
|
# consumer's hot path — nothing reads a staging path);
|
|
# 2. rename the current snapshot aside to .publish-old-<tag>, if present;
|
|
# 3. rename the staged snapshot into place.
|
|
#
|
|
# Step 3 is a single atomic rename onto a path now guaranteed absent, so it
|
|
# can never partially overwrite a live snapshot. A consumer landing in the
|
|
# window between 2 and 3 sees the snapshot as absent and falls through to its
|
|
# own cold-start path — a safe degrade that self-heals on its next run, not
|
|
# corruption.
|
|
#
|
|
# `rm -rf` on the old snapshot removes directory entries only, so a consumer
|
|
# that has ALREADY FINISHED cloning from it keeps every inode alive through
|
|
# its own links. That is the easy half, and on its own it is not enough: a
|
|
# consumer still WALKING the old generation has its entries unlinked out from
|
|
# under it, and `cp -al` does not report a subtree that was removed before it
|
|
# read the parent's listing. That is a silently truncated clone — the failure
|
|
# mode seed-target-dir-selftest.sh's scenario 8b reproduces from the consumer's
|
|
# side, against the unguarded version. (This script's own selftest also has a
|
|
# scenario 8; that one is the abandoned-reader-marker case, not this.)
|
|
#
|
|
# So the unlink is interlocked with the consume side rather than
|
|
# unconditional: after the swap, this script waits for every in-flight reader
|
|
# of this snapshot to drain (see the reader-marker ordering proof in
|
|
# cache-lib.sh) and only then reclaims the rotated-away generation. If the
|
|
# grace period expires first, reclamation is DEFERRED — the directory is left
|
|
# under `.publish-old-<key>-<tag>` and swept by a later publish once no reader
|
|
# holds it. The residual of a very slow consumer is therefore one extra
|
|
# generation of directory entries on disk, never a torn clone.
|
|
set -euo pipefail
|
|
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cache-lib.sh"
|
|
|
|
OWN_KEY="${1:?usage: publish-snapshot.sh <own-key> <cache-root> <tag>}"
|
|
ROOT="${2:?}"; TAG="${3:?}"
|
|
|
|
SRC=$(target_dir_for "$ROOT" "$OWN_KEY")
|
|
DST=$(snapshot_dir_for "$ROOT" "$OWN_KEY")
|
|
# Keyed by cache key as well as tag, so a deferred generation can be matched
|
|
# back to the snapshot whose readers must drain before it is safe to reclaim.
|
|
OLD="${ROOT}/.publish-old-${OWN_KEY}-${TAG}"
|
|
SNAP_NAME=$(basename "$DST")
|
|
GRACE="${CACHE_READ_GRACE_SECONDS}"
|
|
|
|
if [ ! -d "$SRC" ]; then
|
|
echo "publish: no target dir at ${SRC} — nothing to snapshot"
|
|
exit 0
|
|
fi
|
|
|
|
# Leftovers from a run cancelled mid-swap. Scoped to this job's own tag so a
|
|
# concurrently running job's staging directory is never touched.
|
|
rm -rf "${ROOT}/.stage-${TAG}" "$OLD"
|
|
|
|
# Deferred reclamation from an earlier publish of THIS snapshot whose readers
|
|
# had not drained in time. Safe to sweep now only if nothing is reading the
|
|
# snapshot at all: a reader holds the snapshot path, not the deferred name, so
|
|
# "no readers of snapshot-<key>" is the condition that makes every deferred
|
|
# generation of it unreachable. Over-conservative by design — a reader of the
|
|
# CURRENT generation also defers the sweep to the next publish, which costs a
|
|
# directory listing, not correctness.
|
|
if [ "$(live_reader_count "$ROOT" "$SNAP_NAME")" -eq 0 ]; then
|
|
for stale_old in "${ROOT}/.publish-old-${OWN_KEY}-"*; do
|
|
[ -d "$stale_old" ] || continue
|
|
echo "publish: reclaiming deferred snapshot generation $(basename "$stale_old")"
|
|
rm -rf "$stale_old"
|
|
done
|
|
fi
|
|
|
|
start=$(date +%s)
|
|
# The staged snapshot is hardlinked to SRC's artifacts and holds its OWN copy
|
|
# of every file Cargo rewrites in place (unshare_mutable_paths, called inside
|
|
# hardlink_clone_into). Without that, this branch's NEXT build would mutate
|
|
# the snapshot it just published — the same aliasing hazard the consume side
|
|
# closes, pointing the other way. The staging path is not the final name, so
|
|
# `hardlink_clone_into`'s rename lands on DST only after OLD is out of the way.
|
|
TMP_DST="${ROOT}/.publish-new-${TAG}"
|
|
rm -rf "$TMP_DST"
|
|
hardlink_clone_into "$SRC" "$TMP_DST" "$TAG" || {
|
|
echo "::error::publish: failed to stage a snapshot of ${SRC}"
|
|
exit 1
|
|
}
|
|
|
|
if [ -d "$DST" ]; then mv -T "$DST" "$OLD"; fi
|
|
mv -T "$TMP_DST" "$DST"
|
|
|
|
# The scan below happens strictly after the rename above, which is what makes
|
|
# it impossible for a consumer holding the OLD generation to be missed: such a
|
|
# consumer resolved the path before that rename, and published its marker
|
|
# before that. See cache-lib.sh's reader-marker section.
|
|
if [ -d "$OLD" ]; then
|
|
if wait_for_readers "$ROOT" "$SNAP_NAME" "$GRACE"; then
|
|
rm -rf "$OLD"
|
|
else
|
|
echo "::warning::publish: a consumer is still cloning the previous ${SNAP_NAME} after ${GRACE}s — deferring reclamation of $(basename "$OLD") rather than unlinking a tree being read"
|
|
summary_line "- deferred reclaiming the previous \`${SNAP_NAME}\` generation (a consumer is still cloning it); it will be swept by a later publish"
|
|
fi
|
|
fi
|
|
|
|
echo "publish: ${DST} ($(usage_gb "$DST") GB) published in $(( $(date +%s) - start ))s"
|
|
summary_line "- published cache snapshot \`$(basename "$DST")\` ($(usage_gb "$DST") GB)"
|